diff --git a/beagle/settings.py b/beagle/settings.py index fc17dd12a..dbbbe0aa6 100644 --- a/beagle/settings.py +++ b/beagle/settings.py @@ -468,4 +468,8 @@ APP_CACHE = os.environ.get("BEAGLE_APP_CACHE", "/tmp") +REFERENCE_STORAGE_ID = os.environ.get("BEAGLE_REFERENCE_STORAGE_ID") +REFERENCE_FILE_GROUP_ID = os.environ.get("BEAGLE_REFERENCE_FILE_GROUP_ID") +APP_REFERENCE_FILES_PATH = os.environ.get("BEAGLE_APP_REFERENCE_FILES_PATH", "./reference/reference.json") + CONTACT_EMAIL = os.environ.get("EVENTS_CONTACT_EMAIL", "") diff --git a/beagle_etl/celery.py b/beagle_etl/celery.py index ea9553d4a..a3da3d6f9 100644 --- a/beagle_etl/celery.py +++ b/beagle_etl/celery.py @@ -47,6 +47,7 @@ def at_start(sender, **k): "runner.tasks.terminate_job": {"queue": settings.BEAGLE_RUNNER_QUEUE}, "runner.tasks.complete_job": {"queue": settings.BEAGLE_RUNNER_QUEUE}, "runner.tasks.fail_job": {"queue": settings.BEAGLE_RUNNER_QUEUE}, + "runner.tasks.register_reference_files": {"queue": settings.BEAGLE_RUNNER_QUEUE}, "notifier.tasks.send_notification": {"queue": settings.BEAGLE_DEFAULT_QUEUE}, "file_system.tasks.populate_job_group_notifier_metadata": {"queue": settings.BEAGLE_DEFAULT_QUEUE}, "beagle_etl.tasks.job_processor": {"queue": settings.BEAGLE_DEFAULT_QUEUE}, diff --git a/beagle_etl/tests/jobs/test_metadb.py b/beagle_etl/tests/jobs/test_metadb.py index 38a01b438..c17c83ff6 100644 --- a/beagle_etl/tests/jobs/test_metadb.py +++ b/beagle_etl/tests/jobs/test_metadb.py @@ -8,6 +8,7 @@ from deepdiff import DeepDiff from django.test import TestCase from django.conf import settings +from django.db.models import signals from django.contrib.auth.models import User from beagle_etl.models import SMILEMessage from beagle_etl.models import JobGroup, JobGroupNotifier, Notifier @@ -16,9 +17,12 @@ from file_system.models import Request, Sample, Patient, FileMetadata from file_system.repository import FileRepository from study.objects import StudyObject +from runner.tasks import register_pipeline_reference_files +from runner.models import Pipeline class TestNewRequest(TestCase): + signals.post_save.disconnect(register_pipeline_reference_files, sender=Pipeline) fixtures = [ "file_system.filegroup.json", "file_system.filetype.json", diff --git a/notifier/tasks.py b/notifier/tasks.py index adba51f31..f0cedb876 100644 --- a/notifier/tasks.py +++ b/notifier/tasks.py @@ -35,7 +35,7 @@ def notifier_start(job_group, request_id, operator=None, metadata={}): except Notifier.DoesNotExist: pass job_group_notifier = JobGroupNotifier.objects.create( - job_group=job_group, request_id=request_id, notifier_type=notifier + job_group_id=job_group, request_id=request_id, notifier_type=notifier ) eh = event_handler(job_group_notifier.id) notifier_id = eh.start(request_id) diff --git a/runner/migrations/0058_pipeline_status.py b/runner/migrations/0058_pipeline_status.py new file mode 100644 index 000000000..8084f387f --- /dev/null +++ b/runner/migrations/0058_pipeline_status.py @@ -0,0 +1,21 @@ +# Generated by Django 2.2.28 on 2023-05-30 14:43 + +from django.db import migrations, models +import runner.models + + +class Migration(migrations.Migration): + + dependencies = [ + ("runner", "0057_auto_20230424_0743"), + ] + + operations = [ + migrations.AddField( + model_name="pipeline", + name="status", + field=models.IntegerField( + choices=[(0, "CWL"), (1, "NEXTFLOW")], db_index=True, default=runner.models.PipelineStatus(0) + ), + ), + ] diff --git a/runner/models.py b/runner/models.py index 6feb6470b..72de35b28 100644 --- a/runner/models.py +++ b/runner/models.py @@ -54,10 +54,18 @@ class PipelineName(models.Model): name = models.CharField(max_length=30, null=False, blank=False) +class PipelineStatus(IntEnum): + PREPARING = 0 + READY = 1 + + class Pipeline(BaseModel): pipeline_type = models.IntegerField( choices=[(pt.value, pt.name) for pt in ProtocolType], db_index=True, default=ProtocolType.CWL ) + status = models.IntegerField( + choices=[(pt.value, pt.name) for pt in ProtocolType], db_index=True, default=PipelineStatus.PREPARING + ) pipeline_name = models.ForeignKey(PipelineName, null=True, blank=True, on_delete=models.SET_NULL) name = models.CharField(max_length=100, editable=True) github = models.CharField(max_length=300, editable=True) diff --git a/runner/pipeline/pipeline_resolver.py b/runner/pipeline/pipeline_resolver.py index 7046b234d..0acf3df77 100644 --- a/runner/pipeline/pipeline_resolver.py +++ b/runner/pipeline/pipeline_resolver.py @@ -1,9 +1,13 @@ +import json import os import git import uuid import shutil import logging +from django.conf import settings +from file_system.repository import FileRepository from runner.cache.github_cache import GithubCache +from runner.run.processors.file_processor import FileProcessor class PipelineResolver(object): @@ -41,6 +45,30 @@ def _cleanup(self, location): else: shutil.rmtree(location) + def import_reference_files(self): + dir_name = self._dir_name() + pipeline_path = self._git_clone(dir_name) + absolute_path = os.path.join(pipeline_path, settings.APP_REFERENCE_FILES_PATH) + logging.info(f"Locating reference file in {absolute_path}") + if os.path.exists(absolute_path): + with open(absolute_path, "r") as f: + files = json.load(f) + for f in files: + if not FileRepository.filter( + path=f["location"], file_group=settings.REFERENCE_FILE_GROUP_ID + ).first(): + logging.info(f"Registering {f}") + FileProcessor.create_file_obj( + f["location"], + f["size"], + f["checksum"], + settings.REFERENCE_FILE_GROUP_ID, + ) + else: + logging.info(f"Pipeline doesn't have reference file in {absolute_path}") + logging.info(f"Cleanup pipeline directory {dir_name}") + self._cleanup(dir_name) + def load(self): pass diff --git a/runner/tasks.py b/runner/tasks.py index 938877e57..e8bd93594 100644 --- a/runner/tasks.py +++ b/runner/tasks.py @@ -9,8 +9,9 @@ from celery import shared_task from django.conf import settings from django.db.models import Count +from runner.pipeline.pipeline_resolver import PipelineResolver from runner.run.objects.run_object_factory import RunObjectFactory -from .models import Run, RunStatus, OperatorRun, TriggerAggregateConditionType, TriggerRunType, Pipeline +from .models import Run, RunStatus, OperatorRun, TriggerAggregateConditionType, TriggerRunType, Pipeline, PipelineStatus from notifier.events import ( RunFinishedEvent, OperatorRequestEvent, @@ -40,6 +41,8 @@ from study.objects import StudyObject from study.models import JobGroupWatcher, JobGroupWatcherConfig from django.http import HttpResponse +from django.dispatch import receiver +from django.db.models.signals import post_save logger = logging.getLogger(__name__) @@ -817,6 +820,25 @@ def add_pipeline_to_cache(github, version): GithubCache.add(github, version) +@shared_task +def register_reference_files(pipeline_id): + try: + pipeline = Pipeline.objects.get(id=pipeline_id) + except Pipeline.DoesNotExist: + logging.error(f"Pipeline with id:{pipeline_id} doesn't exist") + return + resolver = PipelineResolver(pipeline.github, pipeline.entrypoint, pipeline.version) + resolver.import_reference_files() + pipeline.status = PipelineStatus.READY + pipeline.save() + + +@receiver(post_save, sender=Pipeline) +def register_pipeline_reference_files(sender, instance, created, **kwargs): + if created: + register_reference_files.delay(str(instance.id)) + + class cmo_dmp_manifest: """ Description: diff --git a/runner/tests/operator/access/access_cnv/test_cnv_operator.py b/runner/tests/operator/access/access_cnv/test_cnv_operator.py index 6115433b2..5b2178479 100644 --- a/runner/tests/operator/access/access_cnv/test_cnv_operator.py +++ b/runner/tests/operator/access/access_cnv/test_cnv_operator.py @@ -1,11 +1,13 @@ import os from django.test import TestCase - +from django.db.models import signals from beagle.settings import ROOT_DIR from beagle_etl.models import Operator from runner.operator.operator_factory import OperatorFactory from runner.operator.access.v1_0_0.cnv import AccessLegacyCNVOperator +from runner.models import Pipeline +from runner.tasks import register_pipeline_reference_files FIXTURES = [ @@ -31,7 +33,7 @@ class TestAccessCNVOperator(TestCase): - + signals.post_save.disconnect(register_pipeline_reference_files, sender=Pipeline) fixtures = [os.path.join(ROOT_DIR, f) for f in FIXTURES + COMMON_FIXTURES] def test_access_legacy_cnv_operator(self): diff --git a/runner/tests/operator/access/access_msi/test_msi_operator.py b/runner/tests/operator/access/access_msi/test_msi_operator.py index 4da073a53..75999756b 100644 --- a/runner/tests/operator/access/access_msi/test_msi_operator.py +++ b/runner/tests/operator/access/access_msi/test_msi_operator.py @@ -1,11 +1,13 @@ import os from django.test import TestCase - +from django.db.models import signals from beagle.settings import ROOT_DIR from beagle_etl.models import Operator from runner.operator.operator_factory import OperatorFactory from runner.operator.access.v1_0_0.msi import AccessLegacyMSIOperator +from runner.models import Pipeline +from runner.tasks import register_pipeline_reference_files FIXTURES = [ @@ -31,7 +33,7 @@ class TestAccessMSIOperator(TestCase): - + signals.post_save.disconnect(register_pipeline_reference_files, sender=Pipeline) fixtures = [os.path.join(ROOT_DIR, f) for f in FIXTURES + COMMON_FIXTURES] def test_access_legacy_msi_operator(self): diff --git a/runner/tests/operator/access/access_snps_and_indels/test_snps_and_indels_operator.py b/runner/tests/operator/access/access_snps_and_indels/test_snps_and_indels_operator.py index 465c69076..eb6d3a625 100644 --- a/runner/tests/operator/access/access_snps_and_indels/test_snps_and_indels_operator.py +++ b/runner/tests/operator/access/access_snps_and_indels/test_snps_and_indels_operator.py @@ -3,10 +3,13 @@ from django.test import TestCase from file_system.models import File +from django.db.models import signals from beagle.settings import ROOT_DIR from beagle_etl.models import Operator from runner.operator.operator_factory import OperatorFactory from runner.operator.access.v1_0_0.snps_and_indels import AccessLegacySNVOperator +from runner.models import Pipeline +from runner.tasks import register_pipeline_reference_files REQUEST_ID = "access_legacy_test_request" TEST_RUN_ID = "bc23076e-f477-4578-943c-1fbf6f1fca44" @@ -33,6 +36,7 @@ class TestAccessSNVOperator(TestCase): + signals.post_save.disconnect(register_pipeline_reference_files, sender=Pipeline) fixtures = [os.path.join(ROOT_DIR, f) for f in FIXTURES + COMMON_FIXTURES] def test_access_legacy_snv_operator(self): diff --git a/runner/tests/operator/access/access_sv/test_access_sv_operator.py b/runner/tests/operator/access/access_sv/test_access_sv_operator.py index 80da0dea6..262c8442c 100644 --- a/runner/tests/operator/access/access_sv/test_access_sv_operator.py +++ b/runner/tests/operator/access/access_sv/test_access_sv_operator.py @@ -3,9 +3,12 @@ from django.test import TestCase from beagle.settings import ROOT_DIR +from django.db.models import signals from beagle_etl.models import Operator from runner.operator.operator_factory import OperatorFactory from runner.operator.access.v1_0_0.structural_variants import AccessLegacySVOperator +from runner.models import Pipeline +from runner.tasks import register_pipeline_reference_files FIXTURES = [ @@ -33,7 +36,7 @@ class TestAccessSVOperator(TestCase): - + signals.post_save.disconnect(register_pipeline_reference_files, sender=Pipeline) fixtures = [os.path.join(ROOT_DIR, f) for f in FIXTURES + COMMON_FIXTURES] def test_access_legacy_sv_operator(self): diff --git a/runner/tests/operator/access/legacy_fastq_to_bam/test_legacy_fastq_to_bam.py b/runner/tests/operator/access/legacy_fastq_to_bam/test_legacy_fastq_to_bam.py index 9b0602f96..66cf13a01 100644 --- a/runner/tests/operator/access/legacy_fastq_to_bam/test_legacy_fastq_to_bam.py +++ b/runner/tests/operator/access/legacy_fastq_to_bam/test_legacy_fastq_to_bam.py @@ -1,11 +1,13 @@ import os from django.test import TestCase - +from django.db.models import signals from beagle.settings import ROOT_DIR from beagle_etl.models import Operator from file_system.models import File, FileMetadata from runner.operator.operator_factory import OperatorFactory +from runner.models import Pipeline +from runner.tasks import register_pipeline_reference_files FIXTURES = [ @@ -28,7 +30,7 @@ class TestAccessLegacyOperator(TestCase): - + signals.post_save.disconnect(register_pipeline_reference_files, sender=Pipeline) fixtures = [os.path.join(ROOT_DIR, f) for f in FIXTURES + COMMON_FIXTURES] def test_access_legacy_operator(self): diff --git a/runner/tests/operator/access/merge_fastqs/test_fastq_merge_operator.py b/runner/tests/operator/access/merge_fastqs/test_fastq_merge_operator.py index 215f93084..0ea39b511 100644 --- a/runner/tests/operator/access/merge_fastqs/test_fastq_merge_operator.py +++ b/runner/tests/operator/access/merge_fastqs/test_fastq_merge_operator.py @@ -1,13 +1,13 @@ import os from django.test import TestCase - +from django.db.models import signals from beagle.settings import ROOT_DIR from beagle_etl.models import Operator from file_system.models import File, FileMetadata from runner.operator.operator_factory import OperatorFactory -from runner.operator.access.v1_0_0.merge_fastqs import AccessLegacyFastqMergeOperator, construct_inputs - +from runner.models import Pipeline +from runner.tasks import register_pipeline_reference_files FIXTURES = [ "fixtures/tests/merge_fastqs/10151_F_13.file.json", @@ -27,7 +27,7 @@ class TestAccessFastqMergeOperator(TestCase): - + signals.post_save.disconnect(register_pipeline_reference_files, sender=Pipeline) fixtures = [os.path.join(ROOT_DIR, f) for f in FIXTURES + COMMON_FIXTURES] def test_access_legacy_fastq_merge_operator(self): diff --git a/runner/tests/operator/access/v2_0_0/nucleo/test_nucleo_operator.py b/runner/tests/operator/access/v2_0_0/nucleo/test_nucleo_operator.py index ceeae08ae..5777af288 100644 --- a/runner/tests/operator/access/v2_0_0/nucleo/test_nucleo_operator.py +++ b/runner/tests/operator/access/v2_0_0/nucleo/test_nucleo_operator.py @@ -1,13 +1,13 @@ import os from django.test import TestCase - +from django.db.models import signals from beagle.settings import ROOT_DIR from beagle_etl.models import Operator from file_system.models import File, FileMetadata from runner.operator.operator_factory import OperatorFactory -from runner.operator.access.v1_0_0.merge_fastqs import AccessLegacyFastqMergeOperator, construct_inputs - +from runner.models import Pipeline +from runner.tasks import register_pipeline_reference_files FIXTURES = [ "fixtures/tests/merge_fastqs/10151_F_13.file.json", @@ -27,7 +27,7 @@ class TestAccessNucleoOperator(TestCase): - + signals.post_save.disconnect(register_pipeline_reference_files, sender=Pipeline) fixtures = [os.path.join(ROOT_DIR, f) for f in FIXTURES + COMMON_FIXTURES] def test_access_nucleo_operator(self): diff --git a/runner/tests/operator/access/v2_0_0/qc/test_qc_operator.py b/runner/tests/operator/access/v2_0_0/qc/test_qc_operator.py index 5319ae488..36b511497 100644 --- a/runner/tests/operator/access/v2_0_0/qc/test_qc_operator.py +++ b/runner/tests/operator/access/v2_0_0/qc/test_qc_operator.py @@ -3,10 +3,13 @@ from django.test import TestCase from beagle import settings +from django.db.models import signals from beagle.settings import ROOT_DIR from beagle_etl.models import Operator from file_system.models import File, FileMetadata from runner.operator.operator_factory import OperatorFactory +from runner.models import Pipeline +from runner.tasks import register_pipeline_reference_files FIXTURES = [ @@ -28,7 +31,7 @@ class TestAccessQCOperator(TestCase): - + signals.post_save.disconnect(register_pipeline_reference_files, sender=Pipeline) fixtures = [os.path.join(ROOT_DIR, f) for f in FIXTURES + COMMON_FIXTURES] def test_access_qc_operator(self): diff --git a/runner/tests/operator/access/v2_1_0/manifest/test_access_manifest_operator.py b/runner/tests/operator/access/v2_1_0/manifest/test_access_manifest_operator.py index 4f56a35e9..7ded152bd 100644 --- a/runner/tests/operator/access/v2_1_0/manifest/test_access_manifest_operator.py +++ b/runner/tests/operator/access/v2_1_0/manifest/test_access_manifest_operator.py @@ -1,15 +1,15 @@ import os - +import glob +import shutil from django.test import TestCase - from beagle import settings +from django.db.models import signals from beagle.settings import ROOT_DIR from beagle_etl.models import Operator -from file_system.models import File, FileMetadata +from file_system.models import File from runner.operator.operator_factory import OperatorFactory -import datetime -import glob -import shutil +from runner.models import Pipeline +from runner.tasks import register_pipeline_reference_files # general fixtures COMMON_FIXTURES = [ @@ -30,7 +30,7 @@ class TestAcessManifestOperator(TestCase): - # test db + signals.post_save.disconnect(register_pipeline_reference_files, sender=Pipeline) fixtures = [os.path.join(ROOT_DIR, f) for f in COMMON_FIXTURES] # variables to help check operator output expected_csv_content = 'igoRequestId,primaryId,cmoPatientId,dmpPatientId,dmpImpactSamples,dmpAccessSamples,baitSet,libraryVolume,investigatorSampleId,preservation,species,libraryConcentrationNgul,tissueLocation,sampleClass,sex,cfDNA2dBarcode,sampleOrigin,tubeId,tumorOrNormal,captureConcentrationNm,oncotreeCode,dnaInputNg,collectionYear,captureInputNg\n12345_A,12345_A_3,C-ALLANT,P-0000001,P-0000002-T01-IM6;P-0000001-T01-IM6,,MSK-ACCESS-v1_0-probesAllwFP,25.0,P-REDACT,EDTA-Streck,,102.5,,Blood,M,8042889270,Whole Blood,,Normal,9.756097561,,200.0,,1000.0000000025001\n12345_A,12345_A_1,C-ALLANT,P-0000001,P-0000002-T01-IM6;P-0000001-T01-IM6,,MSK-ACCESS-v1_0-probesAllwFP,25.0,P-REDACT,EDTA-Streck,,69.0,,Blood,M,8042889270,Whole Blood,,Normal,14.49275362,,200.0,,999.99999978\n12345_A,12345_A_2,C-ALLANT,P-0000001,P-0000002-T01-IM6;P-0000001-T01-IM6,,MSK-ACCESS-v1_0-probesAllwFP,25.0,P-REDACT,EDTA-Streck,,74.5,,Blood,M,8042889270,Whole Blood,,Normal,13.42281879,,200.0,,999.999999855\n""\n' diff --git a/runner/tests/operator/alignment_pair_operator/v1_0_0/test_alignment_pair_operator.py b/runner/tests/operator/alignment_pair_operator/v1_0_0/test_alignment_pair_operator.py index 5ae8a6165..4b3cd076e 100644 --- a/runner/tests/operator/alignment_pair_operator/v1_0_0/test_alignment_pair_operator.py +++ b/runner/tests/operator/alignment_pair_operator/v1_0_0/test_alignment_pair_operator.py @@ -3,15 +3,18 @@ """ import os from django.test import TestCase +from django.db.models import signals from runner.operator.operator_factory import OperatorFactory from beagle_etl.models import Operator from django.conf import settings from django.core.management import call_command -from file_system.models import File, FileMetadata, FileGroup, FileType -from pprint import pprint +from file_system.models import File, FileMetadata, FileGroup +from runner.models import Pipeline +from runner.tasks import register_pipeline_reference_files class TestAlignmentPairOperator(TestCase): + signals.post_save.disconnect(register_pipeline_reference_files, sender=Pipeline) fixtures = [ "file_system.filegroup.json", "file_system.filetype.json", diff --git a/runner/tests/operator/alignment_pair_operator/v1_0_0/test_construct_argos_pair.py b/runner/tests/operator/alignment_pair_operator/v1_0_0/test_construct_argos_pair.py index 0d4b63520..3af6d2d16 100644 --- a/runner/tests/operator/alignment_pair_operator/v1_0_0/test_construct_argos_pair.py +++ b/runner/tests/operator/alignment_pair_operator/v1_0_0/test_construct_argos_pair.py @@ -3,9 +3,8 @@ """ import os import json -from pprint import pprint -from uuid import UUID from django.test import TestCase +from django.db.models import signals from runner.operator.alignment_pair_operator.v1_0_0.construct_alignment_argos_inputs import ( construct_alignment_pair_jobs, ) @@ -13,10 +12,12 @@ from file_system.models import File from django.conf import settings from django.core.management import call_command +from runner.models import Pipeline +from runner.tasks import register_pipeline_reference_files class TestConstructPair(TestCase): - # load fixtures for the test case temp db + signals.post_save.disconnect(register_pipeline_reference_files, sender=Pipeline) fixtures = ["file_system.filegroup.json", "file_system.filetype.json", "file_system.storage.json"] def setUp(self): diff --git a/runner/tests/run/test_cwl_run.py b/runner/tests/run/test_cwl_run.py index 82c7eea1a..0f478ab2e 100644 --- a/runner/tests/run/test_cwl_run.py +++ b/runner/tests/run/test_cwl_run.py @@ -1,5 +1,6 @@ import json from mock import patch +from django.db.models import signals from rest_framework.test import APITestCase from runner.models import Port from runner.tasks import complete_job, fail_job @@ -8,6 +9,7 @@ from runner.run.objects.run_object_factory import RunObjectFactory from beagle_etl.models import JobGroup from file_system.models import Storage, StorageType, FileGroup, File, FileType, Sample, Request +from runner.tasks import register_pipeline_reference_files class CWLRunObjectTest(APITestCase): @@ -18,6 +20,7 @@ class CWLRunObjectTest(APITestCase): ] def setUp(self): + signals.post_save.disconnect(register_pipeline_reference_files, sender=Pipeline) self.storage = Storage(name="test", type=StorageType.LOCAL) self.storage.save() self.file_group = FileGroup(name="Test Files", storage=self.storage) diff --git a/runner/tests/run/test_nextflow_run.py b/runner/tests/run/test_nextflow_run.py index 03820a5c6..6388a631a 100644 --- a/runner/tests/run/test_nextflow_run.py +++ b/runner/tests/run/test_nextflow_run.py @@ -1,14 +1,17 @@ import json from mock import patch +from django.db.models import signals from rest_framework.test import APITestCase from runner.run.objects.nextflow.nextflow_run_object import NextflowRunObject from runner.run.objects.nextflow.nextflow_port_object import NextflowPortObject from runner.models import Run, ProtocolType, RunStatus, Pipeline, PortType, Port, Sample from file_system.models import FileGroup, File, FileType, FileExtension +from runner.tasks import register_pipeline_reference_files class NextflowRunObjectTest(APITestCase): def setUp(self): + signals.post_save.disconnect(register_pipeline_reference_files, sender=Pipeline) self.nxf_file_group = FileGroup.objects.create(name="Nextflow Output Group") self.pipeline = Pipeline.objects.create( name="nextflow_pipeline", diff --git a/runner/tests/views/test_run_api_view.py b/runner/tests/views/test_run_api_view.py index 3b3177212..223116853 100644 --- a/runner/tests/views/test_run_api_view.py +++ b/runner/tests/views/test_run_api_view.py @@ -4,6 +4,7 @@ import os from mock import patch, call from rest_framework import status +from django.db.models import signals from rest_framework.test import APITestCase from runner.views.run_api_view import OperatorViewSet from beagle_etl.models import JobGroup, JobGroupNotifier, Notifier @@ -14,6 +15,7 @@ from django.conf import settings from django.core.management import call_command from django.urls import reverse +from runner.tasks import register_pipeline_reference_files import beagle_etl.celery if not beagle_etl.celery.app.conf["task_always_eager"]: @@ -29,6 +31,7 @@ class MockRequest(object): class TestRunAPIList(APITestCase): + signals.post_save.disconnect(register_pipeline_reference_files, sender=Pipeline) fixtures = [ "file_system.filegroup.json", "file_system.filetype.json",