From 1da827dc68bda3baf674e5af89f6addab6eca499 Mon Sep 17 00:00:00 2001 From: Sinisa Ivkovic Date: Mon, 24 Aug 2026 15:57:43 -0400 Subject: [PATCH 1/2] Implement default mapping for file_groups different then IGO --- beagle/settings.py | 1 + file_manager/copy_service/copy_service.py | 17 ++++++++++------- file_manager/file_manager/file_manager.py | 4 ++-- .../tests/copy_service/test_copy_service.py | 18 +++++++++--------- runner/tasks.py | 6 +++--- 5 files changed, 25 insertions(+), 21 deletions(-) diff --git a/beagle/settings.py b/beagle/settings.py index 61a3be5f..ed186ea6 100644 --- a/beagle/settings.py +++ b/beagle/settings.py @@ -488,6 +488,7 @@ FASTQ_DEFAULT_LOCATION_PREFIX = os.environ.get("BEAGLE_FASTQ_DEFAULT_LOCATION_PREFIX") FASTQ_IRIS_LOCATION_PREFIX = os.environ.get("BEAGLE_FASTQ_IRIS_LOCATION_PREFIX") +FASTQ_DEFAULT_STAGING_PATH = os.environ.get("BEAGLE_FASTQ_DEFAULT_STAGING_PATH", "") DEFAULT_LOG_PREFIX = os.environ.get("BEAGLE_DEFAULT_LOG_PREFIX", "") DEFAULT_LOG_PATH = os.environ.get("BEAGLE_DEFAULT_LOG_PATH", "/tmp") diff --git a/file_manager/copy_service/copy_service.py b/file_manager/copy_service/copy_service.py index ef28fe32..7a2be432 100644 --- a/file_manager/copy_service/copy_service.py +++ b/file_manager/copy_service/copy_service.py @@ -25,19 +25,22 @@ def copy(path_from, path_to): os.chmod(path_to, settings.COPY_FILE_PERMISSION) @staticmethod - def remap(gene_panel, path, mapping=settings.DEFAULT_MAPPING): - prefix, dst = CopyService._get_mapping(gene_panel, path, mapping) + def remap(gene_panel, path, file_group=settings.IMPORT_FILE_GROUP, mapping=settings.DEFAULT_MAPPING): + prefix, dst = CopyService._get_mapping(gene_panel, path, file_group, mapping) if prefix and dst: path = path.replace(prefix, dst) logger.info("New path {path}".format(path=path)) return path @staticmethod - def _get_mapping(gene_panel, path, mapping=settings.DEFAULT_MAPPING): - recipe_mapping = mapping.get(gene_panel, {}) - for prefix, dst in recipe_mapping.items(): - if path.startswith(prefix): - return prefix, dst + def _get_mapping(gene_panel, path, file_group=settings.IMPORT_FILE_GROUP, mapping=settings.DEFAULT_MAPPING): + if file_group == settings.IMPORT_FILE_GROUP: + recipe_mapping = mapping.get(gene_panel, {}) + for prefix, dst in recipe_mapping.items(): + if path.startswith(prefix): + return prefix, dst + else: + return settings.FASTQ_IRIS_LOCATION_PREFIX, os.path.join(settings.FASTQ_DEFAULT_STAGING_PATH, file_group) return None, None @staticmethod diff --git a/file_manager/file_manager/file_manager.py b/file_manager/file_manager/file_manager.py index a9a15abf..3020b544 100644 --- a/file_manager/file_manager/file_manager.py +++ b/file_manager/file_manager/file_manager.py @@ -29,7 +29,7 @@ def stage_sample(self, sample_id): files_to_stage = 0 for f in files: if not f.file.is_available: - new_path = CopyService.remap(gene_panel, f.file.path) + new_path = CopyService.remap(gene_panel, f.file.path, str(f.file.file_group.id)) if new_path != f.file.path: files_to_stage += 1 @@ -55,7 +55,7 @@ def stage_file(self, file_obj, gene_panel, sample_job=None): Returns: Task signature or None """ if not file_obj.is_available: - new_path = CopyService.remap(gene_panel, file_obj.path) + new_path = CopyService.remap(gene_panel, file_obj.path, str(file_obj.file_group.id)) if new_path != file_obj.path: fp_job, created = FileProviderJob.objects.provide_file( file_obj, file_obj.path, new_path, sample_job=sample_job diff --git a/file_manager/tests/copy_service/test_copy_service.py b/file_manager/tests/copy_service/test_copy_service.py index 2318f946..6bc12ad0 100644 --- a/file_manager/tests/copy_service/test_copy_service.py +++ b/file_manager/tests/copy_service/test_copy_service.py @@ -11,12 +11,12 @@ def setUp(self): def test_remap(self): old_path = "/path/to/file/file1.fastq" - new_path = CopyService.remap(self.recipe, old_path, self.mapping) + new_path = CopyService.remap(self.recipe, old_path, mapping=self.mapping) self.assertEqual(new_path, "/new/path/to/file/file1.fastq") def test_remap_no_mapping(self): old_path = "/some/other/path/to/file/file1.fastq" - new_path = CopyService.remap(self.recipe, old_path, self.mapping) + new_path = CopyService.remap(self.recipe, old_path, mapping=self.mapping) self.assertEqual(new_path, old_path) def test_remap_multiple_prefixes(self): @@ -29,11 +29,11 @@ def test_remap_multiple_prefixes(self): } path1 = "/path/to/source1/file1.fastq" - new_path1 = CopyService.remap(self.recipe, path1, mapping) + new_path1 = CopyService.remap(self.recipe, path1, mapping=mapping) self.assertEqual(new_path1, "/staging/dest1/file1.fastq") path2 = "/path/to/source2/file2.fastq" - new_path2 = CopyService.remap(self.recipe, path2, mapping) + new_path2 = CopyService.remap(self.recipe, path2, mapping=mapping) self.assertEqual(new_path2, "/staging/dest2/file2.fastq") def test_remap_different_recipe(self): @@ -44,21 +44,21 @@ def test_remap_different_recipe(self): } path = "/path/to/file.fastq" - new_path_impact = CopyService.remap("IMPACT468", path, mapping) + new_path_impact = CopyService.remap("IMPACT468", path, mapping=mapping) self.assertEqual(new_path_impact, "/staging/impact/file.fastq") - new_path_heme = CopyService.remap("HEMEPACT", path, mapping) + new_path_heme = CopyService.remap("HEMEPACT", path, mapping=mapping) self.assertEqual(new_path_heme, "/staging/heme/file.fastq") def test_get_mapping(self): """Test internal _get_mapping method""" - prefix, dst = CopyService._get_mapping(self.recipe, "/path/to/file.fastq", self.mapping) + prefix, dst = CopyService._get_mapping(self.recipe, "/path/to/file.fastq", mapping=self.mapping) self.assertEqual(prefix, "/path/to") self.assertEqual(dst, "/new/path/to") def test_get_mapping_no_match(self): """Test _get_mapping when no prefix matches""" - prefix, dst = CopyService._get_mapping(self.recipe, "/other/path/file.fastq", self.mapping) + prefix, dst = CopyService._get_mapping(self.recipe, "/other/path/file.fastq", mapping=self.mapping) self.assertIsNone(prefix) self.assertIsNone(dst) @@ -81,7 +81,7 @@ def test_reverse_mapping_symmetry(self): original_path = "/path/to/subdir/file.fastq" # Forward mapping - staged_path = CopyService.remap(self.recipe, original_path, self.mapping) + staged_path = CopyService.remap(self.recipe, original_path, mapping=self.mapping) self.assertEqual(staged_path, "/new/path/to/subdir/file.fastq") # Reverse mapping should give us back the components diff --git a/runner/tasks.py b/runner/tasks.py index 207ddaeb..ebdf1ac2 100644 --- a/runner/tasks.py +++ b/runner/tasks.py @@ -56,7 +56,7 @@ def stage_files_for_operator( ): staging_tasks = [] try: - staging_tasks, sample_jobs = stage_files(request_id, pairing, job_group_id) + staging_tasks, sample_jobs = stage_files(request_id, pairing, job_group_id, file_group_id) except Exception as e: logger.warning(format_log(f"Failed to stage files: {str(e)}", job_group_id=job_group_id)) @@ -251,7 +251,7 @@ def create_operator_run_from_jobs( operator_run.save() -def stage_files(request_id=None, pairing=None, job_group_id=None): +def stage_files(request_id=None, pairing=None, job_group_id=None, file_group_id=None): """ Stage files and return list of staging task signatures. Returns (staging_tasks, sample_jobs) where: @@ -278,7 +278,7 @@ def stage_files(request_id=None, pairing=None, job_group_id=None): logger.info(format_log("No samples to stage", job_group_id=job_group_id)) return staging_tasks, sample_jobs - file_manager = FileManager() + file_manager = FileManager(file_group_id) if file_group_id else FileManager() for sample in samples: logger.info(format_log(f"Staging files for sample {sample}", request_id=request_id, job_group_id=job_group_id)) sample_job, task_sigs = file_manager.stage_sample(sample) From 686d573d1d78ac863d10cf29414dd43c252f041c Mon Sep 17 00:00:00 2001 From: Sinisa Ivkovic Date: Wed, 26 Aug 2026 13:14:41 -0400 Subject: [PATCH 2/2] Fix directory path --- file_manager/copy_service/copy_service.py | 5 +++- .../tests/copy_service/test_copy_service.py | 26 ++++++++++++++++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/file_manager/copy_service/copy_service.py b/file_manager/copy_service/copy_service.py index 7a2be432..204c7f12 100644 --- a/file_manager/copy_service/copy_service.py +++ b/file_manager/copy_service/copy_service.py @@ -40,7 +40,10 @@ def _get_mapping(gene_panel, path, file_group=settings.IMPORT_FILE_GROUP, mappin if path.startswith(prefix): return prefix, dst else: - return settings.FASTQ_IRIS_LOCATION_PREFIX, os.path.join(settings.FASTQ_DEFAULT_STAGING_PATH, file_group) + return ( + settings.FASTQ_IRIS_LOCATION_PREFIX, + os.path.join(settings.FASTQ_DEFAULT_STAGING_PATH, file_group) + "/", + ) return None, None @staticmethod diff --git a/file_manager/tests/copy_service/test_copy_service.py b/file_manager/tests/copy_service/test_copy_service.py index 6bc12ad0..14ec08cd 100644 --- a/file_manager/tests/copy_service/test_copy_service.py +++ b/file_manager/tests/copy_service/test_copy_service.py @@ -1,4 +1,7 @@ -from django.test import TestCase +import os + +from django.test import TestCase, override_settings + from file_manager.copy_service.copy_service import CopyService @@ -62,6 +65,27 @@ def test_get_mapping_no_match(self): self.assertIsNone(prefix) self.assertIsNone(dst) + @override_settings(FASTQ_IRIS_LOCATION_PREFIX="/igo/delivery", FASTQ_DEFAULT_STAGING_PATH="/staging") + def test_get_mapping_other_file_group(self): + """Test _get_mapping when file_group is not IMPORT_FILE_GROUP: it should stage under + FASTQ_DEFAULT_STAGING_PATH/, regardless of the recipe mapping.""" + other_file_group = "some-other-file-group-id" + prefix, dst = CopyService._get_mapping( + self.recipe, "/test/delivery/file.fastq", file_group=other_file_group, mapping=self.mapping + ) + self.assertEqual(prefix, "/igo/delivery") + self.assertEqual(dst, os.path.join("/staging", other_file_group) + "/") + + @override_settings(FASTQ_IRIS_LOCATION_PREFIX="/igo/delivery", FASTQ_DEFAULT_STAGING_PATH="/staging") + def test_remap_other_file_group(self): + """Test that remap stages a file under FASTQ_DEFAULT_STAGING_PATH/ + when file_group is different from IMPORT_FILE_GROUP.""" + other_file_group = "some-other-file-group-id" + old_path = "/test/delivery/file/file1.fastq" + new_path = CopyService.remap(self.recipe, old_path, file_group=other_file_group, mapping=self.mapping) + expected_dst = os.path.join("/staging", other_file_group + "/") + self.assertEqual(new_path, old_path.replace("/igo/delivery", expected_dst)) + def test_get_reverse_mapping(self): """Test reverse mapping to convert staged path back to original""" staged_path = "/new/path/to/file/file1.fastq"