Stage file from non Default file group

beagle/settings.py

@@ -488,6 +488,7 @@
 
 FASTQ_DEFAULT_LOCATION_PREFIX = os.environ.get("BEAGLE_FASTQ_DEFAULT_LOCATION_PREFIX")
 FASTQ_IRIS_LOCATION_PREFIX = os.environ.get("BEAGLE_FASTQ_IRIS_LOCATION_PREFIX")
+FASTQ_DEFAULT_STAGING_PATH = os.environ.get("BEAGLE_FASTQ_DEFAULT_STAGING_PATH", "")
 
 DEFAULT_LOG_PREFIX = os.environ.get("BEAGLE_DEFAULT_LOG_PREFIX", "")
 DEFAULT_LOG_PATH = os.environ.get("BEAGLE_DEFAULT_LOG_PATH", "/tmp")

file_manager/copy_service/copy_service.py

@@ -25,19 +25,25 @@ def copy(path_from, path_to):
         os.chmod(path_to, settings.COPY_FILE_PERMISSION)
 
     @staticmethod
-    def remap(gene_panel, path, mapping=settings.DEFAULT_MAPPING):
-        prefix, dst = CopyService._get_mapping(gene_panel, path, mapping)
+    def remap(gene_panel, path, file_group=settings.IMPORT_FILE_GROUP, mapping=settings.DEFAULT_MAPPING):
+        prefix, dst = CopyService._get_mapping(gene_panel, path, file_group, mapping)
         if prefix and dst:
             path = path.replace(prefix, dst)
         logger.info("New path {path}".format(path=path))
         return path
 
     @staticmethod
-    def _get_mapping(gene_panel, path, mapping=settings.DEFAULT_MAPPING):
-        recipe_mapping = mapping.get(gene_panel, {})
-        for prefix, dst in recipe_mapping.items():
-            if path.startswith(prefix):
-                return prefix, dst
+    def _get_mapping(gene_panel, path, file_group=settings.IMPORT_FILE_GROUP, mapping=settings.DEFAULT_MAPPING):
+        if file_group == settings.IMPORT_FILE_GROUP:
+            recipe_mapping = mapping.get(gene_panel, {})
+            for prefix, dst in recipe_mapping.items():
+                if path.startswith(prefix):
+                    return prefix, dst
+        else:
+            return (
+                settings.FASTQ_IRIS_LOCATION_PREFIX,
+                os.path.join(settings.FASTQ_DEFAULT_STAGING_PATH, file_group) + "/",
+            )
         return None, None
 
     @staticmethod

file_manager/file_manager/file_manager.py

@@ -29,7 +29,7 @@ def stage_sample(self, sample_id):
         files_to_stage = 0
         for f in files:
             if not f.file.is_available:
-                new_path = CopyService.remap(gene_panel, f.file.path)
+                new_path = CopyService.remap(gene_panel, f.file.path, str(f.file.file_group.id))
                 if new_path != f.file.path:
                     files_to_stage += 1
 
@@ -55,7 +55,7 @@ def stage_file(self, file_obj, gene_panel, sample_job=None):
         Returns: Task signature or None
         """
         if not file_obj.is_available:
-            new_path = CopyService.remap(gene_panel, file_obj.path)
+            new_path = CopyService.remap(gene_panel, file_obj.path, str(file_obj.file_group.id))
             if new_path != file_obj.path:
                 fp_job, created = FileProviderJob.objects.provide_file(
                     file_obj, file_obj.path, new_path, sample_job=sample_job

file_manager/tests/copy_service/test_copy_service.py

@@ -1,4 +1,7 @@
-from django.test import TestCase
+import os
+
+from django.test import TestCase, override_settings
+
 from file_manager.copy_service.copy_service import CopyService
 
 
@@ -11,12 +14,12 @@ def setUp(self):
 
     def test_remap(self):
         old_path = "/path/to/file/file1.fastq"
-        new_path = CopyService.remap(self.recipe, old_path, self.mapping)
+        new_path = CopyService.remap(self.recipe, old_path, mapping=self.mapping)
         self.assertEqual(new_path, "/new/path/to/file/file1.fastq")
 
     def test_remap_no_mapping(self):
         old_path = "/some/other/path/to/file/file1.fastq"
-        new_path = CopyService.remap(self.recipe, old_path, self.mapping)
+        new_path = CopyService.remap(self.recipe, old_path, mapping=self.mapping)
         self.assertEqual(new_path, old_path)
 
     def test_remap_multiple_prefixes(self):
@@ -29,11 +32,11 @@ def test_remap_multiple_prefixes(self):
         }
 
         path1 = "/path/to/source1/file1.fastq"
-        new_path1 = CopyService.remap(self.recipe, path1, mapping)
+        new_path1 = CopyService.remap(self.recipe, path1, mapping=mapping)
         self.assertEqual(new_path1, "/staging/dest1/file1.fastq")
 
         path2 = "/path/to/source2/file2.fastq"
-        new_path2 = CopyService.remap(self.recipe, path2, mapping)
+        new_path2 = CopyService.remap(self.recipe, path2, mapping=mapping)
         self.assertEqual(new_path2, "/staging/dest2/file2.fastq")
 
     def test_remap_different_recipe(self):
@@ -44,24 +47,45 @@ def test_remap_different_recipe(self):
         }
 
         path = "/path/to/file.fastq"
-        new_path_impact = CopyService.remap("IMPACT468", path, mapping)
+        new_path_impact = CopyService.remap("IMPACT468", path, mapping=mapping)
         self.assertEqual(new_path_impact, "/staging/impact/file.fastq")
 
-        new_path_heme = CopyService.remap("HEMEPACT", path, mapping)
+        new_path_heme = CopyService.remap("HEMEPACT", path, mapping=mapping)
         self.assertEqual(new_path_heme, "/staging/heme/file.fastq")
 
     def test_get_mapping(self):
         """Test internal _get_mapping method"""
-        prefix, dst = CopyService._get_mapping(self.recipe, "/path/to/file.fastq", self.mapping)
+        prefix, dst = CopyService._get_mapping(self.recipe, "/path/to/file.fastq", mapping=self.mapping)
         self.assertEqual(prefix, "/path/to")
         self.assertEqual(dst, "/new/path/to")
 
     def test_get_mapping_no_match(self):
         """Test _get_mapping when no prefix matches"""
-        prefix, dst = CopyService._get_mapping(self.recipe, "/other/path/file.fastq", self.mapping)
+        prefix, dst = CopyService._get_mapping(self.recipe, "/other/path/file.fastq", mapping=self.mapping)
         self.assertIsNone(prefix)
         self.assertIsNone(dst)
 
+    @override_settings(FASTQ_IRIS_LOCATION_PREFIX="/igo/delivery", FASTQ_DEFAULT_STAGING_PATH="/staging")
+    def test_get_mapping_other_file_group(self):
+        """Test _get_mapping when file_group is not IMPORT_FILE_GROUP: it should stage under
+        FASTQ_DEFAULT_STAGING_PATH/<file_group>, regardless of the recipe mapping."""
+        other_file_group = "some-other-file-group-id"
+        prefix, dst = CopyService._get_mapping(
+            self.recipe, "/test/delivery/file.fastq", file_group=other_file_group, mapping=self.mapping
+        )
+        self.assertEqual(prefix, "/igo/delivery")
+        self.assertEqual(dst, os.path.join("/staging", other_file_group) + "/")
+
+    @override_settings(FASTQ_IRIS_LOCATION_PREFIX="/igo/delivery", FASTQ_DEFAULT_STAGING_PATH="/staging")
+    def test_remap_other_file_group(self):
+        """Test that remap stages a file under FASTQ_DEFAULT_STAGING_PATH/<file_group>
+        when file_group is different from IMPORT_FILE_GROUP."""
+        other_file_group = "some-other-file-group-id"
+        old_path = "/test/delivery/file/file1.fastq"
+        new_path = CopyService.remap(self.recipe, old_path, file_group=other_file_group, mapping=self.mapping)
+        expected_dst = os.path.join("/staging", other_file_group + "/")
+        self.assertEqual(new_path, old_path.replace("/igo/delivery", expected_dst))
+
     def test_get_reverse_mapping(self):
         """Test reverse mapping to convert staged path back to original"""
         staged_path = "/new/path/to/file/file1.fastq"
@@ -81,7 +105,7 @@ def test_reverse_mapping_symmetry(self):
         original_path = "/path/to/subdir/file.fastq"
 
         # Forward mapping
-        staged_path = CopyService.remap(self.recipe, original_path, self.mapping)
+        staged_path = CopyService.remap(self.recipe, original_path, mapping=self.mapping)
         self.assertEqual(staged_path, "/new/path/to/subdir/file.fastq")
 
         # Reverse mapping should give us back the components

runner/tasks.py

@@ -56,7 +56,7 @@ def stage_files_for_operator(
 ):
     staging_tasks = []
     try:
-        staging_tasks, sample_jobs = stage_files(request_id, pairing, job_group_id)
+        staging_tasks, sample_jobs = stage_files(request_id, pairing, job_group_id, file_group_id)
     except Exception as e:
         logger.warning(format_log(f"Failed to stage files: {str(e)}", job_group_id=job_group_id))
 
@@ -251,7 +251,7 @@ def create_operator_run_from_jobs(
     operator_run.save()
 
 
-def stage_files(request_id=None, pairing=None, job_group_id=None):
+def stage_files(request_id=None, pairing=None, job_group_id=None, file_group_id=None):
     """
     Stage files and return list of staging task signatures.
     Returns (staging_tasks, sample_jobs) where:
@@ -278,7 +278,7 @@ def stage_files(request_id=None, pairing=None, job_group_id=None):
         logger.info(format_log("No samples to stage", job_group_id=job_group_id))
         return staging_tasks, sample_jobs
 
-    file_manager = FileManager()
+    file_manager = FileManager(file_group_id) if file_group_id else FileManager()
     for sample in samples:
         logger.info(format_log(f"Staging files for sample {sample}", request_id=request_id, job_group_id=job_group_id))
         sample_job, task_sigs = file_manager.stage_sample(sample)