Training results: result_4.3.0-nobackground

.github/workflows/train.yml

@@ -134,7 +134,7 @@ jobs:
       - name: Download test dataset
         run: |
           uv run dvc get https://github.com/pyronear/pyro-dataset \
-            data/processed/yolo_test_corrected --rev v4.0.0-corrected \
+            data/processed/yolo_test --rev v4.3.0 \
             --out ./data/test/yolo_test
 
       - name: Run evaluation
@@ -150,7 +150,7 @@ jobs:
       - name: Download sequential test dataset
         run: |
           uv run dvc get https://github.com/pyronear/pyro-dataset \
-            data/processed/sequential_test --rev v4.0.0-corrected \
+            data/processed/sequential_test --rev v4.3.0 \
             --out ./data/test/sequential_test
 
       - name: Run sequential predictions on test
@@ -298,7 +298,7 @@ jobs:
           {opt_table}
 
           {baseline_note}
-          **Branch:** `{os.environ.get('RESULT_BRANCH', '')}` | **Test dataset:** pyronear/pyro-dataset @ v4.0.0-corrected""")
+          **Branch:** `{os.environ.get('RESULT_BRANCH', '')}` | **Test dataset:** pyronear/pyro-dataset @ v4.3.0""")
           PYEOF
           )
 

dvc.lock

@@ -53,21 +53,21 @@ stages:
     - uv run python ./scripts/data/model_input/build.py --input-dir 
       ./data/01_model_input/yolo_train_val --output-dir 
       ./data/01_model_input/yolo_train_val_small --sampling-ratio 1 
-      --random-seed 0 --loglevel info
+      --random-seed 0 --exclude-background --loglevel info
     - uv run python ./scripts/model/yolo/train.py --data 
       ./data/01_model_input/yolo_train_val_small/datasets/data.yaml --config 
       ./scripts/model/yolo/configs/best.yaml --output-dir ./data/02_models/yolo/
       --experiment-name best --loglevel info
     deps:
     - path: ./data/01_model_input/yolo_train_val
       hash: md5
-      md5: 409302377938ce2a82f5338f3145cbbc.dir
-      size: 1914167911
-      nfiles: 32201
+      md5: da8f531db6ac70886c2793c3a6f18f16.dir
+      size: 2321533653
+      nfiles: 37609
     - path: ./scripts/data/model_input/build.py
       hash: md5
-      md5: 7607af10f59267cf9859cb1d372aff26
-      size: 6432
+      md5: f745456171e8a8b24faca0e5f2bc0a22
+      size: 7800
     - path: ./scripts/model/yolo/configs/best.yaml
       hash: md5
       md5: 82ab9c26b11cfc5f72e1398bfceeb838
@@ -79,8 +79,8 @@ stages:
     outs:
     - path: ./data/02_models/yolo/best
       hash: md5
-      md5: 0e3617cd00b6a1985129110757bca1a0.dir
-      size: 44813862
+      md5: b492ecf31cc3972aac757aaa5a23da2b.dir
+      size: 45354436
       nfiles: 24
   build_manifest_yolo_best:
     cmd:
@@ -90,8 +90,8 @@ stages:
     deps:
     - path: ./data/02_models/yolo/best/
       hash: md5
-      md5: 0e3617cd00b6a1985129110757bca1a0.dir
-      size: 44813862
+      md5: b492ecf31cc3972aac757aaa5a23da2b.dir
+      size: 45354436
       nfiles: 24
     - path: ./scripts/model/yolo/build_manifest.py
       hash: md5
@@ -100,8 +100,8 @@ stages:
     outs:
     - path: ./data/03_reporting/yolo/best/
       hash: md5
-      md5: c01eb3ce79f23e0999d3220813c01881.dir
-      size: 13529
+      md5: d6c6cbd2e10730093dd22a6cfa5bde32.dir
+      size: 13530
       nfiles: 1
   export_yolo_best@onnx-cpu:
     cmd:
@@ -111,8 +111,8 @@ stages:
     deps:
     - path: ./data/02_models/yolo/best/
       hash: md5
-      md5: 0e3617cd00b6a1985129110757bca1a0.dir
-      size: 44813862
+      md5: b492ecf31cc3972aac757aaa5a23da2b.dir
+      size: 45354436
       nfiles: 24
     - path: ./scripts/model/yolo/export.py
       hash: md5
@@ -121,8 +121,8 @@ stages:
     outs:
     - path: ./data/02_models/yolo-export/best/onnx/cpu
       hash: md5
-      md5: 3e101e69ac72dc3109be7afd2b61aa4a.dir
-      size: 38532985
+      md5: cab4b58e34c96a125d3cb6643fb026d4.dir
+      size: 38199772
       nfiles: 1
   export_yolo_best@onnx-mps:
     cmd:
@@ -153,8 +153,8 @@ stages:
     deps:
     - path: ./data/02_models/yolo/best/
       hash: md5
-      md5: 0e3617cd00b6a1985129110757bca1a0.dir
-      size: 44813862
+      md5: b492ecf31cc3972aac757aaa5a23da2b.dir
+      size: 45354436
       nfiles: 24
     - path: ./scripts/model/yolo/export.py
       hash: md5
@@ -163,7 +163,7 @@ stages:
     outs:
     - path: ./data/02_models/yolo-export/best/ncnn/cpu
       hash: md5
-      md5: f6f1f2b316198f427a51b3b8335e7b7c.dir
+      md5: 6ea19bd53eba5519c77a1ceaf7c669c5.dir
       size: 38001906
       nfiles: 5
   export_yolo_best@ncnn-mps:
@@ -190,31 +190,30 @@ stages:
   fetch_model_input:
     cmd:
     - uv run dvc get https://github.com/pyronear/pyro-dataset 
-      data/processed/yolo_train_val_corrected --rev v4.0.0-corrected --out 
+      data/processed/yolo_train_val --rev v4.3.0 --out 
       ./data/01_model_input/yolo_train_val || uv run dvc get 
-      git@github.com:pyronear/pyro-dataset.git 
-      data/processed/yolo_train_val_corrected --rev v4.0.0-corrected --out 
-      ./data/01_model_input/yolo_train_val
+      git@github.com:pyronear/pyro-dataset.git data/processed/yolo_train_val 
+      --rev v4.3.0 --out ./data/01_model_input/yolo_train_val
     outs:
     - path: ./data/01_model_input/yolo_train_val
       hash: md5
-      md5: 409302377938ce2a82f5338f3145cbbc.dir
-      size: 1914167911
-      nfiles: 32201
+      md5: da8f531db6ac70886c2793c3a6f18f16.dir
+      size: 2321533653
+      nfiles: 37609
   fetch_sequential_val:
     cmd:
     - uv run dvc get https://github.com/pyronear/pyro-dataset 
-      data/processed/sequential_train_val/val --rev v4.0.0-corrected --out 
+      data/processed/sequential_train_val/val --rev v4.3.0 --out 
       ./data/01_model_input/sequential_train_val/val || uv run dvc get 
       git@github.com:pyronear/pyro-dataset.git 
-      data/processed/sequential_train_val/val --rev v4.0.0-corrected --out 
+      data/processed/sequential_train_val/val --rev v4.3.0 --out 
       ./data/01_model_input/sequential_train_val/val
     outs:
     - path: ./data/01_model_input/sequential_train_val/val
       hash: md5
-      md5: 652f9f7e73f297b782ed68e1b4844cce.dir
-      size: 671953151
-      nfiles: 11478
+      md5: df9cc34a9f322e39f85bda0d8e183463.dir
+      size: 855129379
+      nfiles: 14054
   evaluate_sequential_yolo_best:
     cmd:
     - uv run python ./scripts/model/yolo/evaluate_sequential.py --model-path 
@@ -252,12 +251,12 @@ stages:
     deps:
     - path: ./data/01_model_input/sequential_train_val/val
       hash: md5
-      md5: 652f9f7e73f297b782ed68e1b4844cce.dir
-      size: 671953151
-      nfiles: 11478
+      md5: df9cc34a9f322e39f85bda0d8e183463.dir
+      size: 855129379
+      nfiles: 14054
     - path: ./data/02_models/yolo/best/weights/best.pt
       hash: md5
-      md5: 550d1b91fb93e52cf233166b03405a10
+      md5: f9740882540ebdc77d788184c0b9c873
       size: 19225626
     - path: predict_sequential.py
       hash: md5
@@ -266,9 +265,9 @@ stages:
     outs:
     - path: ./data/03_reporting/sequential/predictions_labels_val
       hash: md5
-      md5: 796a49a2802fa773158ca168a680f12c.dir
-      size: 109228
-      nfiles: 3340
+      md5: e5d3d7e49f5a964df5521a6acedab22c.dir
+      size: 171362
+      nfiles: 4106
   optimize_sequential_val:
     cmd:
     - uv run python optimize_sequential.py --labels-dir 
@@ -278,19 +277,19 @@ stages:
     deps:
     - path: ./data/03_reporting/sequential/predictions_labels_val
       hash: md5
-      md5: 796a49a2802fa773158ca168a680f12c.dir
-      size: 109228
-      nfiles: 3340
+      md5: e5d3d7e49f5a964df5521a6acedab22c.dir
+      size: 171362
+      nfiles: 4106
     - path: optimize_sequential.py
       hash: md5
-      md5: 115eb45048da93018f7ea86b5f954270
-      size: 8645
+      md5: a032d3a9877382e4ba2d74e5c529aaab
+      size: 9087
     outs:
     - path: ./data/03_reporting/sequential/grid_search_val.tsv
       hash: md5
-      md5: aa6f9da227c79d6ffa2903e7b9a3d18b
-      size: 1873
+      md5: 26192cbae025683faf060aa7c343d1ee
+      size: 1923
     - path: ./data/03_reporting/sequential/grid_search_val_top20.tsv
       hash: md5
-      md5: dd8e0aa4c8bc678deb27acbdd911c065
-      size: 973
+      md5: ab83d953fc37020b24e0e65aeb8f9a55
+      size: 998

dvc.yaml

@@ -3,11 +3,11 @@ stages:
     cmd:
     - >-
       uv run dvc get https://github.com/pyronear/pyro-dataset
-      data/processed/yolo_train_val_corrected --rev v4.0.0-corrected
+      data/processed/yolo_train_val --rev v4.3.0
       --out ./data/01_model_input/yolo_train_val
       ||
       uv run dvc get git@github.com:pyronear/pyro-dataset.git
-      data/processed/yolo_train_val_corrected --rev v4.0.0-corrected
+      data/processed/yolo_train_val --rev v4.3.0
       --out ./data/01_model_input/yolo_train_val
     outs:
     - ./data/01_model_input/yolo_train_val
@@ -20,6 +20,7 @@ stages:
       --output-dir ./data/01_model_input/yolo_train_val_small
       --sampling-ratio 1
       --random-seed 0
+      --exclude-background
       --loglevel info
     - >-
       uv run python ./scripts/model/yolo/train.py
@@ -53,11 +54,11 @@ stages:
     cmd:
     - >-
       uv run dvc get https://github.com/pyronear/pyro-dataset
-      data/processed/sequential_train_val/val --rev v4.0.0-corrected
+      data/processed/sequential_train_val/val --rev v4.3.0
       --out ./data/01_model_input/sequential_train_val/val
       ||
       uv run dvc get git@github.com:pyronear/pyro-dataset.git
-      data/processed/sequential_train_val/val --rev v4.0.0-corrected
+      data/processed/sequential_train_val/val --rev v4.3.0
       --out ./data/01_model_input/sequential_train_val/val
     outs:
     - ./data/01_model_input/sequential_train_val/val

results/eval_results.json

@@ -1,8 +1,8 @@
 {
-  "precision": 0.7801831566503715,
-  "recall": 0.7129963898916968,
-  "map50": 0.7478004027603852,
-  "map50_95": 0.45667696600893326,
+  "precision": 0.6587947887747833,
+  "recall": 0.6950442255173176,
+  "map50": 0.6398575212990499,
+  "map50_95": 0.4020609511911931,
   "model_dir": "data/02_models/yolo/best",
   "data": "data/test/yolo_test/data.yaml",
   "split": "test"

results/eval_sequential_results.json

@@ -1,12 +1,12 @@
 {
-  "nb_consecutive_frames": 4,
-  "conf_thresh": 0.25,
-  "tp": 115,
-  "fn": 4,
-  "fp": 13,
-  "tn": 97,
-  "recall": 0.9664,
-  "fpr": 0.1182,
-  "precision": 0.8984,
-  "f1": 0.9312
+  "nb_consecutive_frames": 6,
+  "conf_thresh": 0.35,
+  "tp": 148,
+  "fn": 7,
+  "fp": 33,
+  "tn": 109,
+  "recall": 0.9548,
+  "fpr": 0.2324,
+  "precision": 0.8177,
+  "f1": 0.881
 }

scripts/data/model_input/build.py

@@ -41,6 +41,11 @@ def make_cli_parser() -> argparse.ArgumentParser:
         default=0,
         type=int,
     )
+    parser.add_argument(
+        "--exclude-background",
+        help="Exclude background images (empty label files) from the train split to train an ultra-sensitive model",
+        action="store_true",
+    )
     parser.add_argument(
         "-log",
         "--loglevel",
@@ -111,11 +116,20 @@ def copy_data(input_dir: Path, output_dir: Path) -> None:
         )
 
 
+def is_background(label_filepath: Path) -> bool:
+    """
+    Return whether the given label file corresponds to a background image,
+    ie. a missing or empty label file (no smoke annotation).
+    """
+    return not label_filepath.exists() or label_filepath.stat().st_size == 0
+
+
 def sample_dataset(
     input_dir: Path,
     output_dir: Path,
     sampling_ratio: float = 0.1,
     random_seed: int = 0,
+    exclude_background: bool = False,
 ) -> list[dict]:
     """
     Return a downsampled list of images and labels for the given
@@ -124,6 +138,9 @@ def sample_dataset(
     Each element in the returned list has the following keys:
     - to: Path - where the image/label comes from
     - from: Path - where the image/label should be copied over - using the provided `output_dir`
+
+    When `exclude_background` is set, background images (empty label files)
+    are dropped from the train split to train an ultra-sensitive model.
     """
     assert 0 <= sampling_ratio <= 1.0, "sampling ratio should be between 0 and 1"
 
@@ -132,6 +149,19 @@ def sample_dataset(
         labels_split_dir = input_dir / "labels" / split
         images_split_dir = input_dir / "images" / split
         images_filepaths = list(images_split_dir.glob("*.jpg"))
+        # Drop background images (empty labels) from the train split only, so
+        # that the val split stays intact for a comparable evaluation.
+        if exclude_background and split == "train":
+            n_before = len(images_filepaths)
+            images_filepaths = [
+                fp
+                for fp in images_filepaths
+                if not is_background(labels_split_dir / f"{fp.stem}.txt")
+            ]
+            logging.info(
+                f"Excluded {n_before - len(images_filepaths)} background images "
+                f"from the {split} split ({len(images_filepaths)} remaining)"
+            )
         n_images = len(images_filepaths)
         k = int(n_images * sampling_ratio)
         # For the val split we do not subsample as we want to eval on the same data as in the full
@@ -202,6 +232,7 @@ def run_file_copy(copy_data: list[dict]) -> None:
                 output_dir=output_dir / "datasets",
                 sampling_ratio=sampling_ratio,
                 random_seed=random_seed,
+                exclude_background=args["exclude_background"],
             )
         )
         write_data_yaml(output_dir / "datasets" / "data.yaml")