diff --git a/.github/workflows/train.yml b/.github/workflows/train.yml index f70cbf7..6521265 100644 --- a/.github/workflows/train.yml +++ b/.github/workflows/train.yml @@ -134,7 +134,7 @@ jobs: - name: Download test dataset run: | uv run dvc get https://github.com/pyronear/pyro-dataset \ - data/processed/yolo_test_corrected --rev v4.0.0-corrected \ + data/processed/yolo_test --rev v4.3.0 \ --out ./data/test/yolo_test - name: Run evaluation @@ -150,7 +150,7 @@ jobs: - name: Download sequential test dataset run: | uv run dvc get https://github.com/pyronear/pyro-dataset \ - data/processed/sequential_test --rev v4.0.0-corrected \ + data/processed/sequential_test --rev v4.3.0 \ --out ./data/test/sequential_test - name: Run sequential predictions on test @@ -298,7 +298,7 @@ jobs: {opt_table} {baseline_note} - **Branch:** `{os.environ.get('RESULT_BRANCH', '')}` | **Test dataset:** pyronear/pyro-dataset @ v4.0.0-corrected""") + **Branch:** `{os.environ.get('RESULT_BRANCH', '')}` | **Test dataset:** pyronear/pyro-dataset @ v4.3.0""") PYEOF ) diff --git a/dvc.lock b/dvc.lock index 12064b0..61b1374 100644 --- a/dvc.lock +++ b/dvc.lock @@ -53,7 +53,7 @@ stages: - uv run python ./scripts/data/model_input/build.py --input-dir ./data/01_model_input/yolo_train_val --output-dir ./data/01_model_input/yolo_train_val_small --sampling-ratio 1 - --random-seed 0 --loglevel info + --random-seed 0 --exclude-background --loglevel info - uv run python ./scripts/model/yolo/train.py --data ./data/01_model_input/yolo_train_val_small/datasets/data.yaml --config ./scripts/model/yolo/configs/best.yaml --output-dir ./data/02_models/yolo/ @@ -61,13 +61,13 @@ stages: deps: - path: ./data/01_model_input/yolo_train_val hash: md5 - md5: 409302377938ce2a82f5338f3145cbbc.dir - size: 1914167911 - nfiles: 32201 + md5: da8f531db6ac70886c2793c3a6f18f16.dir + size: 2321533653 + nfiles: 37609 - path: ./scripts/data/model_input/build.py hash: md5 - md5: 7607af10f59267cf9859cb1d372aff26 - size: 6432 + md5: f745456171e8a8b24faca0e5f2bc0a22 + size: 7800 - path: ./scripts/model/yolo/configs/best.yaml hash: md5 md5: 82ab9c26b11cfc5f72e1398bfceeb838 @@ -79,8 +79,8 @@ stages: outs: - path: ./data/02_models/yolo/best hash: md5 - md5: 0e3617cd00b6a1985129110757bca1a0.dir - size: 44813862 + md5: b492ecf31cc3972aac757aaa5a23da2b.dir + size: 45354436 nfiles: 24 build_manifest_yolo_best: cmd: @@ -90,8 +90,8 @@ stages: deps: - path: ./data/02_models/yolo/best/ hash: md5 - md5: 0e3617cd00b6a1985129110757bca1a0.dir - size: 44813862 + md5: b492ecf31cc3972aac757aaa5a23da2b.dir + size: 45354436 nfiles: 24 - path: ./scripts/model/yolo/build_manifest.py hash: md5 @@ -100,8 +100,8 @@ stages: outs: - path: ./data/03_reporting/yolo/best/ hash: md5 - md5: c01eb3ce79f23e0999d3220813c01881.dir - size: 13529 + md5: d6c6cbd2e10730093dd22a6cfa5bde32.dir + size: 13530 nfiles: 1 export_yolo_best@onnx-cpu: cmd: @@ -111,8 +111,8 @@ stages: deps: - path: ./data/02_models/yolo/best/ hash: md5 - md5: 0e3617cd00b6a1985129110757bca1a0.dir - size: 44813862 + md5: b492ecf31cc3972aac757aaa5a23da2b.dir + size: 45354436 nfiles: 24 - path: ./scripts/model/yolo/export.py hash: md5 @@ -121,8 +121,8 @@ stages: outs: - path: ./data/02_models/yolo-export/best/onnx/cpu hash: md5 - md5: 3e101e69ac72dc3109be7afd2b61aa4a.dir - size: 38532985 + md5: cab4b58e34c96a125d3cb6643fb026d4.dir + size: 38199772 nfiles: 1 export_yolo_best@onnx-mps: cmd: @@ -153,8 +153,8 @@ stages: deps: - path: ./data/02_models/yolo/best/ hash: md5 - md5: 0e3617cd00b6a1985129110757bca1a0.dir - size: 44813862 + md5: b492ecf31cc3972aac757aaa5a23da2b.dir + size: 45354436 nfiles: 24 - path: ./scripts/model/yolo/export.py hash: md5 @@ -163,7 +163,7 @@ stages: outs: - path: ./data/02_models/yolo-export/best/ncnn/cpu hash: md5 - md5: f6f1f2b316198f427a51b3b8335e7b7c.dir + md5: 6ea19bd53eba5519c77a1ceaf7c669c5.dir size: 38001906 nfiles: 5 export_yolo_best@ncnn-mps: @@ -190,31 +190,30 @@ stages: fetch_model_input: cmd: - uv run dvc get https://github.com/pyronear/pyro-dataset - data/processed/yolo_train_val_corrected --rev v4.0.0-corrected --out + data/processed/yolo_train_val --rev v4.3.0 --out ./data/01_model_input/yolo_train_val || uv run dvc get - git@github.com:pyronear/pyro-dataset.git - data/processed/yolo_train_val_corrected --rev v4.0.0-corrected --out - ./data/01_model_input/yolo_train_val + git@github.com:pyronear/pyro-dataset.git data/processed/yolo_train_val + --rev v4.3.0 --out ./data/01_model_input/yolo_train_val outs: - path: ./data/01_model_input/yolo_train_val hash: md5 - md5: 409302377938ce2a82f5338f3145cbbc.dir - size: 1914167911 - nfiles: 32201 + md5: da8f531db6ac70886c2793c3a6f18f16.dir + size: 2321533653 + nfiles: 37609 fetch_sequential_val: cmd: - uv run dvc get https://github.com/pyronear/pyro-dataset - data/processed/sequential_train_val/val --rev v4.0.0-corrected --out + data/processed/sequential_train_val/val --rev v4.3.0 --out ./data/01_model_input/sequential_train_val/val || uv run dvc get git@github.com:pyronear/pyro-dataset.git - data/processed/sequential_train_val/val --rev v4.0.0-corrected --out + data/processed/sequential_train_val/val --rev v4.3.0 --out ./data/01_model_input/sequential_train_val/val outs: - path: ./data/01_model_input/sequential_train_val/val hash: md5 - md5: 652f9f7e73f297b782ed68e1b4844cce.dir - size: 671953151 - nfiles: 11478 + md5: df9cc34a9f322e39f85bda0d8e183463.dir + size: 855129379 + nfiles: 14054 evaluate_sequential_yolo_best: cmd: - uv run python ./scripts/model/yolo/evaluate_sequential.py --model-path @@ -252,12 +251,12 @@ stages: deps: - path: ./data/01_model_input/sequential_train_val/val hash: md5 - md5: 652f9f7e73f297b782ed68e1b4844cce.dir - size: 671953151 - nfiles: 11478 + md5: df9cc34a9f322e39f85bda0d8e183463.dir + size: 855129379 + nfiles: 14054 - path: ./data/02_models/yolo/best/weights/best.pt hash: md5 - md5: 550d1b91fb93e52cf233166b03405a10 + md5: f9740882540ebdc77d788184c0b9c873 size: 19225626 - path: predict_sequential.py hash: md5 @@ -266,9 +265,9 @@ stages: outs: - path: ./data/03_reporting/sequential/predictions_labels_val hash: md5 - md5: 796a49a2802fa773158ca168a680f12c.dir - size: 109228 - nfiles: 3340 + md5: e5d3d7e49f5a964df5521a6acedab22c.dir + size: 171362 + nfiles: 4106 optimize_sequential_val: cmd: - uv run python optimize_sequential.py --labels-dir @@ -278,19 +277,19 @@ stages: deps: - path: ./data/03_reporting/sequential/predictions_labels_val hash: md5 - md5: 796a49a2802fa773158ca168a680f12c.dir - size: 109228 - nfiles: 3340 + md5: e5d3d7e49f5a964df5521a6acedab22c.dir + size: 171362 + nfiles: 4106 - path: optimize_sequential.py hash: md5 - md5: 115eb45048da93018f7ea86b5f954270 - size: 8645 + md5: a032d3a9877382e4ba2d74e5c529aaab + size: 9087 outs: - path: ./data/03_reporting/sequential/grid_search_val.tsv hash: md5 - md5: aa6f9da227c79d6ffa2903e7b9a3d18b - size: 1873 + md5: 26192cbae025683faf060aa7c343d1ee + size: 1923 - path: ./data/03_reporting/sequential/grid_search_val_top20.tsv hash: md5 - md5: dd8e0aa4c8bc678deb27acbdd911c065 - size: 973 + md5: ab83d953fc37020b24e0e65aeb8f9a55 + size: 998 diff --git a/dvc.yaml b/dvc.yaml index a690d4f..22fc901 100644 --- a/dvc.yaml +++ b/dvc.yaml @@ -3,11 +3,11 @@ stages: cmd: - >- uv run dvc get https://github.com/pyronear/pyro-dataset - data/processed/yolo_train_val_corrected --rev v4.0.0-corrected + data/processed/yolo_train_val --rev v4.3.0 --out ./data/01_model_input/yolo_train_val || uv run dvc get git@github.com:pyronear/pyro-dataset.git - data/processed/yolo_train_val_corrected --rev v4.0.0-corrected + data/processed/yolo_train_val --rev v4.3.0 --out ./data/01_model_input/yolo_train_val outs: - ./data/01_model_input/yolo_train_val @@ -20,6 +20,7 @@ stages: --output-dir ./data/01_model_input/yolo_train_val_small --sampling-ratio 1 --random-seed 0 + --exclude-background --loglevel info - >- uv run python ./scripts/model/yolo/train.py @@ -53,11 +54,11 @@ stages: cmd: - >- uv run dvc get https://github.com/pyronear/pyro-dataset - data/processed/sequential_train_val/val --rev v4.0.0-corrected + data/processed/sequential_train_val/val --rev v4.3.0 --out ./data/01_model_input/sequential_train_val/val || uv run dvc get git@github.com:pyronear/pyro-dataset.git - data/processed/sequential_train_val/val --rev v4.0.0-corrected + data/processed/sequential_train_val/val --rev v4.3.0 --out ./data/01_model_input/sequential_train_val/val outs: - ./data/01_model_input/sequential_train_val/val diff --git a/results/eval_results.json b/results/eval_results.json index 1dc53e4..009659a 100644 --- a/results/eval_results.json +++ b/results/eval_results.json @@ -1,8 +1,8 @@ { - "precision": 0.7801831566503715, - "recall": 0.7129963898916968, - "map50": 0.7478004027603852, - "map50_95": 0.45667696600893326, + "precision": 0.6587947887747833, + "recall": 0.6950442255173176, + "map50": 0.6398575212990499, + "map50_95": 0.4020609511911931, "model_dir": "data/02_models/yolo/best", "data": "data/test/yolo_test/data.yaml", "split": "test" diff --git a/results/eval_sequential_results.json b/results/eval_sequential_results.json index 98329ff..461816b 100644 --- a/results/eval_sequential_results.json +++ b/results/eval_sequential_results.json @@ -1,12 +1,12 @@ { - "nb_consecutive_frames": 4, - "conf_thresh": 0.25, - "tp": 115, - "fn": 4, - "fp": 13, - "tn": 97, - "recall": 0.9664, - "fpr": 0.1182, - "precision": 0.8984, - "f1": 0.9312 + "nb_consecutive_frames": 6, + "conf_thresh": 0.35, + "tp": 148, + "fn": 7, + "fp": 33, + "tn": 109, + "recall": 0.9548, + "fpr": 0.2324, + "precision": 0.8177, + "f1": 0.881 } \ No newline at end of file diff --git a/scripts/data/model_input/build.py b/scripts/data/model_input/build.py index 7eadcfa..765360b 100644 --- a/scripts/data/model_input/build.py +++ b/scripts/data/model_input/build.py @@ -41,6 +41,11 @@ def make_cli_parser() -> argparse.ArgumentParser: default=0, type=int, ) + parser.add_argument( + "--exclude-background", + help="Exclude background images (empty label files) from the train split to train an ultra-sensitive model", + action="store_true", + ) parser.add_argument( "-log", "--loglevel", @@ -111,11 +116,20 @@ def copy_data(input_dir: Path, output_dir: Path) -> None: ) +def is_background(label_filepath: Path) -> bool: + """ + Return whether the given label file corresponds to a background image, + ie. a missing or empty label file (no smoke annotation). + """ + return not label_filepath.exists() or label_filepath.stat().st_size == 0 + + def sample_dataset( input_dir: Path, output_dir: Path, sampling_ratio: float = 0.1, random_seed: int = 0, + exclude_background: bool = False, ) -> list[dict]: """ Return a downsampled list of images and labels for the given @@ -124,6 +138,9 @@ def sample_dataset( Each element in the returned list has the following keys: - to: Path - where the image/label comes from - from: Path - where the image/label should be copied over - using the provided `output_dir` + + When `exclude_background` is set, background images (empty label files) + are dropped from the train split to train an ultra-sensitive model. """ assert 0 <= sampling_ratio <= 1.0, "sampling ratio should be between 0 and 1" @@ -132,6 +149,19 @@ def sample_dataset( labels_split_dir = input_dir / "labels" / split images_split_dir = input_dir / "images" / split images_filepaths = list(images_split_dir.glob("*.jpg")) + # Drop background images (empty labels) from the train split only, so + # that the val split stays intact for a comparable evaluation. + if exclude_background and split == "train": + n_before = len(images_filepaths) + images_filepaths = [ + fp + for fp in images_filepaths + if not is_background(labels_split_dir / f"{fp.stem}.txt") + ] + logging.info( + f"Excluded {n_before - len(images_filepaths)} background images " + f"from the {split} split ({len(images_filepaths)} remaining)" + ) n_images = len(images_filepaths) k = int(n_images * sampling_ratio) # For the val split we do not subsample as we want to eval on the same data as in the full @@ -202,6 +232,7 @@ def run_file_copy(copy_data: list[dict]) -> None: output_dir=output_dir / "datasets", sampling_ratio=sampling_ratio, random_seed=random_seed, + exclude_background=args["exclude_background"], ) ) write_data_yaml(output_dir / "datasets" / "data.yaml")