From 4868e3079bd04d8a7dfdc1a042a402a78788cd74 Mon Sep 17 00:00:00 2001 From: Ishika Aahna <123102828+codedbyishika@users.noreply.github.com> Date: Wed, 3 Jun 2026 16:43:23 -0400 Subject: [PATCH 01/24] Refractor Validation.py Move following functions: validate_csv_extension validate_output_directory validate_filename_uniqueness handle_missing_filenames setup_expected_columns --- AGENTS.md | 1 + src/cautiousrobot/__main__.py | 47 +++++---------------------------- src/cautiousrobot/validation.py | 44 ++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 40 deletions(-) create mode 100644 src/cautiousrobot/validation.py diff --git a/AGENTS.md b/AGENTS.md index 12bc364..ab1f09d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -17,6 +17,7 @@ ### Core Modules - `__main__.py` - Main entry point with CLI argument parsing and workflow orchestration +- `validation.py` - CSV validation and input handling functions - `download.py` - Core image downloading functionality with retry logic - `buddy_check.py` - Checksum validation and download verification using BuddyCheck class - `utils.py` - Helper functions for CSV processing, logging, and image downsampling diff --git a/src/cautiousrobot/__main__.py b/src/cautiousrobot/__main__.py index 464847a..7e18f73 100644 --- a/src/cautiousrobot/__main__.py +++ b/src/cautiousrobot/__main__.py @@ -14,6 +14,13 @@ from cautiousrobot.utils import process_csv, check_existing_images from cautiousrobot.buddy_check import BuddyCheck from cautiousrobot.download import download_images +from cautiousrobot.validation import ( + validate_csv_extension, + validate_output_directory, + validate_filename_uniqueness, + handle_missing_filenames, + setup_expected_columns, +) from cautiousrobot.__about__ import __version__ def parse_args(): @@ -53,46 +60,6 @@ def parse_args(): return parser.parse_args() -def validate_csv_extension(csv_path): - """Validate that the input file has a .csv extension.""" - if not csv_path.endswith(".csv"): - sys.exit("Expected CSV for input file; extension should be '.csv'") - - -def setup_expected_columns(args): - """Set up the expected columns dictionary for CSV processing.""" - subfolders = args.subdir_col - expected_cols = { - "filename_col": args.img_name_col.lower(), - "url_col": args.url_col.lower() - } - if subfolders: - subfolders = subfolders.lower() - expected_cols["subfolders"] = subfolders - return expected_cols, subfolders - - -def validate_filename_uniqueness(data_df, filename_col): - """Validate that the filename column contains unique values.""" - if data_df.loc[data_df[filename_col].notna()].shape[0] != data_df[filename_col].nunique(): - sys.exit(f"{filename_col} is not a unique identifier for this dataset, please choose a column with unique values for filenames.") - - -def handle_missing_filenames(data_df, filename_col, url_col): - """Handle cases where URLs exist but filenames are missing.""" - urls_no_name = len(data_df.loc[(data_df[filename_col].isna() & (data_df[url_col].notna()))]) - if urls_no_name > 0: - ignore = input(f"'{filename_col}' is missing values for {urls_no_name} URLs. Proceed with download ignoring these URLs? [y/n]: ") - if ignore.lower() != "y": - sys.exit("Exited without executing.") - - -def validate_output_directory(img_dir): - """Validate that the output directory doesn't already exist.""" - if os.path.exists(img_dir): - sys.exit(f"'{img_dir}' already exists. Exited without executing.") - - def setup_log_paths(csv_path): """Set up the log file paths based on the CSV path.""" metadata_path = csv_path.split(".")[0] diff --git a/src/cautiousrobot/validation.py b/src/cautiousrobot/validation.py new file mode 100644 index 0000000..961279c --- /dev/null +++ b/src/cautiousrobot/validation.py @@ -0,0 +1,44 @@ +#CSV validation and input handling functions. + +import os +import sys + + +def validate_csv_extension(csv_path): + """Validate that the input file has a .csv extension.""" + if not csv_path.endswith(".csv"): + sys.exit("Expected CSV for input file; extension should be '.csv'") + + +def validate_output_directory(img_dir): + """Validate that the output directory doesn't already exist.""" + if os.path.exists(img_dir): + sys.exit(f"'{img_dir}' already exists. Exited without executing.") + + +def validate_filename_uniqueness(data_df, filename_col): + """Validate that the filename column contains unique values.""" + if data_df.loc[data_df[filename_col].notna()].shape[0] != data_df[filename_col].nunique(): + sys.exit(f"{filename_col} is not a unique identifier for this dataset, please choose a column with unique values for filenames.") + + +def handle_missing_filenames(data_df, filename_col, url_col): + """Handle cases where URLs exist but filenames are missing.""" + urls_no_name = len(data_df.loc[(data_df[filename_col].isna() & (data_df[url_col].notna()))]) + if urls_no_name > 0: + ignore = input(f"'{filename_col}' is missing values for {urls_no_name} URLs. Proceed with download ignoring these URLs? [y/n]: ") + if ignore.lower() != "y": + sys.exit("Exited without executing.") + + +def setup_expected_columns(args): + """Set up the expected columns dictionary for CSV processing.""" + subfolders = args.subdir_col + expected_cols = { + "filename_col": args.img_name_col.lower(), + "url_col": args.url_col.lower() + } + if subfolders: + subfolders = subfolders.lower() + expected_cols["subfolders"] = subfolders + return expected_cols, subfolders From 784c5a43b3944c6d6c8faf66965c3b00460b7be6 Mon Sep 17 00:00:00 2001 From: Ishika Aahna <123102828+codedbyishika@users.noreply.github.com> Date: Thu, 4 Jun 2026 13:00:18 -0400 Subject: [PATCH 02/24] Remove unused imports --- src/cautiousrobot/__main__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/cautiousrobot/__main__.py b/src/cautiousrobot/__main__.py index 7e18f73..a9349e8 100644 --- a/src/cautiousrobot/__main__.py +++ b/src/cautiousrobot/__main__.py @@ -8,7 +8,6 @@ import pandas as pd import argparse import hashlib -import os import sys from sumbuddy import get_checksums from cautiousrobot.utils import process_csv, check_existing_images @@ -16,7 +15,6 @@ from cautiousrobot.download import download_images from cautiousrobot.validation import ( validate_csv_extension, - validate_output_directory, validate_filename_uniqueness, handle_missing_filenames, setup_expected_columns, From 5de93ee3e9e1eefde5d730796582e5f74794dd51 Mon Sep 17 00:00:00 2001 From: Ishika Aahna <123102828+codedbyishika@users.noreply.github.com> Date: Thu, 4 Jun 2026 13:25:33 -0400 Subject: [PATCH 03/24] Delete validate_output_directory Co-authored-by: Elizabeth Campolongo <38985481+egrace479@users.noreply.github.com> --- src/cautiousrobot/validation.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/cautiousrobot/validation.py b/src/cautiousrobot/validation.py index 961279c..b7e8872 100644 --- a/src/cautiousrobot/validation.py +++ b/src/cautiousrobot/validation.py @@ -10,12 +10,6 @@ def validate_csv_extension(csv_path): sys.exit("Expected CSV for input file; extension should be '.csv'") -def validate_output_directory(img_dir): - """Validate that the output directory doesn't already exist.""" - if os.path.exists(img_dir): - sys.exit(f"'{img_dir}' already exists. Exited without executing.") - - def validate_filename_uniqueness(data_df, filename_col): """Validate that the filename column contains unique values.""" if data_df.loc[data_df[filename_col].notna()].shape[0] != data_df[filename_col].nunique(): From bb336a6c8b509d1f780018cc8033a619e275fc19 Mon Sep 17 00:00:00 2001 From: Elizabeth Campolongo <38985481+egrace479@users.noreply.github.com> Date: Thu, 4 Jun 2026 13:37:10 -0400 Subject: [PATCH 04/24] Remove unused import --- src/cautiousrobot/validation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/cautiousrobot/validation.py b/src/cautiousrobot/validation.py index b7e8872..af26986 100644 --- a/src/cautiousrobot/validation.py +++ b/src/cautiousrobot/validation.py @@ -1,6 +1,5 @@ #CSV validation and input handling functions. -import os import sys From 930e33d0b67af47b8f70845fa88ca1192ef79b4f Mon Sep 17 00:00:00 2001 From: Elizabeth Campolongo <38985481+egrace479@users.noreply.github.com> Date: Thu, 4 Jun 2026 15:23:53 -0400 Subject: [PATCH 05/24] small formatting fix --- src/cautiousrobot/validation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cautiousrobot/validation.py b/src/cautiousrobot/validation.py index af26986..1d82cc5 100644 --- a/src/cautiousrobot/validation.py +++ b/src/cautiousrobot/validation.py @@ -1,4 +1,4 @@ -#CSV validation and input handling functions. +# CSV validation and input handling functions. import sys From a67fe33358f91066ef7fd779ea5617e298fc4f95 Mon Sep 17 00:00:00 2001 From: Ishika Aahna <123102828+codedbyishika@users.noreply.github.com> Date: Fri, 5 Jun 2026 15:00:54 -0400 Subject: [PATCH 06/24] Create test_validation.py --- tests/test_validation.py | 352 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 352 insertions(+) create mode 100644 tests/test_validation.py diff --git a/tests/test_validation.py b/tests/test_validation.py new file mode 100644 index 0000000..6fa2ecf --- /dev/null +++ b/tests/test_validation.py @@ -0,0 +1,352 @@ +import unittest +from unittest.mock import patch, MagicMock +import pandas as pd +import sys + +from cautiousrobot.validation import ( + validate_csv_extension, + validate_filename_uniqueness, + handle_missing_filenames, + setup_expected_columns, +) + + +class TestValidateCsvExtension(unittest.TestCase): + """Test CSV file extension validation.""" + + def test_valid_csv_extension(self): + """Should not raise exception for valid .csv file.""" + # Should not raise any exception + try: + validate_csv_extension("data.csv") + except SystemExit: + self.fail("validate_csv_extension raised SystemExit unexpectedly") + + def test_csv_extension_case_sensitive(self): + """Should accept .csv regardless of path content.""" + try: + validate_csv_extension("MyData.csv") + validate_csv_extension("path/to/data.csv") + except SystemExit: + self.fail("validate_csv_extension raised SystemExit unexpectedly") + + def test_invalid_txt_extension(self): + """Should exit if file has .txt extension.""" + with self.assertRaises(SystemExit) as cm: + validate_csv_extension("data.txt") + self.assertIn("csv", cm.exception.code.lower()) + + def test_invalid_xlsx_extension(self): + """Should exit if file has .xlsx extension.""" + with self.assertRaises(SystemExit) as cm: + validate_csv_extension("data.xlsx") + self.assertIn("csv", cm.exception.code.lower()) + + def test_invalid_no_extension(self): + """Should exit if file has no extension.""" + with self.assertRaises(SystemExit) as cm: + validate_csv_extension("data") + self.assertIn("csv", cm.exception.code.lower()) + + def test_invalid_csv_as_substring(self): + """Should require .csv as actual extension, not substring.""" + with self.assertRaises(SystemExit) as cm: + validate_csv_extension("data.csv.backup") + self.assertIn("csv", cm.exception.code.lower()) + + +class TestValidateFilenameUniqueness(unittest.TestCase): + """Test filename uniqueness validation.""" + + def test_unique_filenames(self): + """Should not raise exception for unique filenames.""" + df = pd.DataFrame({ + "filename": ["image1.jpg", "image2.jpg", "image3.jpg"] + }) + try: + validate_filename_uniqueness(df, "filename") + except SystemExit: + self.fail("validate_filename_uniqueness raised SystemExit unexpectedly") + + def test_duplicate_filenames(self): + """Should exit if filenames are not unique.""" + df = pd.DataFrame({ + "filename": ["image1.jpg", "image1.jpg", "image2.jpg"] + }) + with self.assertRaises(SystemExit) as cm: + validate_filename_uniqueness(df, "filename") + self.assertIn("unique identifier", cm.exception.code.lower()) + + def test_filenames_with_missing_values(self): + """Should only count non-null filenames for uniqueness.""" + df = pd.DataFrame({ + "filename": ["image1.jpg", None, "image2.jpg", None] + }) + try: + validate_filename_uniqueness(df, "filename") + except SystemExit: + self.fail("validate_filename_uniqueness raised SystemExit unexpectedly") + + def test_all_missing_filenames(self): + """Should not raise exception if all filenames are missing.""" + df = pd.DataFrame({ + "filename": [None, None, None] + }) + try: + validate_filename_uniqueness(df, "filename") + except SystemExit: + self.fail("validate_filename_uniqueness raised SystemExit unexpectedly") + + def test_single_filename(self): + """Should not raise exception for single unique filename.""" + df = pd.DataFrame({ + "filename": ["image1.jpg"] + }) + try: + validate_filename_uniqueness(df, "filename") + except SystemExit: + self.fail("validate_filename_uniqueness raised SystemExit unexpectedly") + + def test_empty_dataframe(self): + """Should not raise exception for empty dataframe.""" + df = pd.DataFrame({ + "filename": [] + }) + try: + validate_filename_uniqueness(df, "filename") + except SystemExit: + self.fail("validate_filename_uniqueness raised SystemExit unexpectedly") + + def test_custom_column_name(self): + """Should work with custom column names.""" + df = pd.DataFrame({ + "image_name": ["photo1.jpg", "photo2.jpg", "photo3.jpg"] + }) + try: + validate_filename_uniqueness(df, "image_name") + except SystemExit: + self.fail("validate_filename_uniqueness raised SystemExit unexpectedly") + + def test_duplicate_with_custom_column(self): + """Should detect duplicates in custom column names.""" + df = pd.DataFrame({ + "image_id": ["img_001", "img_001", "img_002"] + }) + with self.assertRaises(SystemExit) as cm: + validate_filename_uniqueness(df, "image_id") + self.assertIn("unique identifier", cm.exception.code.lower()) + + +class TestHandleMissingFilenames(unittest.TestCase): + """Test handling of missing filename values.""" + + def setUp(self): + """Set up test fixtures.""" + self.filename_col = "filename" + self.url_col = "file_url" + + def test_no_missing_filenames(self): + """Should not prompt if all filenames are present.""" + df = pd.DataFrame({ + self.filename_col: ["image1.jpg", "image2.jpg"], + self.url_col: ["http://url1.com", "http://url2.com"] + }) + with patch("builtins.input") as mock_input: + handle_missing_filenames(df, self.filename_col, self.url_col) + mock_input.assert_not_called() + + def test_missing_filenames_with_urls_user_accepts(self): + """Should continue when user answers 'y' to missing filenames prompt.""" + df = pd.DataFrame({ + self.filename_col: ["image1.jpg", None, "image3.jpg"], + self.url_col: ["http://url1.com", "http://url2.com", "http://url3.com"] + }) + with patch("builtins.input", return_value="y"): + try: + handle_missing_filenames(df, self.filename_col, self.url_col) + except SystemExit: + self.fail("Should not exit when user answers 'y'") + + def test_missing_filenames_with_urls_user_declines(self): + """Should exit when user answers 'n' to missing filenames prompt.""" + df = pd.DataFrame({ + self.filename_col: ["image1.jpg", None, "image3.jpg"], + self.url_col: ["http://url1.com", "http://url2.com", "http://url3.com"] + }) + with patch("builtins.input", return_value="n"): + with self.assertRaises(SystemExit) as cm: + handle_missing_filenames(df, self.filename_col, self.url_col) + self.assertIn("Exited", cm.exception.code) + + def test_missing_filenames_user_input_case_insensitive(self): + """Should accept 'Y' or other case variations for yes response.""" + df = pd.DataFrame({ + self.filename_col: ["image1.jpg", None], + self.url_col: ["http://url1.com", "http://url2.com"] + }) + with patch("builtins.input", return_value="Y"): + try: + handle_missing_filenames(df, self.filename_col, self.url_col) + except SystemExit: + self.fail("Should accept 'Y' as valid yes response") + + def test_missing_filenames_prompt_shows_count(self): + """Should show number of missing filenames in prompt.""" + df = pd.DataFrame({ + self.filename_col: ["img1.jpg", None, None, "img4.jpg"], + self.url_col: ["url1", "url2", "url3", "url4"] + }) + with patch("builtins.input", return_value="n") as mock_input: + with self.assertRaises(SystemExit): + handle_missing_filenames(df, self.filename_col, self.url_col) + # Check that the input prompt includes the count + call_args = mock_input.call_args[0][0] + self.assertIn("2", call_args) + + def test_missing_filenames_no_urls(self): + """Should not prompt if no URLs exist (no missing filenames to handle).""" + df = pd.DataFrame({ + self.filename_col: ["image1.jpg", None, "image3.jpg"], + self.url_col: [None, None, None] + }) + with patch("builtins.input") as mock_input: + handle_missing_filenames(df, self.filename_col, self.url_col) + mock_input.assert_not_called() + + def test_missing_filenames_mixed_urls_and_filenames(self): + """Should only count rows with URLs but missing filenames.""" + df = pd.DataFrame({ + self.filename_col: [None, "image2.jpg", None, "image4.jpg"], + self.url_col: ["http://url1.com", None, "http://url3.com", "http://url4.com"] + }) + with patch("builtins.input", return_value="n") as mock_input: + with self.assertRaises(SystemExit): + handle_missing_filenames(df, self.filename_col, self.url_col) + # Should count 2 URLs without filenames (rows 0 and 2) + call_args = mock_input.call_args[0][0] + self.assertIn("2", call_args) + + def test_missing_filenames_empty_dataframe(self): + """Should not prompt if dataframe is empty.""" + df = pd.DataFrame({ + self.filename_col: [], + self.url_col: [] + }) + with patch("builtins.input") as mock_input: + handle_missing_filenames(df, self.filename_col, self.url_col) + mock_input.assert_not_called() + + +class TestSetupExpectedColumns(unittest.TestCase): + """Test expected columns setup.""" + + def test_basic_columns_setup(self): + """Should return expected columns with basic arguments.""" + args = MagicMock() + args.img_name_col = "filename" + args.url_col = "file_url" + args.subdir_col = None + + expected_cols, subfolders = setup_expected_columns(args) + + self.assertEqual(expected_cols["filename_col"], "filename") + self.assertEqual(expected_cols["url_col"], "file_url") + self.assertNotIn("subfolders", expected_cols) + self.assertIsNone(subfolders) + + def test_columns_lowercase_conversion(self): + """Should convert column names to lowercase.""" + args = MagicMock() + args.img_name_col = "FileName" + args.url_col = "FileURL" + args.subdir_col = None + + expected_cols, subfolders = setup_expected_columns(args) + + self.assertEqual(expected_cols["filename_col"], "filename") + self.assertEqual(expected_cols["url_col"], "fileurl") + + def test_subdir_column_setup(self): + """Should include subdir column when provided.""" + args = MagicMock() + args.img_name_col = "filename" + args.url_col = "file_url" + args.subdir_col = "species" + + expected_cols, subfolders = setup_expected_columns(args) + + self.assertEqual(expected_cols["subfolders"], "species") + self.assertEqual(subfolders, "species") + + def test_subdir_column_lowercase(self): + """Should convert subdir column name to lowercase.""" + args = MagicMock() + args.img_name_col = "filename" + args.url_col = "file_url" + args.subdir_col = "Species" + + expected_cols, subfolders = setup_expected_columns(args) + + self.assertEqual(expected_cols["subfolders"], "species") + self.assertEqual(subfolders, "species") + + def test_custom_column_names_with_subdir(self): + """Should handle custom column names with subdir.""" + args = MagicMock() + args.img_name_col = "ImageID" + args.url_col = "DownloadURL" + args.subdir_col = "Category" + + expected_cols, subfolders = setup_expected_columns(args) + + self.assertEqual(expected_cols["filename_col"], "imageid") + self.assertEqual(expected_cols["url_col"], "downloadurl") + self.assertEqual(expected_cols["subfolders"], "category") + self.assertEqual(subfolders, "category") + + def test_returns_subfolders_none_when_not_provided(self): + """Should return None for subfolders when not provided in args.""" + args = MagicMock() + args.img_name_col = "filename" + args.url_col = "file_url" + args.subdir_col = None + + expected_cols, subfolders = setup_expected_columns(args) + + self.assertIsNone(subfolders) + self.assertNotIn("subfolders", expected_cols) + + def test_returns_subfolders_value_when_provided(self): + """Should return subfolders value when provided.""" + args = MagicMock() + args.img_name_col = "filename" + args.url_col = "file_url" + args.subdir_col = "region" + + expected_cols, subfolders = setup_expected_columns(args) + + self.assertEqual(subfolders, "region") + self.assertIn("subfolders", expected_cols) + + def test_dictionary_structure(self): + """Should return correctly structured dictionary.""" + args = MagicMock() + args.img_name_col = "fname" + args.url_col = "url" + args.subdir_col = "folder" + + expected_cols, subfolders = setup_expected_columns(args) + + # Check dictionary has correct keys + self.assertIn("filename_col", expected_cols) + self.assertIn("url_col", expected_cols) + self.assertIn("subfolders", expected_cols) + + # Check values are correct + self.assertEqual(expected_cols["filename_col"], "fname") + self.assertEqual(expected_cols["url_col"], "url") + self.assertEqual(expected_cols["subfolders"], "folder") + + +if __name__ == "__main__": + unittest.main() From 48e72d619aac9c644161455f8aea038d48083bd3 Mon Sep 17 00:00:00 2001 From: Ishika Aahna <123102828+codedbyishika@users.noreply.github.com> Date: Fri, 5 Jun 2026 15:02:02 -0400 Subject: [PATCH 07/24] Fix imports --- tests/test_validation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_validation.py b/tests/test_validation.py index 6fa2ecf..7a288a3 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -1,7 +1,6 @@ import unittest from unittest.mock import patch, MagicMock import pandas as pd -import sys from cautiousrobot.validation import ( validate_csv_extension, From c2d1b38b40837fc31d0aae96f72ed4b60fd6d0a0 Mon Sep 17 00:00:00 2001 From: Ishika Aahna <123102828+codedbyishika@users.noreply.github.com> Date: Fri, 12 Jun 2026 12:37:41 -0400 Subject: [PATCH 08/24] Create RollCall Module - Change tests - Include check_existing_images --- src/cautiousrobot/__main__.py | 21 ++++----- src/cautiousrobot/rollcall.py | 84 +++++++++++++++++++++++++++++++++ src/cautiousrobot/utils.py | 74 ++--------------------------- src/cautiousrobot/validation.py | 37 --------------- tests/test_validation.py | 2 +- 5 files changed, 98 insertions(+), 120 deletions(-) create mode 100644 src/cautiousrobot/rollcall.py delete mode 100644 src/cautiousrobot/validation.py diff --git a/src/cautiousrobot/__main__.py b/src/cautiousrobot/__main__.py index a9349e8..92715bb 100644 --- a/src/cautiousrobot/__main__.py +++ b/src/cautiousrobot/__main__.py @@ -10,15 +10,10 @@ import hashlib import sys from sumbuddy import get_checksums -from cautiousrobot.utils import process_csv, check_existing_images +from cautiousrobot.utils import process_csv from cautiousrobot.buddy_check import BuddyCheck from cautiousrobot.download import download_images -from cautiousrobot.validation import ( - validate_csv_extension, - validate_filename_uniqueness, - handle_missing_filenames, - setup_expected_columns, -) +from cautiousrobot.rollcall import RollCall from cautiousrobot.__about__ import __version__ def parse_args(): @@ -112,11 +107,13 @@ def main(): args = parse_args() csv_path = args.input_file + roll_call = RollCall() + # Validate CSV extension - validate_csv_extension(csv_path) + roll_call.validate_csv_extension(csv_path) # Set up expected columns and process CSV - expected_cols, subfolders = setup_expected_columns(args) + expected_cols, subfolders = roll_call.setup_expected_columns(args) try: data_df = process_csv(csv_path, expected_cols) except Exception as missing_cols: @@ -125,15 +122,15 @@ def main(): # Validate data and handle missing filenames filename_col = expected_cols["filename_col"] url_col = expected_cols["url_col"] - validate_filename_uniqueness(data_df, filename_col) - handle_missing_filenames(data_df, filename_col, url_col) + roll_call.validate_filename_uniqueness(data_df, filename_col) + roll_call.handle_missing_filenames(data_df, filename_col, url_col) # Set source DataFrame for only non-null filename values source_df = data_df.loc[data_df[filename_col].notna()].copy() # Validate and handle existing output directory img_dir = args.output_dir - source_df, filtered_df = check_existing_images(csv_path, img_dir, source_df, filename_col, subfolders) + source_df, filtered_df = roll_call.check_existing_images(csv_path, img_dir, source_df, filename_col, subfolders) # Set up log paths log_filepath, error_log_filepath, metadata_path = setup_log_paths(csv_path) diff --git a/src/cautiousrobot/rollcall.py b/src/cautiousrobot/rollcall.py new file mode 100644 index 0000000..d5eb4ac --- /dev/null +++ b/src/cautiousrobot/rollcall.py @@ -0,0 +1,84 @@ +# CSV validation and input handling functions. + +import os +import sys +from sumbuddy import gather_file_paths +from sumbuddy.exceptions import EmptyInputDirectoryError + + +class RollCall: + """Pre-download validation and directory verification.""" + + def validate_csv_extension(self, csv_path): + """Validate that the input file has a .csv extension.""" + if not csv_path.endswith(".csv"): + sys.exit("Expected CSV for input file; extension should be '.csv'") + + def validate_filename_uniqueness(self, data_df, filename_col): + """Validate that the filename column contains unique values.""" + if data_df.loc[data_df[filename_col].notna()].shape[0] != data_df[filename_col].nunique(): + sys.exit(f"{filename_col} is not a unique identifier for this dataset, please choose a column with unique values for filenames.") + + def handle_missing_filenames(self, data_df, filename_col, url_col): + """Handle cases where URLs exist but filenames are missing.""" + urls_no_name = len(data_df.loc[(data_df[filename_col].isna() & (data_df[url_col].notna()))]) + if urls_no_name > 0: + ignore = input(f"'{filename_col}' is missing values for {urls_no_name} URLs. Proceed with download ignoring these URLs? [y/n]: ") + if ignore.lower() != "y": + sys.exit("Exited without executing.") + + def setup_expected_columns(self, args): + """Set up the expected columns dictionary for CSV processing.""" + subfolders = args.subdir_col + expected_cols = { + "filename_col": args.img_name_col.lower(), + "url_col": args.url_col.lower() + } + if subfolders: + subfolders = subfolders.lower() + expected_cols["subfolders"] = subfolders + return expected_cols, subfolders + + def check_existing_images(self, csv_path, img_dir, source_df, filename_col, subfolders=None): + """Checks which files from the CSV already exist in the image directory.""" + df = source_df.copy() + + if not os.path.exists(img_dir): + df["in_img_dir"] = False + filtered_df = df[~df["in_img_dir"]].copy() + return df, filtered_df + + try: + existing_files = gather_file_paths(img_dir) + except EmptyInputDirectoryError: + existing_files = [] + + existing_full_paths = {os.path.normpath(os.path.relpath(f, img_dir)) for f in existing_files} + + if subfolders: + raw_paths = df[subfolders].astype(str) + os.sep + df[filename_col].astype(str) + df["expected_path"] = raw_paths.apply(os.path.normpath) + else: + df["expected_path"] = df[filename_col].astype(str).apply(os.path.normpath) + + expected_present = df["expected_path"].isin(existing_full_paths) + df["in_img_dir"] = expected_present.copy() + df = df.drop(columns=["expected_path"]) + filtered_df = df[~df["in_img_dir"]].copy() + + if filtered_df.empty: + sys.exit(f"'{img_dir}' already contains all images. Exited without executing.") + else: + num_existing = len(existing_files) + print(f"There are {num_existing} of the desired files already in {img_dir}. Based on {csv_path}, {filtered_df.shape[0]} images should be downloaded.") + + return df, filtered_df + + +_rollcall = RollCall() + +validate_csv_extension = _rollcall.validate_csv_extension +validate_filename_uniqueness = _rollcall.validate_filename_uniqueness +handle_missing_filenames = _rollcall.handle_missing_filenames +setup_expected_columns = _rollcall.setup_expected_columns +check_existing_images = _rollcall.check_existing_images diff --git a/src/cautiousrobot/utils.py b/src/cautiousrobot/utils.py index ac4f4d9..daefd63 100644 --- a/src/cautiousrobot/utils.py +++ b/src/cautiousrobot/utils.py @@ -7,6 +7,7 @@ from PIL import Image from sumbuddy import gather_file_paths from sumbuddy.exceptions import EmptyInputDirectoryError +from cautiousrobot.rollcall import RollCall def log_response(log_data, index, image, file_path, response_code): @@ -84,73 +85,6 @@ def downsample_and_save_image(image_dir_path, image_name, downsample_dir_path, d ) update_log(log=log_errors, index=image_index, filepath=error_log_filepath) -def check_existing_images(csv_path, img_dir, source_df, filename_col, subfolders = None): - """ - Checks which files from the CSV already exist in the image directory. - - Adds a new boolean column `in_img_dir` to source_df indicating which images - are already in the directory. - - If all images already exist in the directory, the function will exit early - by calling `sys.exit()`, and no further processing will occur. - - Parameters: - csv_path (str): Path to the CSV file containing image information. - img_dir (str): Path to the directory where images are to be stored. - source_df (pd.DataFrame): DataFrame loaded from the CSV, containing image metadata. - filename_col (str): Name of the column in source_df that contains image filenames. - subfolders (str): Name of the column in source_df that contains subfolder names. (optional) - - Returns: - updated_df (pd.DataFrame): DataFrame with new column 'in_img_dir' indicating presence in img_dir. - filtered_df (pd.DataFrame): DataFrame filtered to only files not present in img_dir. - """ - # Create a copy to avoid modifying the original DataFrame - df = source_df.copy() - - if not os.path.exists(img_dir): - # Directory doesn't exist, so nothing to check - df["in_img_dir"] = False - - # Return the updated df and the filtered dataframe of items that still need downloading - filtered_df = df[~df["in_img_dir"]].copy() - return df, filtered_df - - try: - existing_files = gather_file_paths(img_dir) - except EmptyInputDirectoryError: - # If the directory exists but is empty, sumbuddy raises an error. - # We catch it and treat it as an empty file list. - existing_files = [] - - existing_full_paths = {os.path.normpath(os.path.relpath(f, img_dir)) for f in existing_files} - - if subfolders: - # We use a generic join here, but the apply(os.path.normpath) below fixes it for the specific OS - raw_paths = df[subfolders].astype(str) + os.sep + df[filename_col].astype(str) - - # This converts '/' to '\' on Windows, or vice versa, ensuring a match - df["expected_path"] = raw_paths.apply(os.path.normpath) - else: - # Normalize even simple filenames just in case they contain pathing characters - df["expected_path"] = df[filename_col].astype(str).apply(os.path.normpath) - - # Determine which expected paths physically exist - expected_present = df["expected_path"].isin(existing_full_paths) - df["in_img_dir"] = expected_present.copy() - - # Clean up the temporary column before returning. - df = df.drop(columns=["expected_path"]) - - # Create filtered DataFrame - filtered_df = df[~df["in_img_dir"]].copy() - - # Exit if all images are already there - if filtered_df.empty: - sys.exit(f"'{img_dir}' already contains all images. Exited without executing.") - else: - # Print directory status message - pre-download - num_existing = len(existing_files) - print(f"There are {num_existing} of the desired files already in {img_dir}. Based on {csv_path}, {filtered_df.shape[0]} images should be downloaded.") - - return df, filtered_df +def check_existing_images(csv_path, img_dir, source_df, filename_col, subfolders=None): + """Legacy wrapper for RollCall.check_existing_images.""" + return RollCall().check_existing_images(csv_path, img_dir, source_df, filename_col, subfolders) diff --git a/src/cautiousrobot/validation.py b/src/cautiousrobot/validation.py deleted file mode 100644 index 1d82cc5..0000000 --- a/src/cautiousrobot/validation.py +++ /dev/null @@ -1,37 +0,0 @@ -# CSV validation and input handling functions. - -import sys - - -def validate_csv_extension(csv_path): - """Validate that the input file has a .csv extension.""" - if not csv_path.endswith(".csv"): - sys.exit("Expected CSV for input file; extension should be '.csv'") - - -def validate_filename_uniqueness(data_df, filename_col): - """Validate that the filename column contains unique values.""" - if data_df.loc[data_df[filename_col].notna()].shape[0] != data_df[filename_col].nunique(): - sys.exit(f"{filename_col} is not a unique identifier for this dataset, please choose a column with unique values for filenames.") - - -def handle_missing_filenames(data_df, filename_col, url_col): - """Handle cases where URLs exist but filenames are missing.""" - urls_no_name = len(data_df.loc[(data_df[filename_col].isna() & (data_df[url_col].notna()))]) - if urls_no_name > 0: - ignore = input(f"'{filename_col}' is missing values for {urls_no_name} URLs. Proceed with download ignoring these URLs? [y/n]: ") - if ignore.lower() != "y": - sys.exit("Exited without executing.") - - -def setup_expected_columns(args): - """Set up the expected columns dictionary for CSV processing.""" - subfolders = args.subdir_col - expected_cols = { - "filename_col": args.img_name_col.lower(), - "url_col": args.url_col.lower() - } - if subfolders: - subfolders = subfolders.lower() - expected_cols["subfolders"] = subfolders - return expected_cols, subfolders diff --git a/tests/test_validation.py b/tests/test_validation.py index 7a288a3..d776db9 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -2,7 +2,7 @@ from unittest.mock import patch, MagicMock import pandas as pd -from cautiousrobot.validation import ( +from cautiousrobot.rollcall import ( validate_csv_extension, validate_filename_uniqueness, handle_missing_filenames, From ff930b0bdb5678bd2df53c49a029598213f40eb8 Mon Sep 17 00:00:00 2001 From: Ishika Aahna <123102828+codedbyishika@users.noreply.github.com> Date: Fri, 12 Jun 2026 12:41:40 -0400 Subject: [PATCH 09/24] Change Imports --- src/cautiousrobot/utils.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/cautiousrobot/utils.py b/src/cautiousrobot/utils.py index daefd63..e36e0cb 100644 --- a/src/cautiousrobot/utils.py +++ b/src/cautiousrobot/utils.py @@ -1,12 +1,9 @@ # Helper functions for download import json -import sys import pandas as pd import os from PIL import Image -from sumbuddy import gather_file_paths -from sumbuddy.exceptions import EmptyInputDirectoryError from cautiousrobot.rollcall import RollCall From d14d7ac016672359466baa60600b67fa6fca3c7a Mon Sep 17 00:00:00 2001 From: Ishika Aahna <123102828+codedbyishika@users.noreply.github.com> Date: Fri, 12 Jun 2026 14:25:54 -0400 Subject: [PATCH 10/24] Revert "Change Imports" This reverts commit ff930b0bdb5678bd2df53c49a029598213f40eb8. --- src/cautiousrobot/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/cautiousrobot/utils.py b/src/cautiousrobot/utils.py index e36e0cb..daefd63 100644 --- a/src/cautiousrobot/utils.py +++ b/src/cautiousrobot/utils.py @@ -1,9 +1,12 @@ # Helper functions for download import json +import sys import pandas as pd import os from PIL import Image +from sumbuddy import gather_file_paths +from sumbuddy.exceptions import EmptyInputDirectoryError from cautiousrobot.rollcall import RollCall From 9d46b5f9ceb509648ec186c49cff9a433f6d5d96 Mon Sep 17 00:00:00 2001 From: Ishika Aahna <123102828+codedbyishika@users.noreply.github.com> Date: Fri, 12 Jun 2026 15:02:05 -0400 Subject: [PATCH 11/24] Change Tests and Imports --- AGENTS.md | 2 +- src/cautiousrobot/rollcall.py | 7 -- src/cautiousrobot/utils.py | 10 +-- tests/test_existing_images.py | 35 ++++---- .../{test_validation.py => test_rollcall.py} | 79 ++++++++++--------- 5 files changed, 62 insertions(+), 71 deletions(-) rename tests/{test_validation.py => test_rollcall.py} (81%) diff --git a/AGENTS.md b/AGENTS.md index ab1f09d..4dd7e2f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -17,7 +17,7 @@ ### Core Modules - `__main__.py` - Main entry point with CLI argument parsing and workflow orchestration -- `validation.py` - CSV validation and input handling functions +- `rollcall.py` - CSV validation and input handling functions - `download.py` - Core image downloading functionality with retry logic - `buddy_check.py` - Checksum validation and download verification using BuddyCheck class - `utils.py` - Helper functions for CSV processing, logging, and image downsampling diff --git a/src/cautiousrobot/rollcall.py b/src/cautiousrobot/rollcall.py index d5eb4ac..0ca9aeb 100644 --- a/src/cautiousrobot/rollcall.py +++ b/src/cautiousrobot/rollcall.py @@ -75,10 +75,3 @@ def check_existing_images(self, csv_path, img_dir, source_df, filename_col, subf return df, filtered_df -_rollcall = RollCall() - -validate_csv_extension = _rollcall.validate_csv_extension -validate_filename_uniqueness = _rollcall.validate_filename_uniqueness -handle_missing_filenames = _rollcall.handle_missing_filenames -setup_expected_columns = _rollcall.setup_expected_columns -check_existing_images = _rollcall.check_existing_images diff --git a/src/cautiousrobot/utils.py b/src/cautiousrobot/utils.py index daefd63..22900e3 100644 --- a/src/cautiousrobot/utils.py +++ b/src/cautiousrobot/utils.py @@ -1,13 +1,9 @@ # Helper functions for download import json -import sys import pandas as pd import os from PIL import Image -from sumbuddy import gather_file_paths -from sumbuddy.exceptions import EmptyInputDirectoryError -from cautiousrobot.rollcall import RollCall def log_response(log_data, index, image, file_path, response_code): @@ -83,8 +79,4 @@ def downsample_and_save_image(image_dir_path, image_name, downsample_dir_path, d file_path=file_path, response_code=str(e) ) - update_log(log=log_errors, index=image_index, filepath=error_log_filepath) - -def check_existing_images(csv_path, img_dir, source_df, filename_col, subfolders=None): - """Legacy wrapper for RollCall.check_existing_images.""" - return RollCall().check_existing_images(csv_path, img_dir, source_df, filename_col, subfolders) + update_log(log=log_errors, index=image_index, filepath=error_log_filepath) \ No newline at end of file diff --git a/tests/test_existing_images.py b/tests/test_existing_images.py index eb2b405..38c4a6d 100644 --- a/tests/test_existing_images.py +++ b/tests/test_existing_images.py @@ -1,7 +1,7 @@ import unittest import pandas as pd from unittest.mock import patch -from cautiousrobot.utils import check_existing_images +from cautiousrobot.rollcall import RollCall class TestCheckExistingImages(unittest.TestCase): @@ -12,11 +12,12 @@ def setUp(self): self.sample_df = pd.DataFrame({ self.filename_col: ["a.jpg", "b.jpg", "c.jpg"] }) + self.rollcall = RollCall() - @patch("cautiousrobot.utils.os.path.exists", return_value=False) + @patch("cautiousrobot.rollcall.os.path.exists", return_value=False) def test_directory_does_not_exist(self, mock_exists): """If image directory doesn't exist, all images marked as not in directory.""" - updated_df, filtered_df = check_existing_images( + updated_df, filtered_df = self.rollcall.check_existing_images( self.csv_path, self.img_dir, self.sample_df, self.filename_col ) @@ -24,12 +25,12 @@ def test_directory_does_not_exist(self, mock_exists): self.assertEqual(len(filtered_df), len(self.sample_df)) mock_exists.assert_called_once_with(self.img_dir) - @patch("cautiousrobot.utils.os.path.exists", return_value=True) - @patch("cautiousrobot.utils.gather_file_paths", return_value=["test_images/a.jpg"]) - @patch("cautiousrobot.utils.print") + @patch("cautiousrobot.rollcall.os.path.exists", return_value=True) + @patch("cautiousrobot.rollcall.gather_file_paths", return_value=["test_images/a.jpg"]) + @patch("cautiousrobot.rollcall.print") def test_some_files_exist(self, mock_print, mock_gather, mock_exists): """Should mark existing files correctly and print status.""" - updated_df, filtered_df = check_existing_images( + updated_df, filtered_df = self.rollcall.check_existing_images( self.csv_path, self.img_dir, self.sample_df, self.filename_col ) @@ -40,24 +41,24 @@ def test_some_files_exist(self, mock_print, mock_gather, mock_exists): mock_print.assert_called_once() self.assertIn("There are 1 of the desired files", mock_print.call_args[0][0]) - @patch("cautiousrobot.utils.os.path.exists", return_value=True) - @patch("cautiousrobot.utils.gather_file_paths", return_value=["test_images/a.jpg", "test_images/b.jpg", "test_images/c.jpg"]) + @patch("cautiousrobot.rollcall.os.path.exists", return_value=True) + @patch("cautiousrobot.rollcall.gather_file_paths", return_value=["test_images/a.jpg", "test_images/b.jpg", "test_images/c.jpg"]) def test_all_files_exist_exits(self, mock_gather, mock_exists): """If all images exist, should exit early with proper message.""" with self.assertRaises(SystemExit) as cm: - check_existing_images( + self.rollcall.check_existing_images( self.csv_path, self.img_dir, self.sample_df, self.filename_col ) self.assertIn("already contains all images", cm.exception.code) mock_exists.assert_called_once_with(self.img_dir) - @patch("cautiousrobot.utils.os.path.exists", return_value=True) - @patch("cautiousrobot.utils.gather_file_paths", return_value=[]) - @patch("cautiousrobot.utils.print") + @patch("cautiousrobot.rollcall.os.path.exists", return_value=True) + @patch("cautiousrobot.rollcall.gather_file_paths", return_value=[]) + @patch("cautiousrobot.rollcall.print") def test_no_files_exist(self, mock_print, mock_gather, mock_exists): """If no files exist, should mark all as not in directory and print message.""" - updated_df, filtered_df = check_existing_images( + updated_df, filtered_df = self.rollcall.check_existing_images( self.csv_path, self.img_dir, self.sample_df, self.filename_col ) @@ -66,8 +67,8 @@ def test_no_files_exist(self, mock_print, mock_gather, mock_exists): mock_print.assert_called_once() self.assertIn("There are 0 of the desired files", mock_print.call_args[0][0]) - @patch("cautiousrobot.utils.os.path.exists", return_value=True) - @patch("cautiousrobot.utils.gather_file_paths", return_value=["test_images/species1/a.jpg", "test_images/shouldnotcount/b.jpg"]) + @patch("cautiousrobot.rollcall.os.path.exists", return_value=True) + @patch("cautiousrobot.rollcall.gather_file_paths", return_value=["test_images/species1/a.jpg", "test_images/shouldnotcount/b.jpg"]) def test_subfolders_handling(self, mock_gather, mock_exists): """When `subfolders` is provided, expected paths should be constructed and matched.""" sub_df = pd.DataFrame({ @@ -75,7 +76,7 @@ def test_subfolders_handling(self, mock_gather, mock_exists): self.filename_col: ["a.jpg", "b.jpg"] }) - updated_df, filtered_df = check_existing_images( + updated_df, filtered_df = self.rollcall.check_existing_images( self.csv_path, self.img_dir, sub_df, self.filename_col, subfolders="subfolder" ) diff --git a/tests/test_validation.py b/tests/test_rollcall.py similarity index 81% rename from tests/test_validation.py rename to tests/test_rollcall.py index d776db9..8558387 100644 --- a/tests/test_validation.py +++ b/tests/test_rollcall.py @@ -2,68 +2,69 @@ from unittest.mock import patch, MagicMock import pandas as pd -from cautiousrobot.rollcall import ( - validate_csv_extension, - validate_filename_uniqueness, - handle_missing_filenames, - setup_expected_columns, -) +from cautiousrobot.rollcall import RollCall class TestValidateCsvExtension(unittest.TestCase): """Test CSV file extension validation.""" + def setUp(self): + self.rollcall = RollCall() + def test_valid_csv_extension(self): """Should not raise exception for valid .csv file.""" # Should not raise any exception try: - validate_csv_extension("data.csv") + self.rollcall.validate_csv_extension("data.csv") except SystemExit: self.fail("validate_csv_extension raised SystemExit unexpectedly") def test_csv_extension_case_sensitive(self): """Should accept .csv regardless of path content.""" try: - validate_csv_extension("MyData.csv") - validate_csv_extension("path/to/data.csv") + self.rollcall.validate_csv_extension("MyData.csv") + self.rollcall.validate_csv_extension("path/to/data.csv") except SystemExit: self.fail("validate_csv_extension raised SystemExit unexpectedly") def test_invalid_txt_extension(self): """Should exit if file has .txt extension.""" with self.assertRaises(SystemExit) as cm: - validate_csv_extension("data.txt") + self.rollcall.validate_csv_extension("data.txt") self.assertIn("csv", cm.exception.code.lower()) def test_invalid_xlsx_extension(self): """Should exit if file has .xlsx extension.""" with self.assertRaises(SystemExit) as cm: - validate_csv_extension("data.xlsx") + self.rollcall.validate_csv_extension("data.xlsx") self.assertIn("csv", cm.exception.code.lower()) def test_invalid_no_extension(self): """Should exit if file has no extension.""" with self.assertRaises(SystemExit) as cm: - validate_csv_extension("data") + self.rollcall.validate_csv_extension("data") self.assertIn("csv", cm.exception.code.lower()) def test_invalid_csv_as_substring(self): """Should require .csv as actual extension, not substring.""" with self.assertRaises(SystemExit) as cm: - validate_csv_extension("data.csv.backup") + self.rollcall.validate_csv_extension("data.csv.backup") self.assertIn("csv", cm.exception.code.lower()) class TestValidateFilenameUniqueness(unittest.TestCase): """Test filename uniqueness validation.""" + def setUp(self): + self.rollcall = RollCall() + def test_unique_filenames(self): """Should not raise exception for unique filenames.""" df = pd.DataFrame({ "filename": ["image1.jpg", "image2.jpg", "image3.jpg"] }) try: - validate_filename_uniqueness(df, "filename") + self.rollcall.validate_filename_uniqueness(df, "filename") except SystemExit: self.fail("validate_filename_uniqueness raised SystemExit unexpectedly") @@ -73,7 +74,7 @@ def test_duplicate_filenames(self): "filename": ["image1.jpg", "image1.jpg", "image2.jpg"] }) with self.assertRaises(SystemExit) as cm: - validate_filename_uniqueness(df, "filename") + self.rollcall.validate_filename_uniqueness(df, "filename") self.assertIn("unique identifier", cm.exception.code.lower()) def test_filenames_with_missing_values(self): @@ -82,7 +83,7 @@ def test_filenames_with_missing_values(self): "filename": ["image1.jpg", None, "image2.jpg", None] }) try: - validate_filename_uniqueness(df, "filename") + self.rollcall.validate_filename_uniqueness(df, "filename") except SystemExit: self.fail("validate_filename_uniqueness raised SystemExit unexpectedly") @@ -92,7 +93,7 @@ def test_all_missing_filenames(self): "filename": [None, None, None] }) try: - validate_filename_uniqueness(df, "filename") + self.rollcall.validate_filename_uniqueness(df, "filename") except SystemExit: self.fail("validate_filename_uniqueness raised SystemExit unexpectedly") @@ -102,7 +103,7 @@ def test_single_filename(self): "filename": ["image1.jpg"] }) try: - validate_filename_uniqueness(df, "filename") + self.rollcall.validate_filename_uniqueness(df, "filename") except SystemExit: self.fail("validate_filename_uniqueness raised SystemExit unexpectedly") @@ -112,7 +113,7 @@ def test_empty_dataframe(self): "filename": [] }) try: - validate_filename_uniqueness(df, "filename") + self.rollcall.validate_filename_uniqueness(df, "filename") except SystemExit: self.fail("validate_filename_uniqueness raised SystemExit unexpectedly") @@ -122,7 +123,7 @@ def test_custom_column_name(self): "image_name": ["photo1.jpg", "photo2.jpg", "photo3.jpg"] }) try: - validate_filename_uniqueness(df, "image_name") + self.rollcall.validate_filename_uniqueness(df, "image_name") except SystemExit: self.fail("validate_filename_uniqueness raised SystemExit unexpectedly") @@ -132,7 +133,7 @@ def test_duplicate_with_custom_column(self): "image_id": ["img_001", "img_001", "img_002"] }) with self.assertRaises(SystemExit) as cm: - validate_filename_uniqueness(df, "image_id") + self.rollcall.validate_filename_uniqueness(df, "image_id") self.assertIn("unique identifier", cm.exception.code.lower()) @@ -143,6 +144,7 @@ def setUp(self): """Set up test fixtures.""" self.filename_col = "filename" self.url_col = "file_url" + self.rollcall = RollCall() def test_no_missing_filenames(self): """Should not prompt if all filenames are present.""" @@ -151,7 +153,7 @@ def test_no_missing_filenames(self): self.url_col: ["http://url1.com", "http://url2.com"] }) with patch("builtins.input") as mock_input: - handle_missing_filenames(df, self.filename_col, self.url_col) + self.rollcall.handle_missing_filenames(df, self.filename_col, self.url_col) mock_input.assert_not_called() def test_missing_filenames_with_urls_user_accepts(self): @@ -162,7 +164,7 @@ def test_missing_filenames_with_urls_user_accepts(self): }) with patch("builtins.input", return_value="y"): try: - handle_missing_filenames(df, self.filename_col, self.url_col) + self.rollcall.handle_missing_filenames(df, self.filename_col, self.url_col) except SystemExit: self.fail("Should not exit when user answers 'y'") @@ -174,7 +176,7 @@ def test_missing_filenames_with_urls_user_declines(self): }) with patch("builtins.input", return_value="n"): with self.assertRaises(SystemExit) as cm: - handle_missing_filenames(df, self.filename_col, self.url_col) + self.rollcall.handle_missing_filenames(df, self.filename_col, self.url_col) self.assertIn("Exited", cm.exception.code) def test_missing_filenames_user_input_case_insensitive(self): @@ -185,7 +187,7 @@ def test_missing_filenames_user_input_case_insensitive(self): }) with patch("builtins.input", return_value="Y"): try: - handle_missing_filenames(df, self.filename_col, self.url_col) + self.rollcall.handle_missing_filenames(df, self.filename_col, self.url_col) except SystemExit: self.fail("Should accept 'Y' as valid yes response") @@ -197,7 +199,7 @@ def test_missing_filenames_prompt_shows_count(self): }) with patch("builtins.input", return_value="n") as mock_input: with self.assertRaises(SystemExit): - handle_missing_filenames(df, self.filename_col, self.url_col) + self.rollcall.handle_missing_filenames(df, self.filename_col, self.url_col) # Check that the input prompt includes the count call_args = mock_input.call_args[0][0] self.assertIn("2", call_args) @@ -209,7 +211,7 @@ def test_missing_filenames_no_urls(self): self.url_col: [None, None, None] }) with patch("builtins.input") as mock_input: - handle_missing_filenames(df, self.filename_col, self.url_col) + self.rollcall.handle_missing_filenames(df, self.filename_col, self.url_col) mock_input.assert_not_called() def test_missing_filenames_mixed_urls_and_filenames(self): @@ -220,7 +222,7 @@ def test_missing_filenames_mixed_urls_and_filenames(self): }) with patch("builtins.input", return_value="n") as mock_input: with self.assertRaises(SystemExit): - handle_missing_filenames(df, self.filename_col, self.url_col) + self.rollcall.handle_missing_filenames(df, self.filename_col, self.url_col) # Should count 2 URLs without filenames (rows 0 and 2) call_args = mock_input.call_args[0][0] self.assertIn("2", call_args) @@ -232,13 +234,16 @@ def test_missing_filenames_empty_dataframe(self): self.url_col: [] }) with patch("builtins.input") as mock_input: - handle_missing_filenames(df, self.filename_col, self.url_col) + self.rollcall.handle_missing_filenames(df, self.filename_col, self.url_col) mock_input.assert_not_called() class TestSetupExpectedColumns(unittest.TestCase): """Test expected columns setup.""" + def setUp(self): + self.rollcall = RollCall() + def test_basic_columns_setup(self): """Should return expected columns with basic arguments.""" args = MagicMock() @@ -246,7 +251,7 @@ def test_basic_columns_setup(self): args.url_col = "file_url" args.subdir_col = None - expected_cols, subfolders = setup_expected_columns(args) + expected_cols, subfolders = self.rollcall.setup_expected_columns(args) self.assertEqual(expected_cols["filename_col"], "filename") self.assertEqual(expected_cols["url_col"], "file_url") @@ -260,7 +265,7 @@ def test_columns_lowercase_conversion(self): args.url_col = "FileURL" args.subdir_col = None - expected_cols, subfolders = setup_expected_columns(args) + expected_cols, subfolders = self.rollcall.setup_expected_columns(args) self.assertEqual(expected_cols["filename_col"], "filename") self.assertEqual(expected_cols["url_col"], "fileurl") @@ -272,7 +277,7 @@ def test_subdir_column_setup(self): args.url_col = "file_url" args.subdir_col = "species" - expected_cols, subfolders = setup_expected_columns(args) + expected_cols, subfolders = self.rollcall.setup_expected_columns(args) self.assertEqual(expected_cols["subfolders"], "species") self.assertEqual(subfolders, "species") @@ -284,7 +289,7 @@ def test_subdir_column_lowercase(self): args.url_col = "file_url" args.subdir_col = "Species" - expected_cols, subfolders = setup_expected_columns(args) + expected_cols, subfolders = self.rollcall.setup_expected_columns(args) self.assertEqual(expected_cols["subfolders"], "species") self.assertEqual(subfolders, "species") @@ -296,7 +301,7 @@ def test_custom_column_names_with_subdir(self): args.url_col = "DownloadURL" args.subdir_col = "Category" - expected_cols, subfolders = setup_expected_columns(args) + expected_cols, subfolders = self.rollcall.setup_expected_columns(args) self.assertEqual(expected_cols["filename_col"], "imageid") self.assertEqual(expected_cols["url_col"], "downloadurl") @@ -310,7 +315,7 @@ def test_returns_subfolders_none_when_not_provided(self): args.url_col = "file_url" args.subdir_col = None - expected_cols, subfolders = setup_expected_columns(args) + expected_cols, subfolders = self.rollcall.setup_expected_columns(args) self.assertIsNone(subfolders) self.assertNotIn("subfolders", expected_cols) @@ -322,7 +327,7 @@ def test_returns_subfolders_value_when_provided(self): args.url_col = "file_url" args.subdir_col = "region" - expected_cols, subfolders = setup_expected_columns(args) + expected_cols, subfolders = self.rollcall.setup_expected_columns(args) self.assertEqual(subfolders, "region") self.assertIn("subfolders", expected_cols) @@ -334,7 +339,7 @@ def test_dictionary_structure(self): args.url_col = "url" args.subdir_col = "folder" - expected_cols, subfolders = setup_expected_columns(args) + expected_cols, subfolders = self.rollcall.setup_expected_columns(args) # Check dictionary has correct keys self.assertIn("filename_col", expected_cols) From 6e91c0d77a88b5e6111b2ce7ca50f4db9614265f Mon Sep 17 00:00:00 2001 From: Ishika Aahna <123102828+codedbyishika@users.noreply.github.com> Date: Fri, 12 Jun 2026 16:37:33 -0400 Subject: [PATCH 12/24] Name change --- AGENTS.md | 2 +- src/cautiousrobot/__main__.py | 2 +- src/cautiousrobot/{rollcall.py => roll_call.py} | 0 tests/test_existing_images.py | 2 +- tests/{test_rollcall.py => test_roll_call.py} | 2 +- 5 files changed, 4 insertions(+), 4 deletions(-) rename src/cautiousrobot/{rollcall.py => roll_call.py} (100%) rename tests/{test_rollcall.py => test_roll_call.py} (99%) diff --git a/AGENTS.md b/AGENTS.md index 4dd7e2f..d9144b3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -17,7 +17,7 @@ ### Core Modules - `__main__.py` - Main entry point with CLI argument parsing and workflow orchestration -- `rollcall.py` - CSV validation and input handling functions +- `roll_call.py` - CSV validation and input handling functions - `download.py` - Core image downloading functionality with retry logic - `buddy_check.py` - Checksum validation and download verification using BuddyCheck class - `utils.py` - Helper functions for CSV processing, logging, and image downsampling diff --git a/src/cautiousrobot/__main__.py b/src/cautiousrobot/__main__.py index 92715bb..fba820d 100644 --- a/src/cautiousrobot/__main__.py +++ b/src/cautiousrobot/__main__.py @@ -13,7 +13,7 @@ from cautiousrobot.utils import process_csv from cautiousrobot.buddy_check import BuddyCheck from cautiousrobot.download import download_images -from cautiousrobot.rollcall import RollCall +from cautiousrobot.roll_call import RollCall from cautiousrobot.__about__ import __version__ def parse_args(): diff --git a/src/cautiousrobot/rollcall.py b/src/cautiousrobot/roll_call.py similarity index 100% rename from src/cautiousrobot/rollcall.py rename to src/cautiousrobot/roll_call.py diff --git a/tests/test_existing_images.py b/tests/test_existing_images.py index 38c4a6d..e512252 100644 --- a/tests/test_existing_images.py +++ b/tests/test_existing_images.py @@ -1,7 +1,7 @@ import unittest import pandas as pd from unittest.mock import patch -from cautiousrobot.rollcall import RollCall +from cautiousrobot.roll_call import RollCall class TestCheckExistingImages(unittest.TestCase): diff --git a/tests/test_rollcall.py b/tests/test_roll_call.py similarity index 99% rename from tests/test_rollcall.py rename to tests/test_roll_call.py index 8558387..e1be8b2 100644 --- a/tests/test_rollcall.py +++ b/tests/test_roll_call.py @@ -2,7 +2,7 @@ from unittest.mock import patch, MagicMock import pandas as pd -from cautiousrobot.rollcall import RollCall +from cautiousrobot.roll_call import RollCall class TestValidateCsvExtension(unittest.TestCase): From 8fe0e6de6658007e232f0cfacdea1fb70c166803 Mon Sep 17 00:00:00 2001 From: Ishika Aahna <123102828+codedbyishika@users.noreply.github.com> Date: Fri, 12 Jun 2026 16:40:51 -0400 Subject: [PATCH 13/24] Update tests --- tests/test_existing_images.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/test_existing_images.py b/tests/test_existing_images.py index e512252..579e40f 100644 --- a/tests/test_existing_images.py +++ b/tests/test_existing_images.py @@ -14,7 +14,7 @@ def setUp(self): }) self.rollcall = RollCall() - @patch("cautiousrobot.rollcall.os.path.exists", return_value=False) + @patch("cautiousrobot.roll_call.os.path.exists", return_value=False) def test_directory_does_not_exist(self, mock_exists): """If image directory doesn't exist, all images marked as not in directory.""" updated_df, filtered_df = self.rollcall.check_existing_images( @@ -25,9 +25,9 @@ def test_directory_does_not_exist(self, mock_exists): self.assertEqual(len(filtered_df), len(self.sample_df)) mock_exists.assert_called_once_with(self.img_dir) - @patch("cautiousrobot.rollcall.os.path.exists", return_value=True) - @patch("cautiousrobot.rollcall.gather_file_paths", return_value=["test_images/a.jpg"]) - @patch("cautiousrobot.rollcall.print") + @patch("cautiousrobot.roll_call.os.path.exists", return_value=True) + @patch("cautiousrobot.roll_call.gather_file_paths", return_value=["test_images/a.jpg"]) + @patch("cautiousrobot.roll_call.print") def test_some_files_exist(self, mock_print, mock_gather, mock_exists): """Should mark existing files correctly and print status.""" updated_df, filtered_df = self.rollcall.check_existing_images( @@ -41,8 +41,8 @@ def test_some_files_exist(self, mock_print, mock_gather, mock_exists): mock_print.assert_called_once() self.assertIn("There are 1 of the desired files", mock_print.call_args[0][0]) - @patch("cautiousrobot.rollcall.os.path.exists", return_value=True) - @patch("cautiousrobot.rollcall.gather_file_paths", return_value=["test_images/a.jpg", "test_images/b.jpg", "test_images/c.jpg"]) + @patch("cautiousrobot.roll_call.os.path.exists", return_value=True) + @patch("cautiousrobot.roll_call.gather_file_paths", return_value=["test_images/a.jpg", "test_images/b.jpg", "test_images/c.jpg"]) def test_all_files_exist_exits(self, mock_gather, mock_exists): """If all images exist, should exit early with proper message.""" with self.assertRaises(SystemExit) as cm: @@ -53,9 +53,9 @@ def test_all_files_exist_exits(self, mock_gather, mock_exists): self.assertIn("already contains all images", cm.exception.code) mock_exists.assert_called_once_with(self.img_dir) - @patch("cautiousrobot.rollcall.os.path.exists", return_value=True) - @patch("cautiousrobot.rollcall.gather_file_paths", return_value=[]) - @patch("cautiousrobot.rollcall.print") + @patch("cautiousrobot.roll_call.os.path.exists", return_value=True) + @patch("cautiousrobot.roll_call.gather_file_paths", return_value=[]) + @patch("cautiousrobot.roll_call.print") def test_no_files_exist(self, mock_print, mock_gather, mock_exists): """If no files exist, should mark all as not in directory and print message.""" updated_df, filtered_df = self.rollcall.check_existing_images( @@ -67,8 +67,8 @@ def test_no_files_exist(self, mock_print, mock_gather, mock_exists): mock_print.assert_called_once() self.assertIn("There are 0 of the desired files", mock_print.call_args[0][0]) - @patch("cautiousrobot.rollcall.os.path.exists", return_value=True) - @patch("cautiousrobot.rollcall.gather_file_paths", return_value=["test_images/species1/a.jpg", "test_images/shouldnotcount/b.jpg"]) + @patch("cautiousrobot.roll_call.os.path.exists", return_value=True) + @patch("cautiousrobot.roll_call.gather_file_paths", return_value=["test_images/species1/a.jpg", "test_images/shouldnotcount/b.jpg"]) def test_subfolders_handling(self, mock_gather, mock_exists): """When `subfolders` is provided, expected paths should be constructed and matched.""" sub_df = pd.DataFrame({ From 15cd820c0ee7c38573b9d9387a44dd38c62b8f73 Mon Sep 17 00:00:00 2001 From: Ishika Aahna <123102828+codedbyishika@users.noreply.github.com> Date: Wed, 17 Jun 2026 16:31:59 -0400 Subject: [PATCH 14/24] Update interactive to missing filenames error handling --- src/cautiousrobot/__main__.py | 4 +- src/cautiousrobot/roll_call.py | 31 ++++++-- tests/test_download_images.py | 19 +++-- tests/test_roll_call.py | 136 +++++++++++++++------------------ 4 files changed, 100 insertions(+), 90 deletions(-) diff --git a/src/cautiousrobot/__main__.py b/src/cautiousrobot/__main__.py index fba820d..04216ba 100644 --- a/src/cautiousrobot/__main__.py +++ b/src/cautiousrobot/__main__.py @@ -107,7 +107,7 @@ def main(): args = parse_args() csv_path = args.input_file - roll_call = RollCall() + roll_call = RollCall(csv_path) # Validate CSV extension roll_call.validate_csv_extension(csv_path) @@ -123,7 +123,7 @@ def main(): filename_col = expected_cols["filename_col"] url_col = expected_cols["url_col"] roll_call.validate_filename_uniqueness(data_df, filename_col) - roll_call.handle_missing_filenames(data_df, filename_col, url_col) + missing_filenames = roll_call.handle_missing_filenames(data_df, filename_col, url_col) # Set source DataFrame for only non-null filename values source_df = data_df.loc[data_df[filename_col].notna()].copy() diff --git a/src/cautiousrobot/roll_call.py b/src/cautiousrobot/roll_call.py index 0ca9aeb..e0b5743 100644 --- a/src/cautiousrobot/roll_call.py +++ b/src/cautiousrobot/roll_call.py @@ -9,9 +9,12 @@ class RollCall: """Pre-download validation and directory verification.""" + def __init__(self, csv_path=None): + self.csv_path = csv_path + def validate_csv_extension(self, csv_path): """Validate that the input file has a .csv extension.""" - if not csv_path.endswith(".csv"): + if not csv_path.lower().endswith(".csv"): sys.exit("Expected CSV for input file; extension should be '.csv'") def validate_filename_uniqueness(self, data_df, filename_col): @@ -21,12 +24,26 @@ def validate_filename_uniqueness(self, data_df, filename_col): def handle_missing_filenames(self, data_df, filename_col, url_col): """Handle cases where URLs exist but filenames are missing.""" - urls_no_name = len(data_df.loc[(data_df[filename_col].isna() & (data_df[url_col].notna()))]) - if urls_no_name > 0: - ignore = input(f"'{filename_col}' is missing values for {urls_no_name} URLs. Proceed with download ignoring these URLs? [y/n]: ") - if ignore.lower() != "y": - sys.exit("Exited without executing.") - + # Rows where filename is missing but URL exists + missing = data_df.loc[data_df[filename_col].isna() & data_df[url_col].notna()] + if missing.empty: + return None + # Save path follows BuddyCheck pattern: _missing_filenames.csv + csv_base = os.path.splitext(self.csv_path)[0] + save_path = f"{csv_base}_missing_filenames.csv" + if len(missing) <= 5: + print("\n❗ Missing filenames detected (showing all):") + print(missing) + else: + missing.to_csv(save_path, index=False) + print( + f"\n❗ Missing filenames detected for {len(missing)} rows.\n" + f"Because there are more than 5, they were saved to:\n {save_path}\n" + f"Please correct the CSV and re-run." + ) + # Return missing rows for reference + return missing + def setup_expected_columns(self, args): """Set up the expected columns dictionary for CSV processing.""" subfolders = args.subdir_col diff --git a/tests/test_download_images.py b/tests/test_download_images.py index c0c4af1..f9cbbdb 100644 --- a/tests/test_download_images.py +++ b/tests/test_download_images.py @@ -401,26 +401,33 @@ def test_main_non_unique_filenames(self, mock_process_csv, mock_parse_args): @patch('cautiousrobot.__main__.parse_args') @patch('cautiousrobot.__main__.process_csv') - @patch('builtins.input', return_value='n') - def test_main_missing_filenames(self, mock_input, mock_process_csv, mock_parse_args): + def test_main_missing_filenames_noninteractive(self, mock_process_csv, mock_parse_args): mock_args = MagicMock() mock_args.input_file = 'test.csv' mock_args.img_name_col = 'filename_col' mock_args.url_col = 'url_col' mock_args.subdir_col = None + mock_args.output_dir = 'output_dir' + mock_args.side_length = None + mock_args.wait_time = 0 + mock_args.max_retries = 3 + mock_args.checksum_algorithm = 'md5' + mock_args.verifier_col = None + mock_parse_args.return_value = mock_args mock_data = pd.DataFrame({ 'filename_col': [None, None, 'file2', 'file3'], 'url_col': ['url1', 'url2', 'url3', 'url4'] }) - + mock_process_csv.return_value = mock_data - with self.assertRaises(SystemExit) as cm: + # Should NOT exit now + try: main() - - self.assertEqual(cm.exception.code, "Exited without executing.") + except SystemExit as e: + self.fail(f"main() raised SystemExit unexpectedly: {e}") if __name__ == '__main__': unittest.main() diff --git a/tests/test_roll_call.py b/tests/test_roll_call.py index e1be8b2..d3c1280 100644 --- a/tests/test_roll_call.py +++ b/tests/test_roll_call.py @@ -138,104 +138,90 @@ def test_duplicate_with_custom_column(self): class TestHandleMissingFilenames(unittest.TestCase): - """Test handling of missing filename values.""" + """Test handling of missing filename values under new non-interactive behavior.""" def setUp(self): - """Set up test fixtures.""" self.filename_col = "filename" self.url_col = "file_url" - self.rollcall = RollCall() + # RollCall now requires csv_path for saving missing CSVs + self.rollcall = RollCall(csv_path="testdata.csv") - def test_no_missing_filenames(self): - """Should not prompt if all filenames are present.""" + @patch("builtins.print") + def test_no_missing_filenames(self, mock_print): + """Should not print anything when no filenames are missing.""" df = pd.DataFrame({ self.filename_col: ["image1.jpg", "image2.jpg"], self.url_col: ["http://url1.com", "http://url2.com"] }) - with patch("builtins.input") as mock_input: - self.rollcall.handle_missing_filenames(df, self.filename_col, self.url_col) - mock_input.assert_not_called() + result = self.rollcall.handle_missing_filenames(df, self.filename_col, self.url_col) + self.assertIsNone(result) + mock_print.assert_not_called() - def test_missing_filenames_with_urls_user_accepts(self): - """Should continue when user answers 'y' to missing filenames prompt.""" - df = pd.DataFrame({ - self.filename_col: ["image1.jpg", None, "image3.jpg"], - self.url_col: ["http://url1.com", "http://url2.com", "http://url3.com"] - }) - with patch("builtins.input", return_value="y"): - try: - self.rollcall.handle_missing_filenames(df, self.filename_col, self.url_col) - except SystemExit: - self.fail("Should not exit when user answers 'y'") - - def test_missing_filenames_with_urls_user_declines(self): - """Should exit when user answers 'n' to missing filenames prompt.""" + @patch("builtins.print") + def test_missing_filenames_prints_when_five_or_fewer(self, mock_print): + """Should print missing rows when count <= 5.""" df = pd.DataFrame({ - self.filename_col: ["image1.jpg", None, "image3.jpg"], - self.url_col: ["http://url1.com", "http://url2.com", "http://url3.com"] + self.filename_col: [None, None, "img3.jpg"], + self.url_col: ["url1", "url2", "url3"] }) - with patch("builtins.input", return_value="n"): - with self.assertRaises(SystemExit) as cm: - self.rollcall.handle_missing_filenames(df, self.filename_col, self.url_col) - self.assertIn("Exited", cm.exception.code) + result = self.rollcall.handle_missing_filenames(df, self.filename_col, self.url_col) - def test_missing_filenames_user_input_case_insensitive(self): - """Should accept 'Y' or other case variations for yes response.""" - df = pd.DataFrame({ - self.filename_col: ["image1.jpg", None], - self.url_col: ["http://url1.com", "http://url2.com"] - }) - with patch("builtins.input", return_value="Y"): - try: - self.rollcall.handle_missing_filenames(df, self.filename_col, self.url_col) - except SystemExit: - self.fail("Should accept 'Y' as valid yes response") - - def test_missing_filenames_prompt_shows_count(self): - """Should show number of missing filenames in prompt.""" + self.assertEqual(len(result), 2) + mock_print.assert_called() # printed the missing rows + + @patch("pandas.DataFrame.to_csv") + @patch("builtins.print") + def test_missing_filenames_saved_when_more_than_five(self, mock_print, mock_to_csv): + """Should save missing rows to CSV when count > 5.""" df = pd.DataFrame({ - self.filename_col: ["img1.jpg", None, None, "img4.jpg"], - self.url_col: ["url1", "url2", "url3", "url4"] + self.filename_col: [None] * 6, + self.url_col: ["url"] * 6 }) - with patch("builtins.input", return_value="n") as mock_input: - with self.assertRaises(SystemExit): - self.rollcall.handle_missing_filenames(df, self.filename_col, self.url_col) - # Check that the input prompt includes the count - call_args = mock_input.call_args[0][0] - self.assertIn("2", call_args) - - def test_missing_filenames_no_urls(self): - """Should not prompt if no URLs exist (no missing filenames to handle).""" + + result = self.rollcall.handle_missing_filenames(df, self.filename_col, self.url_col) + + self.assertEqual(len(result), 6) + mock_to_csv.assert_called_once() + args, kwargs = mock_to_csv.call_args + self.assertIn("testdata_missing_filenames.csv", args[0]) + + @patch("builtins.print") + def test_missing_filenames_no_urls(self, mock_print): + """Should not print anything when URLs are missing (no actionable missing filenames).""" df = pd.DataFrame({ - self.filename_col: ["image1.jpg", None, "image3.jpg"], + self.filename_col: ["img1.jpg", None, "img3.jpg"], self.url_col: [None, None, None] }) - with patch("builtins.input") as mock_input: - self.rollcall.handle_missing_filenames(df, self.filename_col, self.url_col) - mock_input.assert_not_called() + result = self.rollcall.handle_missing_filenames(df, self.filename_col, self.url_col) - def test_missing_filenames_mixed_urls_and_filenames(self): - """Should only count rows with URLs but missing filenames.""" - df = pd.DataFrame({ - self.filename_col: [None, "image2.jpg", None, "image4.jpg"], - self.url_col: ["http://url1.com", None, "http://url3.com", "http://url4.com"] - }) - with patch("builtins.input", return_value="n") as mock_input: - with self.assertRaises(SystemExit): - self.rollcall.handle_missing_filenames(df, self.filename_col, self.url_col) - # Should count 2 URLs without filenames (rows 0 and 2) - call_args = mock_input.call_args[0][0] - self.assertIn("2", call_args) - - def test_missing_filenames_empty_dataframe(self): - """Should not prompt if dataframe is empty.""" + self.assertIsNone(result) + mock_print.assert_not_called() + + @patch("builtins.print") + def test_missing_filenames_empty_dataframe(self, mock_print): + """Should not print anything for empty dataframe.""" df = pd.DataFrame({ self.filename_col: [], self.url_col: [] }) - with patch("builtins.input") as mock_input: - self.rollcall.handle_missing_filenames(df, self.filename_col, self.url_col) - mock_input.assert_not_called() + result = self.rollcall.handle_missing_filenames(df, self.filename_col, self.url_col) + + self.assertIsNone(result) + mock_print.assert_not_called() + + @patch("builtins.print") + def test_missing_filenames_returns_correct_subset(self, mock_print): + """Should return only rows with missing filenames and valid URLs.""" + df = pd.DataFrame({ + self.filename_col: [None, "img2.jpg", None, "img4.jpg"], + self.url_col: ["url1", None, "url3", "url4"] + }) + + result = self.rollcall.handle_missing_filenames(df, self.filename_col, self.url_col) + + # Only rows 0 and 2 qualify + self.assertEqual(len(result), 2) + self.assertListEqual(result.index.tolist(), [0, 2]) class TestSetupExpectedColumns(unittest.TestCase): From af134f8794a33e05da4437f428771203c4fb9c93 Mon Sep 17 00:00:00 2001 From: Ishika Aahna <123102828+codedbyishika@users.noreply.github.com> Date: Wed, 17 Jun 2026 16:34:35 -0400 Subject: [PATCH 15/24] Small error fix --- src/cautiousrobot/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cautiousrobot/__main__.py b/src/cautiousrobot/__main__.py index 04216ba..9c11ff8 100644 --- a/src/cautiousrobot/__main__.py +++ b/src/cautiousrobot/__main__.py @@ -123,7 +123,7 @@ def main(): filename_col = expected_cols["filename_col"] url_col = expected_cols["url_col"] roll_call.validate_filename_uniqueness(data_df, filename_col) - missing_filenames = roll_call.handle_missing_filenames(data_df, filename_col, url_col) + roll_call.handle_missing_filenames(data_df, filename_col, url_col) # Set source DataFrame for only non-null filename values source_df = data_df.loc[data_df[filename_col].notna()].copy() From 1271c263fc46bb17710f947d501e3bf48e23b6b0 Mon Sep 17 00:00:00 2001 From: Ishika Aahna <123102828+codedbyishika@users.noreply.github.com> Date: Fri, 19 Jun 2026 13:22:53 -0400 Subject: [PATCH 16/24] Update for csv to initialize when dataframe is missing --- src/cautiousrobot/roll_call.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cautiousrobot/roll_call.py b/src/cautiousrobot/roll_call.py index e0b5743..abe51ef 100644 --- a/src/cautiousrobot/roll_call.py +++ b/src/cautiousrobot/roll_call.py @@ -28,13 +28,13 @@ def handle_missing_filenames(self, data_df, filename_col, url_col): missing = data_df.loc[data_df[filename_col].isna() & data_df[url_col].notna()] if missing.empty: return None - # Save path follows BuddyCheck pattern: _missing_filenames.csv - csv_base = os.path.splitext(self.csv_path)[0] - save_path = f"{csv_base}_missing_filenames.csv" if len(missing) <= 5: print("\n❗ Missing filenames detected (showing all):") print(missing) else: + # Save path follows BuddyCheck pattern: _missing_filenames.csv + csv_base = os.path.splitext(self.csv_path)[0] + save_path = f"{csv_base}_missing_filenames.csv" missing.to_csv(save_path, index=False) print( f"\n❗ Missing filenames detected for {len(missing)} rows.\n" From c905318fbb56017035ed92e11533841c2e995656 Mon Sep 17 00:00:00 2001 From: Ishika Aahna <123102828+codedbyishika@users.noreply.github.com> Date: Fri, 19 Jun 2026 14:43:11 -0400 Subject: [PATCH 17/24] duplicate checksums --- src/cautiousrobot/__main__.py | 8 +++ src/cautiousrobot/roll_call.py | 43 +++++++++++++++ tests/test_roll_call.py | 95 ++++++++++++++++++++++++++++++++++ 3 files changed, 146 insertions(+) diff --git a/src/cautiousrobot/__main__.py b/src/cautiousrobot/__main__.py index 9c11ff8..b93992a 100644 --- a/src/cautiousrobot/__main__.py +++ b/src/cautiousrobot/__main__.py @@ -128,6 +128,14 @@ def main(): # Set source DataFrame for only non-null filename values source_df = data_df.loc[data_df[filename_col].notna()].copy() + # If a verifier column is provided, check for duplicate checksums before downloading + if args.verifier_col: + roll_call.check_duplicate_checksums( + data_df=source_df, + hash_col=args.verifier_col.lower(), + ignore_duplicates=getattr(args, "ignore_duplicates", False) + ) + # Validate and handle existing output directory img_dir = args.output_dir source_df, filtered_df = roll_call.check_existing_images(csv_path, img_dir, source_df, filename_col, subfolders) diff --git a/src/cautiousrobot/roll_call.py b/src/cautiousrobot/roll_call.py index abe51ef..22f9dfc 100644 --- a/src/cautiousrobot/roll_call.py +++ b/src/cautiousrobot/roll_call.py @@ -91,4 +91,47 @@ def check_existing_images(self, csv_path, img_dir, source_df, filename_col, subf return df, filtered_df + def _preview_or_save(self, label, df): + """Print up to 5 rows or save to CSV if larger.""" + if len(df) <= 5: + print(f"\n❗ {label.replace('_', ' ').title()} detected (showing all):") + print(df) + else: + csv_base = os.path.splitext(self.csv_path)[0] + save_path = f"{csv_base}_{label}.csv" + df.to_csv(save_path, index=False) + print( + f"\n❗ {len(df)} {label.replace('_', ' ')} detected.\n" + f"Full list saved to:\n {save_path}\n" + ) + + def check_duplicate_checksums(self, data_df, hash_col, ignore_duplicates=False): + """Detect duplicate checksum values and optionally block execution.""" + + # Only consider non-null hashes + dupes = ( + data_df[data_df[hash_col].notna()] + .groupby(hash_col) + .filter(lambda x: len(x) > 1) + ) + + if dupes.empty: + print("✔ No duplicate checksums detected.") + return + + # If duplicates exist, preview or save + self._preview_or_save("duplicate_checksums", dupes) + + if ignore_duplicates: + print( + f"\n⚠ Duplicate checksums detected ({len(dupes)} rows), " + f"but --ignore-duplicates was passed. Continuing.\n" + ) + return + + # Default behavior: block execution + sys.exit( + "❗ Duplicate checksums detected. " + "Use --ignore-duplicates to allow downloading duplicates." + ) diff --git a/tests/test_roll_call.py b/tests/test_roll_call.py index d3c1280..5e83058 100644 --- a/tests/test_roll_call.py +++ b/tests/test_roll_call.py @@ -337,6 +337,101 @@ def test_dictionary_structure(self): self.assertEqual(expected_cols["url_col"], "url") self.assertEqual(expected_cols["subfolders"], "folder") +class TestCheckDuplicateChecksums(unittest.TestCase): + """Tests for RollCall.check_duplicate_checksums.""" + + def setUp(self): + self.rollcall = RollCall(csv_path="testdata.csv") + + @patch("builtins.print") + def test_no_duplicates(self, mock_print): + """Should not exit when no duplicate checksums exist.""" + df = pd.DataFrame({"hash": ["a", "b", "c"]}) + + try: + self.rollcall.check_duplicate_checksums(df, "hash") + except SystemExit: + self.fail("Unexpected SystemExit for no duplicates") + + mock_print.assert_called_once() # "✔ No duplicate checksums detected." + + @patch("builtins.print") + def test_duplicates_without_ignore(self, mock_print): + """Should exit when duplicates exist and ignore flag is False.""" + df = pd.DataFrame({"hash": ["a", "a", "b"]}) + + with self.assertRaises(SystemExit): + self.rollcall.check_duplicate_checksums(df, "hash", ignore_duplicates=False) + + # Should have printed preview/save message + mock_print.assert_called() + + @patch("builtins.print") + def test_duplicates_with_ignore(self, mock_print): + """Should not exit when duplicates exist and ignore flag is True.""" + df = pd.DataFrame({"hash": ["a", "a", "b"]}) + + try: + self.rollcall.check_duplicate_checksums(df, "hash", ignore_duplicates=True) + except SystemExit: + self.fail("Unexpected SystemExit when ignore_duplicates=True") + + # Should print warning + self.assertTrue(any("ignore-duplicates" in call.args[0] for call in mock_print.call_args_list)) + + @patch("builtins.print") + def test_preview_mode_for_small_duplicate_set(self, mock_print): + """Should print preview when <=5 duplicate rows.""" + df = pd.DataFrame({"hash": ["x", "x", "x"]}) + + with self.assertRaises(SystemExit): + self.rollcall.check_duplicate_checksums(df, "hash") + + # Should print the DataFrame preview + mock_print.assert_called() + + @patch("pandas.DataFrame.to_csv") + @patch("builtins.print") + def test_save_mode_for_large_duplicate_set(self, mock_print, mock_to_csv): + """Should save CSV when >5 duplicate rows.""" + df = pd.DataFrame({"hash": ["x"] * 10}) + + with self.assertRaises(SystemExit): + self.rollcall.check_duplicate_checksums(df, "hash") + + mock_to_csv.assert_called_once() + args, kwargs = mock_to_csv.call_args + self.assertIn("testdata_duplicate_checksums.csv", args[0]) + +class TestPreviewOrSave(unittest.TestCase): + """Tests for RollCall._preview_or_save.""" + + def setUp(self): + self.rollcall = RollCall(csv_path="testdata.csv") + + @patch("builtins.print") + @patch("pandas.DataFrame.to_csv") + def test_preview_small_dataframe(self, mock_to_csv, mock_print): + """Should print when <=5 rows.""" + df = pd.DataFrame({"x": [1, 2, 3]}) + + self.rollcall._preview_or_save("test_label", df) + + mock_print.assert_called() + mock_to_csv.assert_not_called() + + @patch("builtins.print") + @patch("pandas.DataFrame.to_csv") + def test_save_large_dataframe(self, mock_to_csv, mock_print): + """Should save CSV when >5 rows.""" + df = pd.DataFrame({"x": list(range(10))}) + + self.rollcall._preview_or_save("test_label", df) + + mock_to_csv.assert_called_once() + args, kwargs = mock_to_csv.call_args + self.assertIn("testdata_test_label.csv", args[0]) + mock_print.assert_called() if __name__ == "__main__": unittest.main() From 2f34879e8af40e2aa54753c3b3aefc91b89a3ecc Mon Sep 17 00:00:00 2001 From: Ishika Aahna <123102828+codedbyishika@users.noreply.github.com> Date: Fri, 19 Jun 2026 15:02:28 -0400 Subject: [PATCH 18/24] summary of where images & downsized images will be downloaded --- src/cautiousrobot/__main__.py | 12 +++++ src/cautiousrobot/roll_call.py | 17 +++++++ tests/test_roll_call.py | 84 ++++++++++++++++++++++++++++++++++ 3 files changed, 113 insertions(+) diff --git a/src/cautiousrobot/__main__.py b/src/cautiousrobot/__main__.py index b93992a..5b564ac 100644 --- a/src/cautiousrobot/__main__.py +++ b/src/cautiousrobot/__main__.py @@ -140,6 +140,18 @@ def main(): img_dir = args.output_dir source_df, filtered_df = roll_call.check_existing_images(csv_path, img_dir, source_df, filename_col, subfolders) + # Prepare summary info + num_images = filtered_df.shape[0] + downsample_dir = img_dir + "_downsized" if isinstance(args.side_length, int) else None + + # Print RollCall summary + roll_call.print_download_summary( + img_dir=img_dir, + downsample_dir=downsample_dir, + subfolders=subfolders, + num_images=num_images + ) + # Set up log paths log_filepath, error_log_filepath, metadata_path = setup_log_paths(csv_path) diff --git a/src/cautiousrobot/roll_call.py b/src/cautiousrobot/roll_call.py index 22f9dfc..fc9aee9 100644 --- a/src/cautiousrobot/roll_call.py +++ b/src/cautiousrobot/roll_call.py @@ -135,3 +135,20 @@ def check_duplicate_checksums(self, data_df, hash_col, ignore_duplicates=False): "Use --ignore-duplicates to allow downloading duplicates." ) + def print_download_summary(self, img_dir, downsample_dir, subfolders, num_images): + """Print a summary of where images and downsized images will be saved.""" + print("\n📦 Download Summary") + print("--------------------") + print(f"Images will be downloaded to: {img_dir}") + + if downsample_dir: + print(f"Downsampled images will be saved to: {downsample_dir}") + else: + print("Downsampled images: not requested") + + if subfolders: + print(f"Subfolders enabled: {subfolders}") + else: + print("Subfolders: none") + + print(f"Images to download: {num_images}\n") \ No newline at end of file diff --git a/tests/test_roll_call.py b/tests/test_roll_call.py index 5e83058..98bf5cf 100644 --- a/tests/test_roll_call.py +++ b/tests/test_roll_call.py @@ -433,5 +433,89 @@ def test_save_large_dataframe(self, mock_to_csv, mock_print): self.assertIn("testdata_test_label.csv", args[0]) mock_print.assert_called() +class TestPrintDownloadSummary(unittest.TestCase): + """Tests for RollCall.print_download_summary.""" + + def setUp(self): + self.rollcall = RollCall(csv_path="testdata.csv") + + @patch("builtins.print") + def test_summary_with_downsampling_and_subfolders(self, mock_print): + """Should print full summary including downsized path and subfolders.""" + self.rollcall.print_download_summary( + img_dir="/images", + downsample_dir="/images_downsized", + subfolders="species", + num_images=42 + ) + + printed = " ".join(call.args[0] for call in mock_print.call_args_list) + + self.assertIn("Images will be downloaded to: /images", printed) + self.assertIn("Downsampled images will be saved to: /images_downsized", printed) + self.assertIn("Subfolders enabled: species", printed) + self.assertIn("Images to download: 42", printed) + + @patch("builtins.print") + def test_summary_without_downsampling(self, mock_print): + """Should indicate that downsampling is not requested.""" + self.rollcall.print_download_summary( + img_dir="/images", + downsample_dir=None, + subfolders=None, + num_images=10 + ) + + printed = " ".join(call.args[0] for call in mock_print.call_args_list) + + self.assertIn("Images will be downloaded to: /images", printed) + self.assertIn("Downsampled images: not requested", printed) + self.assertIn("Subfolders: none", printed) + self.assertIn("Images to download: 10", printed) + + @patch("builtins.print") + def test_summary_with_subfolders_only(self, mock_print): + """Should print subfolder info even without downsampling.""" + self.rollcall.print_download_summary( + img_dir="/images", + downsample_dir=None, + subfolders="category", + num_images=5 + ) + + printed = " ".join(call.args[0] for call in mock_print.call_args_list) + + self.assertIn("Subfolders enabled: category", printed) + self.assertIn("Images to download: 5", printed) + + @patch("builtins.print") + def test_summary_zero_images(self, mock_print): + """Should correctly print zero image count.""" + self.rollcall.print_download_summary( + img_dir="/images", + downsample_dir=None, + subfolders=None, + num_images=0 + ) + + printed = " ".join(call.args[0] for call in mock_print.call_args_list) + self.assertIn("Images to download: 0", printed) + + @patch("builtins.print") + def test_summary_formatting_header(self, mock_print): + """Should print the summary header and separator.""" + self.rollcall.print_download_summary( + img_dir="/images", + downsample_dir=None, + subfolders=None, + num_images=1 + ) + + printed_lines = [call.args[0] for call in mock_print.call_args_list] + + self.assertIn("📦 Download Summary", printed_lines[0]) + self.assertIn("--------------------", printed_lines[1]) + + if __name__ == "__main__": unittest.main() From 6c0b5d4e79a06ecdfde7b780657cee9424338da0 Mon Sep 17 00:00:00 2001 From: Ishika Aahna <123102828+codedbyishika@users.noreply.github.com> Date: Fri, 3 Jul 2026 12:50:12 -0400 Subject: [PATCH 19/24] Include comments and docstrings --- src/cautiousrobot/roll_call.py | 151 ++++++++++++++++++++++++++++----- tests/test_roll_call.py | 2 +- 2 files changed, 131 insertions(+), 22 deletions(-) diff --git a/src/cautiousrobot/roll_call.py b/src/cautiousrobot/roll_call.py index fc9aee9..7a51127 100644 --- a/src/cautiousrobot/roll_call.py +++ b/src/cautiousrobot/roll_call.py @@ -10,105 +10,203 @@ class RollCall: """Pre-download validation and directory verification.""" def __init__(self, csv_path=None): + """ + Initialize the roll-call validator with an optional CSV path. + + Parameters: + csv_path (str): Path to the source CSV file used for reporting and output naming. + """ self.csv_path = csv_path def validate_csv_extension(self, csv_path): - """Validate that the input file has a .csv extension.""" + """ + Validate that the supplied input path points to a CSV file. + + Parameters: + csv_path (str): Path to the input CSV file. + + Raises: + SystemExit: If the input file does not end with the '.csv' extension. + """ + # The CLI expects CSV input, so the validation stops early when the file type is wrong. if not csv_path.lower().endswith(".csv"): sys.exit("Expected CSV for input file; extension should be '.csv'") def validate_filename_uniqueness(self, data_df, filename_col): - """Validate that the filename column contains unique values.""" + """ + Validate that the selected filename column contains unique values. + + Parameters: + data_df (pd.DataFrame): DataFrame loaded from the CSV file. + filename_col (str): Name of the column that should uniquely identify each image. + + Raises: + SystemExit: If duplicate filenames are found in the column. + """ + # Count only non-missing values because empty filenames should not be treated as duplicates. if data_df.loc[data_df[filename_col].notna()].shape[0] != data_df[filename_col].nunique(): sys.exit(f"{filename_col} is not a unique identifier for this dataset, please choose a column with unique values for filenames.") def handle_missing_filenames(self, data_df, filename_col, url_col): - """Handle cases where URLs exist but filenames are missing.""" - # Rows where filename is missing but URL exists + """ + Handle cases where URLs are present but filenames are missing. + + Parameters: + data_df (pd.DataFrame): DataFrame loaded from the CSV file. + filename_col (str): Name of the column containing image filenames. + url_col (str): Name of the column containing image URLs. + + Returns: + pd.DataFrame | None: Rows with missing filenames when present; otherwise None. + """ + # Find rows that have a URL but no filename, since those records cannot be downloaded safely. missing = data_df.loc[data_df[filename_col].isna() & data_df[url_col].notna()] if missing.empty: + # No missing names were found, so there is nothing to report. return None if len(missing) <= 5: - print("\n❗ Missing filenames detected (showing all):") + # Show the full set of problematic rows directly in the console for small cases. + print("\n Missing filenames detected (showing all):") print(missing) else: - # Save path follows BuddyCheck pattern: _missing_filenames.csv + # For larger problems, save the affected rows to disk so they can be fixed offline. csv_base = os.path.splitext(self.csv_path)[0] save_path = f"{csv_base}_missing_filenames.csv" missing.to_csv(save_path, index=False) print( - f"\n❗ Missing filenames detected for {len(missing)} rows.\n" + f"\n Missing filenames detected for {len(missing)} rows.\n" f"Because there are more than 5, they were saved to:\n {save_path}\n" f"Please correct the CSV and re-run." ) - # Return missing rows for reference + # Return the flagged rows so the calling code can inspect or report them further. return missing def setup_expected_columns(self, args): - """Set up the expected columns dictionary for CSV processing.""" + """ + Build the expected column mapping used by the download workflow. + + Parameters: + args (argparse.Namespace): Parsed command-line arguments. + + Returns: + tuple[dict, str | None]: Column mapping to use for filename, URL, and optional subfolder handling. + """ + # Normalize the CLI input into the internal column names used throughout the workflow. subfolders = args.subdir_col expected_cols = { "filename_col": args.img_name_col.lower(), "url_col": args.url_col.lower() } if subfolders: + # Keep the subfolder column name consistent with the rest of the input handling. subfolders = subfolders.lower() expected_cols["subfolders"] = subfolders return expected_cols, subfolders def check_existing_images(self, csv_path, img_dir, source_df, filename_col, subfolders=None): - """Checks which files from the CSV already exist in the image directory.""" + """ + Checks which files from the CSV already exist in the image directory. + + Adds a new boolean column `in_img_dir` to source_df indicating which images + are already in the directory. + + If all images already exist in the directory, the function will exit early + by calling `sys.exit()`, and no further processing will occur. + + Parameters: + csv_path (str): Path to the CSV file containing image information. + img_dir (str): Path to the directory where images are to be stored. + source_df (pd.DataFrame): DataFrame loaded from the CSV, containing image metadata. + filename_col (str): Name of the column in source_df that contains image filenames. + subfolders (str): Name of the column in source_df that contains subfolder names. (optional) + + Returns: + updated_df (pd.DataFrame): DataFrame with new column 'in_img_dir' indicating presence in img_dir. + filtered_df (pd.DataFrame): DataFrame filtered to only files not present in img_dir. + """ + # Create a copy to avoid modifying the original DataFrame df = source_df.copy() if not os.path.exists(img_dir): + # Directory doesn't exist, so nothing to check df["in_img_dir"] = False + # Return the updated df and the filtered dataframe of items that still need downloading filtered_df = df[~df["in_img_dir"]].copy() return df, filtered_df try: existing_files = gather_file_paths(img_dir) except EmptyInputDirectoryError: + # If the directory exists but is empty, sumbuddy raises an error. + # We catch it and treat it as an empty file list. existing_files = [] existing_full_paths = {os.path.normpath(os.path.relpath(f, img_dir)) for f in existing_files} if subfolders: + # We use a generic join here, but the apply(os.path.normpath) below fixes it for the specific OS raw_paths = df[subfolders].astype(str) + os.sep + df[filename_col].astype(str) + # This converts '/' to '\' on Windows, or vice versa, ensuring a match df["expected_path"] = raw_paths.apply(os.path.normpath) else: + # Normalize even simple filenames just in case they contain pathing characters df["expected_path"] = df[filename_col].astype(str).apply(os.path.normpath) - + + # Determine which expected paths physically exist expected_present = df["expected_path"].isin(existing_full_paths) df["in_img_dir"] = expected_present.copy() + + # Clean up the temporary column before returning. df = df.drop(columns=["expected_path"]) + + # Create filtered DataFrame filtered_df = df[~df["in_img_dir"]].copy() + # Exit if all images are already there if filtered_df.empty: sys.exit(f"'{img_dir}' already contains all images. Exited without executing.") else: + # Print directory status message - pre-download num_existing = len(existing_files) print(f"There are {num_existing} of the desired files already in {img_dir}. Based on {csv_path}, {filtered_df.shape[0]} images should be downloaded.") return df, filtered_df def _preview_or_save(self, label, df): - """Print up to 5 rows or save to CSV if larger.""" + """ + Print a small preview of a DataFrame or save it to disk when the result is larger. + + Parameters: + label (str): Descriptive prefix for the output file and console message. + df (pd.DataFrame): DataFrame containing the rows to preview or save. + """ if len(df) <= 5: - print(f"\n❗ {label.replace('_', ' ').title()} detected (showing all):") + # Small result sets are easier to inspect directly in the terminal. + print(f"\n {label.replace('_', ' ').title()} detected (showing all):") print(df) else: + # Larger result sets are written to a CSV so the user can review them without clutter. csv_base = os.path.splitext(self.csv_path)[0] save_path = f"{csv_base}_{label}.csv" df.to_csv(save_path, index=False) print( - f"\n❗ {len(df)} {label.replace('_', ' ')} detected.\n" + f"\n {len(df)} {label.replace('_', ' ')} detected.\n" f"Full list saved to:\n {save_path}\n" ) def check_duplicate_checksums(self, data_df, hash_col, ignore_duplicates=False): - """Detect duplicate checksum values and optionally block execution.""" + """ + Detect duplicate checksum values and optionally block execution. - # Only consider non-null hashes + Parameters: + data_df (pd.DataFrame): DataFrame containing the checksum column. + hash_col (str): Name of the checksum column to inspect. + ignore_duplicates (bool): If True, allow execution to continue after reporting duplicates. + + Raises: + SystemExit: If duplicate checksums are found and duplicates are not ignored. + """ + # Only consider non-null hashes when searching for repeated values. dupes = ( data_df[data_df[hash_col].notna()] .groupby(hash_col) @@ -116,28 +214,39 @@ def check_duplicate_checksums(self, data_df, hash_col, ignore_duplicates=False): ) if dupes.empty: - print("✔ No duplicate checksums detected.") + # No repeated checksums were found, so the download process can continue. + print(" No duplicate checksums detected.") return # If duplicates exist, preview or save self._preview_or_save("duplicate_checksums", dupes) if ignore_duplicates: + # The user explicitly allowed duplicates, so the workflow reports them and proceeds. print( - f"\n⚠ Duplicate checksums detected ({len(dupes)} rows), " + f"\n Duplicate checksums detected ({len(dupes)} rows), " f"but --ignore-duplicates was passed. Continuing.\n" ) return # Default behavior: block execution sys.exit( - "❗ Duplicate checksums detected. " + " Duplicate checksums detected. " "Use --ignore-duplicates to allow downloading duplicates." ) def print_download_summary(self, img_dir, downsample_dir, subfolders, num_images): - """Print a summary of where images and downsized images will be saved.""" - print("\n📦 Download Summary") + """ + Print a summary of where images and downsized images will be saved. + + Parameters: + img_dir (str): Destination directory for downloaded images. + downsample_dir (str | None): Optional directory for resized images. + subfolders (str | None): Optional subfolder naming column. + num_images (int): Number of images that will be downloaded. + """ + # Print the planned download layout so the user can confirm the output locations. + print("\n Download Summary") print("--------------------") print(f"Images will be downloaded to: {img_dir}") diff --git a/tests/test_roll_call.py b/tests/test_roll_call.py index 98bf5cf..1cdd795 100644 --- a/tests/test_roll_call.py +++ b/tests/test_roll_call.py @@ -513,7 +513,7 @@ def test_summary_formatting_header(self, mock_print): printed_lines = [call.args[0] for call in mock_print.call_args_list] - self.assertIn("📦 Download Summary", printed_lines[0]) + self.assertIn(" Download Summary", printed_lines[0]) self.assertIn("--------------------", printed_lines[1]) From 20237222b51ef0bdcfe6c9a0f5e5ec4b57712b7d Mon Sep 17 00:00:00 2001 From: Ishika Aahna <123102828+codedbyishika@users.noreply.github.com> Date: Fri, 17 Jul 2026 12:49:05 -0400 Subject: [PATCH 20/24] Apply suggestions from code review Co-authored-by: Elizabeth Campolongo <38985481+egrace479@users.noreply.github.com> --- AGENTS.md | 2 +- src/cautiousrobot/__main__.py | 6 +++--- src/cautiousrobot/roll_call.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index d9144b3..169edb7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -17,7 +17,7 @@ ### Core Modules - `__main__.py` - Main entry point with CLI argument parsing and workflow orchestration -- `roll_call.py` - CSV validation and input handling functions +- `roll_call.py` - Pre-download checks and processing. - `download.py` - Core image downloading functionality with retry logic - `buddy_check.py` - Checksum validation and download verification using BuddyCheck class - `utils.py` - Helper functions for CSV processing, logging, and image downsampling diff --git a/src/cautiousrobot/__main__.py b/src/cautiousrobot/__main__.py index 5b564ac..2c40b87 100644 --- a/src/cautiousrobot/__main__.py +++ b/src/cautiousrobot/__main__.py @@ -157,7 +157,7 @@ def main(): # Download images with or without downsampling if isinstance(args.side_length, int): - downsample_dest_path = img_dir + "_downsized" + if downsample_dir: # Download images from urls & save downsample copy download_images(filtered_df, img_dir=img_dir, @@ -165,12 +165,12 @@ def main(): error_log_filepath=error_log_filepath, filename=filename_col, subfolders=subfolders, - downsample_path=downsample_dest_path, + downsample_path=downsample_dir, downsample=args.side_length, file_url=url_col, wait=args.wait_time, retry=args.max_retries) - print(f"Images downloaded from {csv_path} to {img_dir}, with downsampled images in {downsample_dest_path}.") + print(f"Images downloaded from {csv_path} to {img_dir}, with downsampled images in {downsample_dir}.") else: # Download images from urls without downsample copy download_images(filtered_df, diff --git a/src/cautiousrobot/roll_call.py b/src/cautiousrobot/roll_call.py index 7a51127..26b57a7 100644 --- a/src/cautiousrobot/roll_call.py +++ b/src/cautiousrobot/roll_call.py @@ -57,7 +57,7 @@ def handle_missing_filenames(self, data_df, filename_col, url_col): url_col (str): Name of the column containing image URLs. Returns: - pd.DataFrame | None: Rows with missing filenames when present; otherwise None. + missing (pd.DataFrame) | None: DataFrame of entries with URLs but missing filenames or None. """ # Find rows that have a URL but no filename, since those records cannot be downloaded safely. missing = data_df.loc[data_df[filename_col].isna() & data_df[url_col].notna()] @@ -89,7 +89,7 @@ def setup_expected_columns(self, args): args (argparse.Namespace): Parsed command-line arguments. Returns: - tuple[dict, str | None]: Column mapping to use for filename, URL, and optional subfolder handling. + (expected_cols, subfolders) tuple[dict, str | None]: Column mapping to use for filename, URL, and optional subfolder handling. """ # Normalize the CLI input into the internal column names used throughout the workflow. subfolders = args.subdir_col From dc279670dace1132eaa34ffcc054ac800862d0b9 Mon Sep 17 00:00:00 2001 From: Ishika Aahna <123102828+codedbyishika@users.noreply.github.com> Date: Sun, 19 Jul 2026 23:01:58 -0400 Subject: [PATCH 21/24] Update error in main --- src/cautiousrobot/__main__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/cautiousrobot/__main__.py b/src/cautiousrobot/__main__.py index 2c40b87..0484437 100644 --- a/src/cautiousrobot/__main__.py +++ b/src/cautiousrobot/__main__.py @@ -156,7 +156,6 @@ def main(): log_filepath, error_log_filepath, metadata_path = setup_log_paths(csv_path) # Download images with or without downsampling - if isinstance(args.side_length, int): if downsample_dir: # Download images from urls & save downsample copy download_images(filtered_df, From 4e21d0f8570ba165c355c735f4791d8c92b24ea4 Mon Sep 17 00:00:00 2001 From: Ishika Aahna <123102828+codedbyishika@users.noreply.github.com> Date: Sun, 19 Jul 2026 23:42:02 -0400 Subject: [PATCH 22/24] Allow user to ignore duplicate checksums --- src/cautiousrobot/__main__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cautiousrobot/__main__.py b/src/cautiousrobot/__main__.py index 0484437..d67636e 100644 --- a/src/cautiousrobot/__main__.py +++ b/src/cautiousrobot/__main__.py @@ -49,6 +49,7 @@ def parse_args(): help = f"checksum algorithm to use on images (default: md5, available: {available_algorithms})" ) opt_args.add_argument("-v", "--verifier-col", required = False, help = "name of column in source CSV with checksums (same hash as -a) to verify download", nargs = "?") + opt_args.add_argument("--ignore-duplicates", action="store_true", help="Continue download when duplicate checksum values are detected in the source CSV. Duplicates are still printed or saved.") return parser.parse_args() From 2af2fb44c0322c12a7666c60100ad67031d74040 Mon Sep 17 00:00:00 2001 From: Ishika Aahna <123102828+codedbyishika@users.noreply.github.com> Date: Tue, 21 Jul 2026 13:36:29 -0400 Subject: [PATCH 23/24] Remove 5 value threshold --- src/cautiousrobot/roll_call.py | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/src/cautiousrobot/roll_call.py b/src/cautiousrobot/roll_call.py index 26b57a7..52dda65 100644 --- a/src/cautiousrobot/roll_call.py +++ b/src/cautiousrobot/roll_call.py @@ -180,20 +180,15 @@ def _preview_or_save(self, label, df): label (str): Descriptive prefix for the output file and console message. df (pd.DataFrame): DataFrame containing the rows to preview or save. """ - if len(df) <= 5: - # Small result sets are easier to inspect directly in the terminal. - print(f"\n {label.replace('_', ' ').title()} detected (showing all):") - print(df) - else: - # Larger result sets are written to a CSV so the user can review them without clutter. - csv_base = os.path.splitext(self.csv_path)[0] - save_path = f"{csv_base}_{label}.csv" - df.to_csv(save_path, index=False) - print( - f"\n {len(df)} {label.replace('_', ' ')} detected.\n" - f"Full list saved to:\n {save_path}\n" - ) - + # Result sets are written to a CSV so the user can review them without clutter. + csv_base = os.path.splitext(self.csv_path)[0] + save_path = f"{csv_base}_{label}.csv" + df.to_csv(save_path, index=False) + print( + f"\n {len(df)} {label.replace('_', ' ')} detected.\n" + f"Full list saved to:\n {save_path}\n" + ) + def check_duplicate_checksums(self, data_df, hash_col, ignore_duplicates=False): """ Detect duplicate checksum values and optionally block execution. From 63e93a7ae2cb9f6f0cfadf76cd14287f8538c10b Mon Sep 17 00:00:00 2001 From: Ishika Aahna <123102828+codedbyishika@users.noreply.github.com> Date: Tue, 21 Jul 2026 13:43:43 -0400 Subject: [PATCH 24/24] Test Fixture --- tests/test_roll_call.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_roll_call.py b/tests/test_roll_call.py index 1cdd795..40d5cc0 100644 --- a/tests/test_roll_call.py +++ b/tests/test_roll_call.py @@ -417,8 +417,10 @@ def test_preview_small_dataframe(self, mock_to_csv, mock_print): self.rollcall._preview_or_save("test_label", df) + mock_to_csv.assert_called_once() + args, kwargs = mock_to_csv.call_args + self.assertIn("testdata_test_label.csv", args[0]) mock_print.assert_called() - mock_to_csv.assert_not_called() @patch("builtins.print") @patch("pandas.DataFrame.to_csv")