Refactor Validation and Create RollCall module

AGENTS.md

@@ -17,6 +17,7 @@
 
 ### Core Modules
 - `__main__.py` - Main entry point with CLI argument parsing and workflow orchestration
+- `roll_call.py` - Pre-download checks and processing.
 - `download.py` - Core image downloading functionality with retry logic
 - `buddy_check.py` - Checksum validation and download verification using BuddyCheck class
 - `utils.py` - Helper functions for CSV processing, logging, and image downsampling

src/cautiousrobot/__main__.py

@@ -8,12 +8,12 @@
 import pandas as pd
 import argparse
 import hashlib
-import os
 import sys
 from sumbuddy import get_checksums
-from cautiousrobot.utils import process_csv, check_existing_images
+from cautiousrobot.utils import process_csv
 from cautiousrobot.buddy_check import BuddyCheck
 from cautiousrobot.download import download_images
+from cautiousrobot.roll_call import RollCall
 from cautiousrobot.__about__ import __version__
 
 def parse_args():
@@ -49,50 +49,11 @@ def parse_args():
                         help = f"checksum algorithm to use on images (default: md5, available: {available_algorithms})"
                         )
     opt_args.add_argument("-v", "--verifier-col", required = False, help = "name of column in source CSV with checksums (same hash as -a) to verify download", nargs = "?")
+    opt_args.add_argument("--ignore-duplicates", action="store_true", help="Continue download when duplicate checksum values are detected in the source CSV. Duplicates are still printed or saved.")
 
     return parser.parse_args()
 
 
-def validate_csv_extension(csv_path):
-    """Validate that the input file has a .csv extension."""
-    if not csv_path.endswith(".csv"):
-        sys.exit("Expected CSV for input file; extension should be '.csv'")
-
-
-def setup_expected_columns(args):
-    """Set up the expected columns dictionary for CSV processing."""
-    subfolders = args.subdir_col
-    expected_cols = {
-        "filename_col": args.img_name_col.lower(),
-        "url_col": args.url_col.lower()
-    }
-    if subfolders:
-        subfolders = subfolders.lower()
-        expected_cols["subfolders"] = subfolders
-    return expected_cols, subfolders
-
-
-def validate_filename_uniqueness(data_df, filename_col):
-    """Validate that the filename column contains unique values."""
-    if data_df.loc[data_df[filename_col].notna()].shape[0] != data_df[filename_col].nunique():
-        sys.exit(f"{filename_col} is not a unique identifier for this dataset, please choose a column with unique values for filenames.")
-
-
-def handle_missing_filenames(data_df, filename_col, url_col):
-    """Handle cases where URLs exist but filenames are missing."""
-    urls_no_name = len(data_df.loc[(data_df[filename_col].isna() & (data_df[url_col].notna()))])
-    if urls_no_name > 0:
-        ignore = input(f"'{filename_col}' is missing values for {urls_no_name} URLs. Proceed with download ignoring these URLs? [y/n]: ")
-        if ignore.lower() != "y":
-            sys.exit("Exited without executing.")
-
-
-def validate_output_directory(img_dir):
-    """Validate that the output directory doesn't already exist."""
-    if os.path.exists(img_dir):
-        sys.exit(f"'{img_dir}' already exists. Exited without executing.")
-
-
 def setup_log_paths(csv_path):
     """Set up the log file paths based on the CSV path."""
     metadata_path = csv_path.split(".")[0]
@@ -147,11 +108,13 @@ def main():
     args = parse_args()
     csv_path = args.input_file
 
+    roll_call = RollCall(csv_path)
+
     # Validate CSV extension
-    validate_csv_extension(csv_path)
+    roll_call.validate_csv_extension(csv_path)
 
     # Set up expected columns and process CSV
-    expected_cols, subfolders = setup_expected_columns(args)
+    expected_cols, subfolders = roll_call.setup_expected_columns(args)
     try:
         data_df = process_csv(csv_path, expected_cols)
     except Exception as missing_cols:
@@ -160,35 +123,54 @@ def main():
     # Validate data and handle missing filenames
     filename_col = expected_cols["filename_col"]
     url_col = expected_cols["url_col"]
-    validate_filename_uniqueness(data_df, filename_col)
-    handle_missing_filenames(data_df, filename_col, url_col)
+    roll_call.validate_filename_uniqueness(data_df, filename_col)
+    roll_call.handle_missing_filenames(data_df, filename_col, url_col)
 
     # Set source DataFrame for only non-null filename values
     source_df = data_df.loc[data_df[filename_col].notna()].copy()
 
+    # If a verifier column is provided, check for duplicate checksums before downloading
+    if args.verifier_col:
+        roll_call.check_duplicate_checksums(
+            data_df=source_df,
+            hash_col=args.verifier_col.lower(),
+            ignore_duplicates=getattr(args, "ignore_duplicates", False)
+        )
+
     # Validate and handle existing output directory
     img_dir = args.output_dir
-    source_df, filtered_df = check_existing_images(csv_path, img_dir, source_df, filename_col, subfolders)
+    source_df, filtered_df = roll_call.check_existing_images(csv_path, img_dir, source_df, filename_col, subfolders)
+
+    # Prepare summary info
+    num_images = filtered_df.shape[0]
+    downsample_dir = img_dir + "_downsized" if isinstance(args.side_length, int) else None
+
+    # Print RollCall summary
+    roll_call.print_download_summary(
+        img_dir=img_dir,
+        downsample_dir=downsample_dir,
+        subfolders=subfolders,
+        num_images=num_images
+    )
 
     # Set up log paths
     log_filepath, error_log_filepath, metadata_path = setup_log_paths(csv_path)
 
     # Download images with or without downsampling
-    if isinstance(args.side_length, int):
-        downsample_dest_path = img_dir + "_downsized"
+    if downsample_dir:
         # Download images from urls & save downsample copy
         download_images(filtered_df,
                        img_dir=img_dir,
                        log_filepath=log_filepath,
                        error_log_filepath=error_log_filepath,
                        filename=filename_col,
                        subfolders=subfolders,
-                       downsample_path=downsample_dest_path,
+                       downsample_path=downsample_dir,
                        downsample=args.side_length,
                        file_url=url_col,
                        wait=args.wait_time,
                        retry=args.max_retries)
-        print(f"Images downloaded from {csv_path} to {img_dir}, with downsampled images in {downsample_dest_path}.")
+        print(f"Images downloaded from {csv_path} to {img_dir}, with downsampled images in {downsample_dir}.")
     else:
         # Download images from urls without downsample copy
         download_images(filtered_df,