Add tropomi NO2 and CO, public S3 aquire.

src/swell/configuration/jedi/interfaces/geos_cf/convert_observations/tropomi_s5p_co_total.yaml

@@ -0,0 +1,26 @@
+# TROPOMI S5P CO Total Column L2 — IODA converter configuration
+# Converter: tropomi_no2_co_nc2ioda.py (must be available in `converter_path`
+#            or in jedi_bundle/build/bin/)
+#
+# Invocation (one call per cycle, all granules passed together):
+#   python3 <jedi_bundle>/build/bin/tropomi_no2_co_nc2ioda.py
+#       -i <granule1.nc> <granule2.nc> ...
+#       -o <cycle_dir>/ioda/tropomi_s5p_co_total/tropomi_s5p_co_total_YYYYMMDDHH.nc
+#       -v co
+#       -c total
+#       -n 0.0
+#       -q 0.5
+
+# Name of the ioda-converters Python script.
+converter_script: tropomi_no2_co_nc2ioda.py
+
+# Output filename template (must match the source pattern in
+# ingest_observations/tropomi_s5p_co_total.yaml).
+output_filename_template: "tropomi_s5p_co_total_%Y%m%d%H.nc"
+
+# Additional flags passed verbatim to the converter after -i and -o.
+extra_flags:
+  -v: co     # variable name
+  -c: total  # column type
+  -n: 0.0    # no thinning
+  -q: 0.5    # quality value threshold

src/swell/configuration/jedi/interfaces/geos_cf/convert_observations/tropomi_s5p_no2_tropo.yaml

@@ -0,0 +1,26 @@
+# TROPOMI S5P NO2 Tropospheric Column L2 — IODA converter configuration
+# Converter: tropomi_no2_co_nc2ioda.py (must be available in `converter_path`
+#            or in jedi_bundle/build/bin/)
+#
+# Invocation (one call per cycle, all granules passed together):
+#   python3 <jedi_bundle>/build/bin/tropomi_no2_co_nc2ioda.py
+#       -i <granule1.nc> <granule2.nc> ...
+#       -o <cycle_dir>/ioda/tropomi_s5p_no2_tropo/tropomi_s5p_no2_tropo_YYYYMMDDHH.nc
+#       -v no2
+#       -c troposphere
+#       -n 0.0
+#       -q 0.75
+
+# Name of the ioda-converters Python script.
+converter_script: tropomi_no2_co_nc2ioda.py
+
+# Output filename template (must match the source pattern in
+# ingest_observations/tropomi_s5p_no2_tropo.yaml).
+output_filename_template: "tropomi_s5p_no2_tropo_%Y%m%d%H.nc"
+
+# Additional flags passed verbatim to the converter after -i and -o.
+extra_flags:
+  -v: no2           # variable name
+  -c: troposphere   # column type
+  -n: 0.0           # no thinning
+  -q: 0.75          # quality value threshold

src/swell/configuration/jedi/interfaces/geos_cf/download_observations/tropomi_s5p_co_total.yaml

@@ -0,0 +1,16 @@
+# TROPOMI S5P CO Total Column L2 — download configuration
+# Instrument: TROPOMI on Sentinel-5P
+# Product:    NRTI L2__CO____ (Near Real-Time total column CO)
+# Source:     MEEO S3 public bucket (no credentials required)
+#
+# Files are named:
+#   S5P_NRTI_L2__CO_____YYYYMMDDTHHmmss_*.nc
+
+retrieval_method: s3_public
+s3_source: 's3://meeo-s5p/NRTI/L2__CO____/YYYY/MM/DD'
+
+# Maximum duration of a single TROPOMI orbit granule. Used to extend the
+# file search window backwards so that orbits starting before window_begin
+# but containing data within the DA window are not missed.
+# TROPOMI orbits are approximately 101 minutes; PT2H provides a safe margin.
+max_orbit_duration: PT2H

src/swell/configuration/jedi/interfaces/geos_cf/download_observations/tropomi_s5p_no2_tropo.yaml

@@ -0,0 +1,16 @@
+# TROPOMI S5P NO2 Tropospheric Column L2 — download configuration
+# Instrument: TROPOMI on Sentinel-5P
+# Product:    NRTI L2__NO2___ (Near Real-Time tropospheric NO2)
+# Source:     MEEO S3 public bucket (no credentials required)
+#
+# Files are named:
+#   S5P_NRTI_L2__NO2____YYYYMMDDTHHmmss_*.nc
+
+retrieval_method: s3_public
+s3_source: 's3://meeo-s5p/NRTI/L2__NO2___/YYYY/MM/DD'
+
+# Maximum duration of a single TROPOMI orbit granule. Used to extend the
+# file search window backwards so that orbits starting before window_begin
+# but containing data within the DA window are not missed.
+# TROPOMI orbits are approximately 101 minutes; PT2H provides a safe margin.
+max_orbit_duration: PT2H

src/swell/configuration/jedi/interfaces/geos_cf/ingest_observations/tropomi_s5p_co_total.yaml

@@ -0,0 +1,5 @@
+# TROPOMI S5P CO Total Column L2 — R2D2 ingestion configuration
+# Source: output of ConvertObsToIoda, relative to the cycle work directory.
+
+acquisition_method: local
+source: ioda/tropomi_s5p_co_total/tropomi_s5p_co_total_%Y%m%d%H.nc # in CYCLE_DIR

src/swell/configuration/jedi/interfaces/geos_cf/ingest_observations/tropomi_s5p_no2_tropo.yaml

@@ -0,0 +1,5 @@
+# TROPOMI S5P NO2 Tropospheric Column L2 — R2D2 ingestion configuration
+# Source: output of ConvertObsToIoda, relative to the cycle work directory.
+
+acquisition_method: local
+source: ioda/tropomi_s5p_no2_tropo/tropomi_s5p_no2_tropo_%Y%m%d%H.nc # in CYCLE_DIR

src/swell/configuration/jedi/observation_ioda_names.yaml

@@ -354,8 +354,8 @@ ioda instrument names:
     full name: TROPOMI tropospheric column NO2
     inst type: retrieval
     provider : esa
-  - ioda name: tropomi_s5p_co_tropo
-    full name: TROPOMI tropospheric column CO
+  - ioda name: tropomi_s5p_co_total
+    full name: TROPOMI S5P total column CO
     inst type: retrieval
     provider : esa
   - ioda name: omps_o3_nm_total

src/swell/suites/ingest_obs/suite_config.py

@@ -53,8 +53,10 @@ class SuiteConfig(QuestionContainer, Enum):
         ],
         geos_cf=[
             qd.window_length("PT6H"),
-            qd.obs_to_download(['tempo_no2_tropo']),
-            qd.obs_to_ingest(['tempo_no2_tropo']),
+            qd.obs_to_download(['omps_o3_nm_total', 'tropomi_s5p_no2_tropo',
+                                'tropomi_s5p_co_total', 'tempo_no2_tropo']),
+            qd.obs_to_ingest(['omps_o3_nm_total', 'tropomi_s5p_no2_tropo',
+                              'tropomi_s5p_co_total', 'tempo_no2_tropo']),
             qd.converter_path(
                 "/discover/nobackup/projects/jcsda/s2127/maryamao/"
                 "jedi-bundle/build-intel-1.9/bin/"

src/swell/tasks/download_obs.py

@@ -7,13 +7,18 @@
 """
 Task for downloading raw observation files from remote servers.
 
-Two retrieval methods are supported, selected by ``retrieval_method`` in
-the per-obs download YAML:
+Downloads native observation files (e.g. HDF5, NetCDF) from either HTTPS
+servers such as NASA GES DISC, from public AWS S3 buckets, or from
+authenticated HTTPS endpoints for NASA ASDC datasets.
 
 ``https`` (default)
     Scrapes an HTML directory listing and streams files via HTTPS.
     Authentication uses ``~/.netrc``.
 
+``s3_public``
+    Downloads from a publicly readable S3 bucket using anonymous boto3
+    access (e.g. ``s3://meeo-s5p``).  Requires ``boto3``.
+
 ``cmr``
     Queries the NASA CMR API for granule URLs, then downloads via
     authenticated HTTPS (Earthdata ``~/.netrc``).  Use this for NASA ASDC
@@ -136,6 +141,10 @@ def _download_obs(
         """
         retrieval_method = obs_config.get('retrieval_method', 'https')
 
+        if retrieval_method == 's3_public':
+            return self._download_obs_s3_public(
+                obs_config, obs_name, window_begin_dto, window_end_dto, dry_run)
+
         if retrieval_method == 'cmr':
             return self._download_obs_cmr(
                 obs_config, obs_name, window_begin_dto, window_end_dto, dry_run)
@@ -220,6 +229,114 @@ def _download_obs(
 
         return downloaded, failed
 
+    def _download_obs_s3_public(
+        self,
+        obs_config: dict,
+        obs_name: str,
+        window_begin_dto: datetime.datetime,
+        window_end_dto: datetime.datetime,
+        dry_run: bool,
+    ) -> tuple[int, int]:
+        """Download files for one observation type from a publicly readable
+        S3 bucket using anonymous (unsigned) boto3 access.
+
+        The ``obs_config`` dict must contain:
+
+        - ``s3_source``: S3 URI template with ``YYYY``, ``MM``, ``DD``
+          placeholders, e.g.
+          ``s3://meeo-s5p/NRTI/L2__NO2___/YYYY/MM/DD``.
+
+        Optional keys:
+
+        - ``max_orbit_duration``: ISO-8601 duration; extends the search
+          window backwards (default ``PT0H``).
+
+        Returns ``(n_downloaded, n_failed)``.
+        """
+        try:
+            import boto3
+            from botocore import UNSIGNED
+            from botocore.config import Config as BotocoreConfig
+            from botocore.exceptions import BotoCoreError, ClientError
+        except ImportError:
+            self.logger.abort(
+                "boto3 is required for 's3_public' retrieval but is not installed. "
+                'Install it with: pip install boto3')
+
+        s3_source_template = obs_config['s3_source']
+        max_orbit_dur = isodate.parse_duration(
+            obs_config.get('max_orbit_duration', 'PT0H'))
+
+        search_start = window_begin_dto - max_orbit_dur
+        search_end = window_end_dto
+
+        dest_dir = os.path.join(self.cycle_dir(), 'download', obs_name)
+        if not dry_run:
+            os.makedirs(dest_dir, exist_ok=True)
+
+        without_scheme = s3_source_template[len('s3://'):]
+        bucket, _, prefix_template = without_scheme.partition('/')
+
+        if dry_run:
+            for day_date in self._day_slots(search_start, search_end):
+                prefix = self._resolve_path(prefix_template, day_date)
+                self.logger.info(
+                    f'  [DRY RUN] Would list s3://{bucket}/{prefix}')
+            return 0, 0
+
+        s3_client = boto3.client(
+            's3',
+            config=BotocoreConfig(signature_version=UNSIGNED))
+
+        downloaded = 0
+        failed = 0
+
+        for day_date in self._day_slots(search_start, search_end):
+            prefix = self._resolve_path(prefix_template, day_date)
+            self.logger.info(f'  Listing s3://{bucket}/{prefix}')
+
+            try:
+                paginator = s3_client.get_paginator('list_objects_v2')
+                pages = paginator.paginate(Bucket=bucket, Prefix=prefix)
+                keys = [
+                    obj['Key']
+                    for page in pages
+                    for obj in page.get('Contents', [])
+                ]
+            except (BotoCoreError, ClientError) as exc:
+                self.logger.error(
+                    f'  Failed to list s3://{bucket}/{prefix}: {exc}')
+                failed += 1
+                continue
+
+            if not keys:
+                self.logger.info(
+                    f'  No objects found under s3://{bucket}/{prefix}')
+                continue
+
+            self.logger.info(
+                f'  {day_date}: {len(keys)} object(s) found')
+
+            for key in keys:
+                filename = os.path.basename(key)
+                dest_path = os.path.join(dest_dir, filename)
+
+                if os.path.exists(dest_path):
+                    self.logger.info(f'  Already exists, skipping: {filename}')
+                    downloaded += 1
+                    continue
+
+                try:
+                    s3_client.download_file(bucket, key, dest_path)
+                    self.logger.info(f'  Downloaded: {filename}')
+                    downloaded += 1
+                except (BotoCoreError, ClientError) as exc:
+                    self.logger.error(
+                        f'  Failed to download {filename}: {exc}')
+                    failed += 1
+
+        return downloaded, failed
+
     def _download_obs_cmr(
         self,
         obs_config: dict,
@@ -414,6 +531,20 @@ def _create_earthdata_session(self) -> requests.Session:
     # Slot/date helpers
     # ------------------------------------------------------------------
 
+    def _day_slots(
+        self,
+        search_start: datetime.datetime,
+        search_end: datetime.datetime,
+    ) -> list[datetime.date]:
+        """Return a list of unique date objects from search_start to search_end."""
+        days = []
+        current = search_start.date()
+        end_date = search_end.date()
+        while current <= end_date:
+            days.append(current)
+            current += datetime.timedelta(days=1)
+        return days
+
     def _hour_slots(
         self,
         search_start: datetime.datetime,