Skip to content

Commit 970233f

Browse files
committed
Fix pipeline safety, debug flow, and packaging issues
1 parent 77db193 commit 970233f

21 files changed

Lines changed: 513 additions & 292 deletions

‎.github/workflows/ci.yml‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
name: CI
2+
3+
on:
4+
push:
5+
pull_request:
6+
7+
jobs:
8+
checks:
9+
runs-on: ubuntu-latest
10+
steps:
11+
- uses: actions/checkout@v4
12+
- uses: actions/setup-python@v5
13+
with:
14+
python-version: "3.12"
15+
- name: Install test runner
16+
run: python -m pip install --upgrade pip pytest
17+
- name: Syntax check
18+
run: python -m compileall opensr_utils setup.py demo.py tests
19+
- name: Shell syntax check
20+
run: bash -n run_batch.sh
21+
- name: Import tests
22+
run: pytest -q

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ uncertainty.py
2727

2828
# other
2929
__pycache__/*
30+
__pycache__
3031
*.pyc
3132

3233
# Byte-compiled / optimized / DLL files

‎CITATION.cff‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,4 +28,4 @@ keywords:
2828
- PyTorch
2929
- OpenSR
3030

31-
license: Apache-2.0
31+
license: MIT

‎README.md‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ opensr-run /data/flood_scene.tif LDSRS2 --window_size 128 128 --factor 4 -
5757

5858
**Arguments explained:**
5959
- `root` → Path to the input file or folder (`.tif`, `.SAFE`, or S2GM).
60-
- `model` → Must be `LDSRS2` for CLI usage (any other value falls back to a no-model pipeline).
60+
- `model` → `LDSRS2` or `None` for the interpolation placeholder.
6161
- `--window_size` → Patch size in LR pixels (default: `128 128`).
6262
- `--factor` → Upscaling factor (default: `4`).
6363
- `--overlap` → Overlap in pixels to avoid patch seams (default: `8`).
@@ -66,6 +66,8 @@ opensr-run /data/flood_scene.tif LDSRS2 --window_size 128 128 --factor 4 -
6666
- `--gpus` → List of GPU IDs (e.g. `0 1`).
6767
- `--save_preview` → Saves LR/SR preview images every 10% of progress.
6868
- `--debug` → Process only ~100 windows for testing.
69+
- `--overwrite` → Replace an existing `sr.tif`; otherwise existing outputs are preserved.
70+
- `--keep_temp` → Keep temporary patch files after stitching.
6971

7072
---
7173

@@ -81,15 +83,14 @@ First, download a .SAFE tile from the Copernicus Browser, a Mosaic folder from S
8183

8284
#### 1. install libraries
8385
```sql
84-
pip install opensr-utils
85-
pip install opensr-model
86+
pip install "opensr-utils[model]"
8687
```
8788
#### 2. Create Model - in this case our LDSR-S2 Model
8889
```python
89-
# Instanciate Model
90-
import opensr_model # import pachage
91-
model = opensr_model.SRLatentDiffusion(config, device=device) # create model
92-
model.load_pretrained(config.ckpt_version) # load checkpoint
90+
from opensr_utils.model_utils.get_models import get_ldsrs2
91+
92+
device = "cuda"
93+
model = get_ldsrs2(device="cpu")
9394
```
9495
#### 3. Run large-scale Inference
9596

@@ -110,7 +111,7 @@ sr_object = opensr_utils.large_file_processing(
110111
debug=False,
111112
)
112113

113-
sr_object.start_super_resolution()
114+
sr_object.run()
114115
```
115116

116117

@@ -131,11 +132,11 @@ If you use this code in your research, please cite:
131132
```bibtex
132133
@software{Donike_OpenSR-Utils_2025,
133134
author = {Donike, Simon and Gomez-Chova, Luis},
134-
license = {Apache-2.0},
135+
license = {MIT},
135136
month = nov,
136137
title = {{OpenSR-Utils}},
137138
url = {https://github.com/ESAOpenSR/opensr-utils},
138-
version = {0.3.0},
139+
version = {1.2.0},
139140
year = {2025},
140141
doi = {10.5281/zenodo.17599310}
141142
}

‎demo.py‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,16 +6,10 @@
66
1. How to instantiate an SR model from opensr-model
77
2. How to run large-scale inference with opensr-utils
88
"""
9-
import torch
10-
import requests
11-
from omegaconf import OmegaConf
12-
from io import StringIO
13-
14-
159
# --- 1) Create Model -------------------------------------------------
1610
device = "cuda" # or "cpu" - Dont use the automated detection, it messes up the lightning trainer multi-GPU setup
1711
from opensr_utils.model_utils.get_models import get_ldsrs2
18-
model = get_ldsrs2()
12+
model = get_ldsrs2(device="cpu")
1913

2014
# --- 2) Run large-scale Inference ------------------------------------
2115
import opensr_utils
@@ -32,3 +26,4 @@
3226
device=device, # "cuda" for GPU-accelerated inference
3327
gpus=0, # pass GPU ID or list of GPUs
3428
)
29+
sr_object.run()

‎opensr_utils/__init__.py‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,12 @@
88
except Exception:
99
__version__ = "unknown"
1010

11-
# expose high-level class for convenience
12-
from .pipeline import large_file_processing
11+
__all__ = ["large_file_processing", "__version__"]
1312

14-
__all__ = ["large_file_processing", "__version__"]
13+
14+
def __getattr__(name):
15+
if name == "large_file_processing":
16+
from .pipeline import large_file_processing
17+
18+
return large_file_processing
19+
raise AttributeError(f"module 'opensr_utils' has no attribute {name!r}")

‎opensr_utils/cli.py‎

Lines changed: 25 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
def main():
22
import argparse
3-
import math
43
from opensr_utils.pipeline import large_file_processing
54

65

@@ -28,24 +27,30 @@ def main():
2827
help="🖼️ Also save one georeferenced crop and 10 LR/SR previews")
2928
parser.add_argument("--debug", action="store_true",
3029
help="🐞 Debug mode: process only ~100 windows")
30+
parser.add_argument("--overwrite", action="store_true",
31+
help="Replace an existing sr.tif output")
32+
parser.add_argument("--keep_temp", action="store_true",
33+
help="Keep temporary patch files after stitching")
34+
parser.add_argument("--delete_input_zip", action="store_true",
35+
help="Delete the source .zip after successful extraction")
36+
parser.add_argument("--batch_size", type=int, default=16,
37+
help="Prediction batch size (default: 16)")
38+
parser.add_argument("--num_workers", type=int, default=4,
39+
help="DataLoader worker count (default: 4)")
40+
parser.add_argument("--prefetch_factor", type=int, default=2,
41+
help="DataLoader prefetch factor when workers > 0 (default: 2)")
42+
parser.add_argument("--compressed_patches", action="store_true",
43+
help="Store temporary patches as compressed .npz files")
3144

3245
args = parser.parse_args()
3346

3447
# --- Resolve model selection ---
3548
if args.model == "LDSRS2":
3649
# Lazy import to avoid CUDA init unless needed
3750
try:
38-
from omegaconf import OmegaConf
39-
from io import StringIO
40-
import requests # your model package
41-
import opensr_model # your model package
42-
# Instantiate model
51+
from opensr_utils.model_utils.get_models import get_ldsrs2
4352
device = "cpu" # "cuda" or "cpu" - Dont use the torch automated detection here, it messes up the lightning trainer multi-GPU setup
44-
config_url = "https://raw.githubusercontent.com/ESAOpenSR/opensr-model/refs/heads/main/opensr_model/configs/config_10m.yaml"
45-
response = requests.get(config_url)
46-
config = OmegaConf.load(StringIO(response.text))
47-
model = opensr_model.SRLatentDiffusion(config, device=device)
48-
model.load_pretrained(config.ckpt_version)
53+
model = get_ldsrs2(device=device)
4954

5055
except Exception as e:
5156
print("❌ Could not load LDSR-S2 model from 'opensr_model'.")
@@ -67,7 +72,15 @@ def main():
6772
gpus=args.gpus if args.device == "cuda" else None,
6873
save_preview=args.save_preview,
6974
debug=args.debug,
75+
cleanup=not args.keep_temp,
76+
overwrite=args.overwrite,
77+
delete_input_zip=args.delete_input_zip,
78+
batch_size=args.batch_size,
79+
num_workers=args.num_workers,
80+
prefetch_factor=args.prefetch_factor,
81+
compressed_patches=args.compressed_patches,
7082
)
83+
runner.run()
7184

7285
if __name__ == "__main__":
73-
main()
86+
main()
Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,24 @@
11
# -*- coding: utf-8 -*-
22
"""Data utilities for OpenSR-Utils."""
33

4-
from .writing_utils import write_to_placeholder
5-
from .datamodule import PredictionDataModule
6-
from .reading_utils import can_read_directly_with_rasterio
7-
84
__all__ = [
95
"write_to_placeholder",
106
"PredictionDataModule",
117
"can_read_directly_with_rasterio",
12-
]
8+
]
9+
10+
11+
def __getattr__(name):
12+
if name == "write_to_placeholder":
13+
from .writing_utils import write_to_placeholder
14+
15+
return write_to_placeholder
16+
if name == "PredictionDataModule":
17+
from .datamodule import PredictionDataModule
18+
19+
return PredictionDataModule
20+
if name == "can_read_directly_with_rasterio":
21+
from .reading_utils import can_read_directly_with_rasterio
22+
23+
return can_read_directly_with_rasterio
24+
raise AttributeError(f"module 'opensr_utils.data_utils' has no attribute {name!r}")

‎opensr_utils/data_utils/datamodule.py‎

Lines changed: 26 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,7 @@ def __init__(self,input_type,root,windows,lr_file_dict):
9090
self.input_type = input_type
9191
self.root = root
9292
self.windows = windows
93+
self._src_cache = {}
9394

9495
# 1. If file
9596
if self.input_type == "file":
@@ -101,6 +102,26 @@ def __init__(self,input_type,root,windows,lr_file_dict):
101102
elif self.input_type == "S2GM":
102103
self.lr_file_dict = lr_file_dict
103104

105+
def __getstate__(self):
106+
state = self.__dict__.copy()
107+
state["_src_cache"] = {}
108+
return state
109+
110+
def _open_cached(self, path):
111+
if path not in self._src_cache:
112+
self._src_cache[path] = rasterio.open(path)
113+
return self._src_cache[path]
114+
115+
def close(self):
116+
for src in self._src_cache.values():
117+
src.close()
118+
self._src_cache.clear()
119+
120+
def __del__(self):
121+
try:
122+
self.close()
123+
except Exception:
124+
pass
104125

105126
def __len__(self):
106127
return len(self.windows)
@@ -148,8 +169,8 @@ def get_from_file(self, idx):
148169
(C,H,W) patch in native band order, float32.
149170
"""
150171
window = self.windows[idx]
151-
with rasterio.open(self.root) as src:
152-
data = src.read(window=window) # np.ndarray
172+
src = self._open_cached(self.root)
173+
data = src.read(window=window) # np.ndarray
153174
data = data.astype(np.float32)
154175
return torch.from_numpy(data)
155176

@@ -171,9 +192,9 @@ def get_from_dict(self, idx):
171192
bands_order = ["R", "G", "B", "NIR"]
172193
tiles = []
173194
for b in bands_order:
174-
with rasterio.open(self.lr_file_dict[b]) as src:
175-
tile = src.read(1, window=window)
176-
tiles.append(tile)
195+
src = self._open_cached(self.lr_file_dict[b])
196+
tile = src.read(1, window=window)
197+
tiles.append(tile)
177198
arr = np.stack(tiles, axis=0).astype(np.float32)
178199
return torch.from_numpy(arr)
179200

0 commit comments

Comments
 (0)