Skip to content

Commit 1e899a5

Browse files
authored
Merge pull request #19 from jqnatividad/qsv-client-adoption
Run every qsv call through qsv-client
2 parents 7c6bb1c + 3148522 commit 1e899a5

9 files changed

Lines changed: 330 additions & 99 deletions

File tree

‎Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
1717
# TARGETARCH is supplied automatically by BuildKit. The release ships one zip
1818
# per architecture holding several build variants; only the full "qsv" binary
1919
# has describegpt, so that is the one extracted (qsvlite omits it).
20-
ARG QSV_VERSION=22.0.1
20+
ARG QSV_VERSION=24.0.0
2121
ARG TARGETARCH=amd64
2222
RUN set -eux; \
2323
case "${TARGETARCH}" in \

‎pyproject.toml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,9 @@ dependencies = [
4141
# HTTP Client
4242
"httpx>=0.26.0",
4343

44+
# qsv runner (timeouts, typed errors) for the onboarding profiling passes
45+
"qsv-client>=0.1.0",
46+
4447
# Vector Search (Pinecone for CKAN semantic search)
4548
# Note: Use the new `pinecone` package (v5+), NOT the legacy
4649
# `pinecone-client` — the code uses the new Pinecone class and

‎scripts/onboard_ckan.py‎

Lines changed: 8 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,12 @@
1414
import json
1515
import os
1616
import re
17-
import subprocess
1817
import sys
1918
from io import StringIO
2019
from pathlib import Path
2120

21+
from qsv_client import QsvNotFound, find_qsv
22+
2223
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src"))
2324

2425
csv.field_size_limit(10 * 1024 * 1024) # 10 MB — qsv stats can produce large fields
@@ -32,6 +33,7 @@
3233
build_index,
3334
download_csv,
3435
merge_columns,
36+
run_qsv_count,
3537
run_qsv_describegpt,
3638
run_qsv_frequency,
3739
run_qsv_stats,
@@ -211,17 +213,9 @@ async def process_resource(
211213

212214
# Step 7: Count rows from CSV
213215
if csv_path.exists():
214-
try:
215-
proc = await asyncio.create_subprocess_exec(
216-
"qsv", "count", str(csv_path),
217-
stdout=asyncio.subprocess.PIPE,
218-
stderr=asyncio.subprocess.PIPE,
219-
)
220-
stdout, _ = await proc.communicate()
221-
if proc.returncode == 0:
222-
result["row_count"] = int(stdout.decode().strip())
223-
except Exception:
224-
pass
216+
row_count = await run_qsv_count(csv_path)
217+
if row_count is not None:
218+
result["row_count"] = row_count
225219

226220
# Save meta.json
227221
resource_dir.mkdir(parents=True, exist_ok=True)
@@ -372,8 +366,8 @@ async def main() -> None:
372366

373367
# Verify qsv is installed
374368
try:
375-
subprocess.run(["qsv", "--version"], capture_output=True, check=True)
376-
except (FileNotFoundError, subprocess.CalledProcessError):
369+
find_qsv()
370+
except QsvNotFound:
377371
print("Error: qsv is not installed or not in PATH.")
378372
print("Install from: https://github.com/dathere/qsv")
379373
sys.exit(1)

‎scripts/onboard_dcat.py‎

Lines changed: 4 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848
_extract_qsv_tags,
4949
_now_iso,
5050
merge_columns,
51+
run_qsv_count,
5152
run_qsv_describegpt,
5253
run_qsv_frequency,
5354
run_qsv_stats,
@@ -218,17 +219,9 @@ async def process_dataset(
218219

219220
# Step 5: exact row count
220221
if csv_path.exists():
221-
try:
222-
proc = await asyncio.create_subprocess_exec(
223-
"qsv", "count", str(csv_path),
224-
stdout=asyncio.subprocess.PIPE,
225-
stderr=asyncio.subprocess.PIPE,
226-
)
227-
stdout, _ = await proc.communicate()
228-
if proc.returncode == 0:
229-
result["row_count"] = int(stdout.decode().strip())
230-
except Exception:
231-
pass
222+
row_count = await run_qsv_count(csv_path)
223+
if row_count is not None:
224+
result["row_count"] = row_count
232225

233226
resource_dir.mkdir(parents=True, exist_ok=True)
234227
meta_path.write_text(json.dumps(result, indent=2))

‎src/data_concierge/data_layer/qsv_profiling.py‎

Lines changed: 137 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,11 @@
1919
``count``
2020
Exact row count.
2121
22+
Every pass runs through `qsv-client <https://github.com/dathere/qsv-client>`_, which
23+
kills a run that exceeds its timeout (see ``_DEFAULT_TIMEOUTS``; each one can be
24+
overridden with ``VERIKAN_QSV_<PASS>_TIMEOUT``). A failed, timed-out, or missing
25+
qsv makes the pass return ``None``, and the onboarding scripts carry on without it.
26+
2227
:func:`merge_columns` folds all of those, plus the portal's own field
2328
metadata when it has any, into the single column list that
2429
``data_layer.onboard_index`` searches and that
@@ -27,16 +32,21 @@
2732

2833
from __future__ import annotations
2934

30-
import asyncio
3135
import csv
3236
import json
3337
import os
3438
from datetime import UTC, datetime
35-
from io import StringIO
3639
from pathlib import Path
3740
from typing import Any
3841

3942
import httpx
43+
from qsv_client import (
44+
AsyncQsv,
45+
QsvError,
46+
QsvNotFound,
47+
QsvTimeout,
48+
QsvVersionError,
49+
)
4050

4151
from data_concierge.data_layer.storage import storage
4252

@@ -61,6 +71,71 @@ async def download_csv(url: str, dest: Path) -> int:
6171
return size
6272

6373

74+
# Default per-pass timeouts in seconds. ``VERIKAN_QSV_<PASS>_TIMEOUT`` overrides one,
75+
# e.g. ``VERIKAN_QSV_STATS_TIMEOUT=1800`` for a portal with very large CSVs.
76+
_DEFAULT_TIMEOUTS: dict[str, float] = {
77+
"describegpt": 900.0, # several LLM round trips
78+
"stats": 900.0,
79+
"frequency": 600.0,
80+
"count": 120.0,
81+
}
82+
83+
# A failed run, a missing or too-old binary, or output that does not parse. QsvNotFound and
84+
# QsvVersionError are not QsvError subclasses, and the client raises QsvNotFound from its
85+
# constructor.
86+
_QSV_FAILURES = (QsvError, QsvNotFound, QsvVersionError, ValueError)
87+
88+
# Binaries whose missing describegpt has already been reported, so a portal run warns once.
89+
_describegpt_checked: set[str] = set()
90+
91+
92+
def _timeout(command: str) -> float:
93+
var = f"VERIKAN_QSV_{command.upper()}_TIMEOUT"
94+
raw = os.environ.get(var)
95+
if raw:
96+
try:
97+
return float(raw)
98+
except ValueError:
99+
print(f" ignoring {var}={raw!r}: not a number")
100+
return _DEFAULT_TIMEOUTS[command]
101+
102+
103+
def _qsv(api_key: str | None = None) -> AsyncQsv:
104+
"""The qsv client every profiling pass runs through.
105+
106+
The binary is ``$QSV_BIN`` if set, else the first of qsv, qsvmcp, qsvdp, qsvlite on
107+
``PATH``. Each run is killed (with its whole process group) once it passes its timeout.
108+
"""
109+
return AsyncQsv(llm_api_key=api_key)
110+
111+
112+
def _report_failure(command: str, exc: Exception) -> None:
113+
kind = getattr(exc, "kind", None)
114+
message = getattr(exc, "message", None) or str(exc)
115+
label = f"{type(exc).__name__}, {kind}" if kind else type(exc).__name__
116+
print(f" qsv {command} failed ({label}): {message[:500]}")
117+
118+
119+
async def _explain_missing_describegpt(qsv: AsyncQsv) -> None:
120+
"""After a describegpt failure, say so if the binary has no describegpt at all.
121+
122+
qsvlite and qsvdp lack it, and binary discovery falls back to them. Checked lazily so a
123+
successful run pays nothing for the probe.
124+
"""
125+
if qsv.binary in _describegpt_checked:
126+
return
127+
_describegpt_checked.add(qsv.binary)
128+
try:
129+
caps = await qsv.capabilities()
130+
except _QSV_FAILURES:
131+
return
132+
if caps.commands and not caps.has_command("describegpt"):
133+
print(
134+
f" {qsv.binary} ({caps.binary} {caps.version}) has no describegpt command; "
135+
"install the full qsv binary or point QSV_BIN at it"
136+
)
137+
138+
64139
async def run_qsv_describegpt(
65140
csv_path: Path,
66141
api_key: str,
@@ -71,58 +146,66 @@ async def run_qsv_describegpt(
71146
The API key goes to qsv in ``QSV_LLM_APIKEY``, not ``--api-key``: describegpt
72147
copies its command line into the descriptions it generates, and argv is also
73148
visible to other local users in ``ps``.
149+
150+
The output is re-serialised with ``json.dump(indent=2)`` rather than streamed to
151+
``output_path`` as qsv wrote it: ``qsv_dict.json`` is published and hashed, so its
152+
bytes must not change with qsv's own formatting.
74153
"""
75-
cmd = [
76-
"qsv", "describegpt", str(csv_path),
77-
"--all",
78-
"--format", "json",
79-
"--base-url", "https://openrouter.ai/api/v1",
80-
"--model", "google/gemini-2.5-flash-lite",
81-
]
82-
83-
proc = await asyncio.create_subprocess_exec(
84-
*cmd,
85-
stdout=asyncio.subprocess.PIPE,
86-
stderr=asyncio.subprocess.PIPE,
87-
env={**os.environ, "QSV_LLM_APIKEY": api_key},
88-
)
89-
stdout, stderr = await proc.communicate()
90-
91-
if proc.returncode != 0:
92-
print(f" qsv describegpt failed (exit {proc.returncode}): {stderr.decode()[:500]}")
154+
qsv: AsyncQsv | None = None
155+
try:
156+
qsv = _qsv(api_key)
157+
data = await qsv.describegpt(
158+
csv_path,
159+
"--all",
160+
"--format", "json",
161+
"--base-url", "https://openrouter.ai/api/v1",
162+
"--model", "google/gemini-2.5-flash-lite",
163+
timeout=_timeout("describegpt"),
164+
)
165+
except _QSV_FAILURES as e:
166+
_report_failure("describegpt", e)
167+
if qsv is not None and isinstance(e, QsvError) and not isinstance(e, QsvTimeout):
168+
await _explain_missing_describegpt(qsv)
93169
return None
94170

171+
output_path.parent.mkdir(parents=True, exist_ok=True)
172+
with open(output_path, "w") as f:
173+
json.dump(data, f, indent=2)
174+
return data
175+
176+
177+
async def _run_qsv_csv(
178+
command: str, csv_path: Path, output_path: Path, *args: str
179+
) -> list[dict[str, str]] | None:
180+
"""Stream ``qsv <command>``'s CSV output to ``output_path`` and return its rows.
181+
182+
qsv writes to a sibling ``.part`` file that replaces ``output_path`` only once the run
183+
succeeds and parses, so a failed or killed run leaves the previous output in place
184+
rather than a truncated file.
185+
"""
186+
part = output_path.with_name(output_path.name + ".part")
95187
try:
96-
data = json.loads(stdout.decode())
97-
output_path.parent.mkdir(parents=True, exist_ok=True)
98-
with open(output_path, "w") as f:
99-
json.dump(data, f, indent=2)
100-
return data
101-
except json.JSONDecodeError as e:
102-
print(f" qsv output not valid JSON: {e}")
188+
res = await _qsv().run(
189+
command, csv_path, *args, stdout_path=part, timeout=_timeout(command)
190+
)
191+
rows = res.csv_rows()
192+
os.replace(part, output_path)
193+
return rows
194+
except _QSV_FAILURES as e:
195+
_report_failure(command, e)
103196
return None
197+
finally:
198+
part.unlink(missing_ok=True)
104199

105200

106201
async def run_qsv_stats(csv_path: Path, output_path: Path) -> dict[str, dict] | None:
107202
"""Run qsv stats and return per-column stats keyed by column name."""
108-
proc = await asyncio.create_subprocess_exec(
109-
"qsv", "stats", str(csv_path), "--everything",
110-
stdout=asyncio.subprocess.PIPE,
111-
stderr=asyncio.subprocess.PIPE,
112-
)
113-
stdout, stderr = await proc.communicate()
114-
115-
if proc.returncode != 0:
116-
print(f" qsv stats failed: {stderr.decode()[:300]}")
203+
rows = await _run_qsv_csv("stats", csv_path, output_path, "--everything")
204+
if rows is None:
117205
return None
118206

119-
raw = stdout.decode()
120-
output_path.parent.mkdir(parents=True, exist_ok=True)
121-
output_path.write_text(raw)
122-
123-
reader = csv.DictReader(StringIO(raw))
124207
stats_by_col: dict[str, dict] = {}
125-
for row in reader:
208+
for row in rows:
126209
col_name = row.get("field", "")
127210
if col_name:
128211
stats_by_col[col_name] = {k: v for k, v in row.items() if k != "field"}
@@ -131,24 +214,12 @@ async def run_qsv_stats(csv_path: Path, output_path: Path) -> dict[str, dict] |
131214

132215
async def run_qsv_frequency(csv_path: Path, output_path: Path) -> dict[str, list[dict]] | None:
133216
"""Run qsv frequency and return top values per column."""
134-
proc = await asyncio.create_subprocess_exec(
135-
"qsv", "frequency", str(csv_path), "--limit", "20",
136-
stdout=asyncio.subprocess.PIPE,
137-
stderr=asyncio.subprocess.PIPE,
138-
)
139-
stdout, stderr = await proc.communicate()
140-
141-
if proc.returncode != 0:
142-
print(f" qsv frequency failed: {stderr.decode()[:300]}")
217+
rows = await _run_qsv_csv("frequency", csv_path, output_path, "--limit", "20")
218+
if rows is None:
143219
return None
144220

145-
raw = stdout.decode()
146-
output_path.parent.mkdir(parents=True, exist_ok=True)
147-
output_path.write_text(raw)
148-
149-
reader = csv.DictReader(StringIO(raw))
150221
freq_by_col: dict[str, list[dict]] = {}
151-
for row in reader:
222+
for row in rows:
152223
col_name = row.get("field", "")
153224
if col_name:
154225
freq_by_col.setdefault(col_name, []).append({
@@ -158,6 +229,15 @@ async def run_qsv_frequency(csv_path: Path, output_path: Path) -> dict[str, list
158229
return freq_by_col
159230

160231

232+
async def run_qsv_count(csv_path: Path) -> int | None:
233+
"""Exact row count of ``csv_path``, or ``None`` if qsv could not count it."""
234+
try:
235+
return await _qsv().count(csv_path, timeout=_timeout("count"))
236+
except _QSV_FAILURES as e:
237+
_report_failure("count", e)
238+
return None
239+
240+
161241
def _extract_qsv_fields(qsv_data: dict) -> list[dict]:
162242
"""Extract field list from qsv describegpt JSON, handling both key casings."""
163243
for key in ("Dictionary", "dictionary"):

‎src/data_concierge/gateway/onboarding_jobs.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,6 @@
5858
import json
5959
import os
6060
import re
61-
import shutil
6261
import sys
6362
import tempfile
6463
import threading
@@ -69,6 +68,8 @@
6968
from pathlib import Path
7069
from typing import Any
7170

71+
from qsv_client import QsvNotFound, find_qsv
72+
7273
from data_concierge.core.logging import get_logger
7374
from data_concierge.data_layer.storage import storage
7475

@@ -191,7 +192,9 @@ def runtime_warnings() -> list[str]:
191192
"The onboarding scripts are not present in this deployment, so jobs "
192193
"cannot be started. The container image must copy scripts/ to /app/scripts."
193194
)
194-
if shutil.which("qsv") is None:
195+
try:
196+
find_qsv()
197+
except QsvNotFound:
195198
warnings.append(
196199
"qsv is not installed on this host. Downloads will succeed but every "
197200
"profiling pass (stats, frequency, describegpt, count) will be skipped, "

0 commit comments

Comments
 (0)