1919``count``
2020 Exact row count.
2121
22+ Every pass runs through `qsv-client <https://github.com/dathere/qsv-client>`_, which
23+ kills a run that exceeds its timeout (see ``_DEFAULT_TIMEOUTS``; each one can be
24+ overridden with ``VERIKAN_QSV_<PASS>_TIMEOUT``). A failed, timed-out, or missing
25+ qsv makes the pass return ``None``, and the onboarding scripts carry on without it.
26+
2227:func:`merge_columns` folds all of those, plus the portal's own field
2328metadata when it has any, into the single column list that
2429``data_layer.onboard_index`` searches and that
2732
2833from __future__ import annotations
2934
30- import asyncio
3135import csv
3236import json
3337import os
3438from datetime import UTC , datetime
35- from io import StringIO
3639from pathlib import Path
3740from typing import Any
3841
3942import httpx
43+ from qsv_client import (
44+ AsyncQsv ,
45+ QsvError ,
46+ QsvNotFound ,
47+ QsvTimeout ,
48+ QsvVersionError ,
49+ )
4050
4151from data_concierge .data_layer .storage import storage
4252
@@ -61,6 +71,71 @@ async def download_csv(url: str, dest: Path) -> int:
6171 return size
6272
6373
74+ # Default per-pass timeouts in seconds. ``VERIKAN_QSV_<PASS>_TIMEOUT`` overrides one,
75+ # e.g. ``VERIKAN_QSV_STATS_TIMEOUT=1800`` for a portal with very large CSVs.
76+ _DEFAULT_TIMEOUTS : dict [str , float ] = {
77+ "describegpt" : 900.0 , # several LLM round trips
78+ "stats" : 900.0 ,
79+ "frequency" : 600.0 ,
80+ "count" : 120.0 ,
81+ }
82+
83+ # A failed run, a missing or too-old binary, or output that does not parse. QsvNotFound and
84+ # QsvVersionError are not QsvError subclasses, and the client raises QsvNotFound from its
85+ # constructor.
86+ _QSV_FAILURES = (QsvError , QsvNotFound , QsvVersionError , ValueError )
87+
88+ # Binaries whose missing describegpt has already been reported, so a portal run warns once.
89+ _describegpt_checked : set [str ] = set ()
90+
91+
92+ def _timeout (command : str ) -> float :
93+ var = f"VERIKAN_QSV_{ command .upper ()} _TIMEOUT"
94+ raw = os .environ .get (var )
95+ if raw :
96+ try :
97+ return float (raw )
98+ except ValueError :
99+ print (f" ignoring { var } ={ raw !r} : not a number" )
100+ return _DEFAULT_TIMEOUTS [command ]
101+
102+
103+ def _qsv (api_key : str | None = None ) -> AsyncQsv :
104+ """The qsv client every profiling pass runs through.
105+
106+ The binary is ``$QSV_BIN`` if set, else the first of qsv, qsvmcp, qsvdp, qsvlite on
107+ ``PATH``. Each run is killed (with its whole process group) once it passes its timeout.
108+ """
109+ return AsyncQsv (llm_api_key = api_key )
110+
111+
112+ def _report_failure (command : str , exc : Exception ) -> None :
113+ kind = getattr (exc , "kind" , None )
114+ message = getattr (exc , "message" , None ) or str (exc )
115+ label = f"{ type (exc ).__name__ } , { kind } " if kind else type (exc ).__name__
116+ print (f" qsv { command } failed ({ label } ): { message [:500 ]} " )
117+
118+
119+ async def _explain_missing_describegpt (qsv : AsyncQsv ) -> None :
120+ """After a describegpt failure, say so if the binary has no describegpt at all.
121+
122+ qsvlite and qsvdp lack it, and binary discovery falls back to them. Checked lazily so a
123+ successful run pays nothing for the probe.
124+ """
125+ if qsv .binary in _describegpt_checked :
126+ return
127+ _describegpt_checked .add (qsv .binary )
128+ try :
129+ caps = await qsv .capabilities ()
130+ except _QSV_FAILURES :
131+ return
132+ if caps .commands and not caps .has_command ("describegpt" ):
133+ print (
134+ f" { qsv .binary } ({ caps .binary } { caps .version } ) has no describegpt command; "
135+ "install the full qsv binary or point QSV_BIN at it"
136+ )
137+
138+
64139async def run_qsv_describegpt (
65140 csv_path : Path ,
66141 api_key : str ,
@@ -71,58 +146,66 @@ async def run_qsv_describegpt(
71146 The API key goes to qsv in ``QSV_LLM_APIKEY``, not ``--api-key``: describegpt
72147 copies its command line into the descriptions it generates, and argv is also
73148 visible to other local users in ``ps``.
149+
150+ The output is re-serialised with ``json.dump(indent=2)`` rather than streamed to
151+ ``output_path`` as qsv wrote it: ``qsv_dict.json`` is published and hashed, so its
152+ bytes must not change with qsv's own formatting.
74153 """
75- cmd = [
76- "qsv" , "describegpt" , str (csv_path ),
77- "--all" ,
78- "--format" , "json" ,
79- "--base-url" , "https://openrouter.ai/api/v1" ,
80- "--model" , "google/gemini-2.5-flash-lite" ,
81- ]
82-
83- proc = await asyncio .create_subprocess_exec (
84- * cmd ,
85- stdout = asyncio .subprocess .PIPE ,
86- stderr = asyncio .subprocess .PIPE ,
87- env = {** os .environ , "QSV_LLM_APIKEY" : api_key },
88- )
89- stdout , stderr = await proc .communicate ()
90-
91- if proc .returncode != 0 :
92- print (f" qsv describegpt failed (exit { proc .returncode } ): { stderr .decode ()[:500 ]} " )
154+ qsv : AsyncQsv | None = None
155+ try :
156+ qsv = _qsv (api_key )
157+ data = await qsv .describegpt (
158+ csv_path ,
159+ "--all" ,
160+ "--format" , "json" ,
161+ "--base-url" , "https://openrouter.ai/api/v1" ,
162+ "--model" , "google/gemini-2.5-flash-lite" ,
163+ timeout = _timeout ("describegpt" ),
164+ )
165+ except _QSV_FAILURES as e :
166+ _report_failure ("describegpt" , e )
167+ if qsv is not None and isinstance (e , QsvError ) and not isinstance (e , QsvTimeout ):
168+ await _explain_missing_describegpt (qsv )
93169 return None
94170
171+ output_path .parent .mkdir (parents = True , exist_ok = True )
172+ with open (output_path , "w" ) as f :
173+ json .dump (data , f , indent = 2 )
174+ return data
175+
176+
177+ async def _run_qsv_csv (
178+ command : str , csv_path : Path , output_path : Path , * args : str
179+ ) -> list [dict [str , str ]] | None :
180+ """Stream ``qsv <command>``'s CSV output to ``output_path`` and return its rows.
181+
182+ qsv writes to a sibling ``.part`` file that replaces ``output_path`` only once the run
183+ succeeds and parses, so a failed or killed run leaves the previous output in place
184+ rather than a truncated file.
185+ """
186+ part = output_path .with_name (output_path .name + ".part" )
95187 try :
96- data = json .loads (stdout .decode ())
97- output_path .parent .mkdir (parents = True , exist_ok = True )
98- with open (output_path , "w" ) as f :
99- json .dump (data , f , indent = 2 )
100- return data
101- except json .JSONDecodeError as e :
102- print (f" qsv output not valid JSON: { e } " )
188+ res = await _qsv ().run (
189+ command , csv_path , * args , stdout_path = part , timeout = _timeout (command )
190+ )
191+ rows = res .csv_rows ()
192+ os .replace (part , output_path )
193+ return rows
194+ except _QSV_FAILURES as e :
195+ _report_failure (command , e )
103196 return None
197+ finally :
198+ part .unlink (missing_ok = True )
104199
105200
106201async def run_qsv_stats (csv_path : Path , output_path : Path ) -> dict [str , dict ] | None :
107202 """Run qsv stats and return per-column stats keyed by column name."""
108- proc = await asyncio .create_subprocess_exec (
109- "qsv" , "stats" , str (csv_path ), "--everything" ,
110- stdout = asyncio .subprocess .PIPE ,
111- stderr = asyncio .subprocess .PIPE ,
112- )
113- stdout , stderr = await proc .communicate ()
114-
115- if proc .returncode != 0 :
116- print (f" qsv stats failed: { stderr .decode ()[:300 ]} " )
203+ rows = await _run_qsv_csv ("stats" , csv_path , output_path , "--everything" )
204+ if rows is None :
117205 return None
118206
119- raw = stdout .decode ()
120- output_path .parent .mkdir (parents = True , exist_ok = True )
121- output_path .write_text (raw )
122-
123- reader = csv .DictReader (StringIO (raw ))
124207 stats_by_col : dict [str , dict ] = {}
125- for row in reader :
208+ for row in rows :
126209 col_name = row .get ("field" , "" )
127210 if col_name :
128211 stats_by_col [col_name ] = {k : v for k , v in row .items () if k != "field" }
@@ -131,24 +214,12 @@ async def run_qsv_stats(csv_path: Path, output_path: Path) -> dict[str, dict] |
131214
132215async def run_qsv_frequency (csv_path : Path , output_path : Path ) -> dict [str , list [dict ]] | None :
133216 """Run qsv frequency and return top values per column."""
134- proc = await asyncio .create_subprocess_exec (
135- "qsv" , "frequency" , str (csv_path ), "--limit" , "20" ,
136- stdout = asyncio .subprocess .PIPE ,
137- stderr = asyncio .subprocess .PIPE ,
138- )
139- stdout , stderr = await proc .communicate ()
140-
141- if proc .returncode != 0 :
142- print (f" qsv frequency failed: { stderr .decode ()[:300 ]} " )
217+ rows = await _run_qsv_csv ("frequency" , csv_path , output_path , "--limit" , "20" )
218+ if rows is None :
143219 return None
144220
145- raw = stdout .decode ()
146- output_path .parent .mkdir (parents = True , exist_ok = True )
147- output_path .write_text (raw )
148-
149- reader = csv .DictReader (StringIO (raw ))
150221 freq_by_col : dict [str , list [dict ]] = {}
151- for row in reader :
222+ for row in rows :
152223 col_name = row .get ("field" , "" )
153224 if col_name :
154225 freq_by_col .setdefault (col_name , []).append ({
@@ -158,6 +229,15 @@ async def run_qsv_frequency(csv_path: Path, output_path: Path) -> dict[str, list
158229 return freq_by_col
159230
160231
232+ async def run_qsv_count (csv_path : Path ) -> int | None :
233+ """Exact row count of ``csv_path``, or ``None`` if qsv could not count it."""
234+ try :
235+ return await _qsv ().count (csv_path , timeout = _timeout ("count" ))
236+ except _QSV_FAILURES as e :
237+ _report_failure ("count" , e )
238+ return None
239+
240+
161241def _extract_qsv_fields (qsv_data : dict ) -> list [dict ]:
162242 """Extract field list from qsv describegpt JSON, handling both key casings."""
163243 for key in ("Dictionary" , "dictionary" ):
0 commit comments