diff --git a/builders/server/core/api/routes.py b/builders/server/core/api/routes.py index 9a2f883..b55ed16 100644 --- a/builders/server/core/api/routes.py +++ b/builders/server/core/api/routes.py @@ -1,9 +1,12 @@ from datetime import datetime import structlog -from fastapi import APIRouter, HTTPException, Query +from fastapi import APIRouter, Depends, HTTPException, Query from fastapi.responses import JSONResponse +from pydantic import BaseModel, ConfigDict, Field +from core.auth import verify_api_key +from core.github import GitHubError from core.service.builder import ( DatasetNotFoundError, NoDataInRangeError, @@ -13,6 +16,13 @@ get_data, ) from core.service.catalog import list_datasets +from core.service.proposals import ( + DatasetProposal, + InvalidProposalError, + ProposalConflictError, + ProposedDependency, + propose_dataset, +) from core.utils.semver import SemVer logger = structlog.get_logger() @@ -45,6 +55,89 @@ def datasets_list() -> dict: } +class DependencyIn(BaseModel): + name: str + version: str + lookback: str | None = None + + +class DatasetProposalIn(BaseModel): + model_config = ConfigDict(populate_by_name=True) + + name: str + version: str + calendar: str + granularity: str + start_date: str + builder_script: str + author_name: str + team: str + discord_user: str + description: str + # 'schema' shadows a deprecated BaseModel attr, so alias it + data_schema: dict[str, str] = Field(alias="schema") + dependencies: list[DependencyIn] = Field(default_factory=list) + env_vars: bool = False + requirements_txt: str | None = None + env_template: str | None = None + + +@router.post("/datasets") +def datasets_propose( + payload: DatasetProposalIn, team: str = Depends(verify_api_key) +) -> dict: + """Propose a new dataset: validate the submission and open a GitHub PR. + + Nothing is written to the server; the dataset goes live only after the + PR is reviewed, merged, and the server restarts. + """ + structlog.contextvars.bind_contextvars( + dataset_name=payload.name, version=payload.version + ) + proposal = DatasetProposal( + name=payload.name, + version=payload.version, + calendar=payload.calendar, + granularity=payload.granularity, + start_date=payload.start_date, + schema=payload.data_schema, + builder_script=payload.builder_script, + author_name=payload.author_name, + team=payload.team, + discord_user=payload.discord_user, + description=payload.description, + dependencies=[ + ProposedDependency(name=d.name, version=d.version, lookback=d.lookback) + for d in payload.dependencies + ], + env_vars=payload.env_vars, + requirements_txt=payload.requirements_txt, + env_template=payload.env_template, + ) + + try: + result = propose_dataset(proposal, requested_by=team) + except InvalidProposalError as e: + logger.warning("proposal rejected", error=str(e)) + raise HTTPException(status_code=400, detail=str(e)) from e + except ProposalConflictError as e: + logger.warning("proposal conflict", error=str(e)) + raise HTTPException(status_code=409, detail=str(e)) from e + except GitHubError as e: + logger.exception("proposal github call failed") + raise HTTPException(status_code=502, detail=f"github error: {e.message}") from e + except Exception as e: + logger.exception("proposal failed") + raise HTTPException(status_code=500, detail=str(e)) from e + + return { + "dataset_name": payload.name, + "dataset_version": payload.version, + "pr_url": result.pr_url, + "branch": result.branch, + } + + @router.post("/build/{dataset_name}/{dataset_version}") def build( dataset_name: str, diff --git a/builders/server/core/service/proposals.py b/builders/server/core/service/proposals.py new file mode 100644 index 0000000..0605223 --- /dev/null +++ b/builders/server/core/service/proposals.py @@ -0,0 +1,419 @@ +"""Dataset proposal service: validate a submission and open a GitHub PR. + +Proposed datasets are never written to the server's scripts directory — code +review stays the gate for anything that will execute on the server. This +module turns a submission into the exact files that will land in +builders/scripts///, re-validates the generated config bytes +with the same checks the server runs at startup, and opens the PR via the +github client. +""" + +import ast +import json +import os +import re +import subprocess +import sys +import tempfile +import tomllib +from dataclasses import dataclass, field +from pathlib import Path +from typing import Protocol + +import structlog + +from core.github import BranchAlreadyExistsError, GitHubClient +from core.runtime import registry +from core.runtime.config import DatasetConfig, normalize_config, validate_config +from core.utils.semver import SemVer + +logger = structlog.get_logger() + +# dataset names become directory names, branch names, and toml keys +_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9_-]*$") +_BARE_TOML_KEY_RE = re.compile(r"^[A-Za-z0-9_-]+$") + +SCRIPTS_PREFIX = "builders/scripts" + +# keep in sync with [tool.ruff.lint] select in the repo-root pyproject.toml: +# proposal prs must pass the same ci lint that runs over builders/scripts/** +RUFF_SELECT = "B,E,F,I,PIE,SIM,T20,UP" +RUFF_TIMEOUT_SECONDS = 30 + +# github logins asked to review every proposal pr (comma-separated env override) +DEFAULT_REVIEWERS = "Blackgaurd,Scr4tch587" + + +class InvalidProposalError(ValueError): + """Submission failed validation; safe to show the message to the caller.""" + + +class ProposalConflictError(Exception): + """The dataset or its proposal branch already exists.""" + + +@dataclass(frozen=True) +class ProposedDependency: + name: str + version: str + lookback: str | None = None + + +@dataclass(frozen=True) +class DatasetProposal: + name: str + version: str + calendar: str + granularity: str + start_date: str + schema: dict[str, str] + builder_script: str + # who is proposing and why -- surfaced in the pr body for reviewers + author_name: str + team: str + discord_user: str + description: str + dependencies: list[ProposedDependency] = field(default_factory=list) + env_vars: bool = False + requirements_txt: str | None = None + env_template: str | None = None + + +@dataclass(frozen=True) +class ProposalResult: + pr_url: str + branch: str + + +class PullRequestOpener(Protocol): + """The slice of GitHubClient the proposal service needs (test-fakeable).""" + + def open_pr_with_files( + self, + branch: str, + base: str, + title: str, + body: str, + commit_message: str, + files: dict[str, str], + reviewers: list[str] | None = None, + ) -> str: ... + + +def _toml_str(value: str) -> str: + """quote a string as a toml basic string + + (json escaping is a valid subset). + """ + return json.dumps(value) + + +def _toml_key(key: str) -> str: + return key if _BARE_TOML_KEY_RE.fullmatch(key) else json.dumps(key) + + +def generate_config_toml(proposal: DatasetProposal) -> str: + """Render the canonical config.toml for a proposal. + + The returned text is exactly what gets committed; callers re-parse and + re-validate these bytes so the PR can never contain a config that fails + server startup. + """ + lines = [ + f"name = {_toml_str(proposal.name)}", + f"version = {_toml_str(proposal.version)}", + 'builder = "builder.py"', + f"calendar = {_toml_str(proposal.calendar)}", + f"granularity = {_toml_str(proposal.granularity)}", + f"start-date = {_toml_str(proposal.start_date)}", + ] + if proposal.env_vars: + lines.append("env-vars = true") + + lines += ["", "[schema]"] + lines += [ + f"{_toml_key(key)} = {_toml_str(type_)}" + for key, type_ in proposal.schema.items() + ] + + if proposal.dependencies: + lines += ["", "[dependencies]"] + for dep in proposal.dependencies: + if dep.lookback: + lines.append( + f"{_toml_key(dep.name)} = {{ version = {_toml_str(dep.version)}," + f" lookback = {_toml_str(dep.lookback)} }}" + ) + else: + lines.append(f"{_toml_key(dep.name)} = {_toml_str(dep.version)}") + + return "\n".join(lines) + "\n" + + +def _validate_builder_script(script: str) -> None: + """Same convention the ci gate enforces: + + parseable, top-level build(dependencies, timestamp). + """ + try: + tree = ast.parse(script) + except SyntaxError as e: + raise InvalidProposalError(f"builder script has a syntax error: {e}") from e + + build_fns = [ + node + for node in tree.body + if isinstance(node, ast.FunctionDef) and node.name == "build" + ] + if not build_fns: + raise InvalidProposalError( + "builder script must define a top-level build() function" + ) + args = build_fns[0].args + positional = args.posonlyargs + args.args + if len(positional) != 2: + raise InvalidProposalError( + "build() must take exactly two arguments (dependencies, timestamp), " + f"got {[a.arg for a in positional]}" + ) + + +def _ruff_output(result: subprocess.CompletedProcess[str]) -> str: + """Combines a ruff run's stdout and stderr into one diagnostic string. + + Violations land on stdout, but a ruff-internal failure (bad config, + unreadable file) reports only on stderr, so reading either stream alone can + produce an empty — and useless — error message. + """ + parts = [part for part in (result.stdout.strip(), result.stderr.strip()) if part] + if not parts: + # ruff produced nothing at all, so at least report the exit code + return f"ruff exited with code {result.returncode} and no output" + return "\n".join(parts) + + +def _lint_builder_script(script: str) -> str: + """Autofix + format the script with ruff so the proposal pr passes repo ci. + + Returns the cleaned script. Raises InvalidProposalError when violations + remain after autofix (the message includes ruff's output). + """ + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "builder.py" + path.write_text(script) + common = [sys.executable, "-m", "ruff"] + check = subprocess.run( + [ + *common, + "check", + "--isolated", + "--select", + RUFF_SELECT, + "--fix", + str(path), + ], + capture_output=True, + text=True, + timeout=RUFF_TIMEOUT_SECONDS, + ) + if check.returncode != 0: + raise InvalidProposalError( + f"builder script fails lint:\n{_ruff_output(check)}" + ) + fmt = subprocess.run( + [*common, "format", "--isolated", str(path)], + capture_output=True, + text=True, + timeout=RUFF_TIMEOUT_SECONDS, + ) + if fmt.returncode != 0: + raise InvalidProposalError( + f"builder script fails formatting:\n{_ruff_output(fmt)}" + ) + return path.read_text() + + +def _validate_against_registry(cfg: DatasetConfig) -> None: + """Cross-checks against currently registered datasets. + + A new dataset is a leaf no existing config references, so it cannot + introduce a cycle — only dep existence, granularity, and start-date + ordering need checking (mirrors registry startup validation). + """ + for dep_name, dep_info in cfg.dependencies.items(): + try: + dep_cfg = registry.get_config(dep_name, dep_info.version) + except ValueError as e: + raise InvalidProposalError( + f"dependency {dep_name}/{dep_info.version} is not a known dataset" + ) from e + if cfg.granularity < dep_cfg.granularity: + raise InvalidProposalError( + f"granularity is finer than dependency {dep_name}/{dep_info.version}" + ) + if cfg.start_date < dep_cfg.start_date: + raise InvalidProposalError( + f"start-date is before dependency {dep_name}/{dep_info.version}'s " + f"start-date ({dep_cfg.start_date.date()})" + ) + + +def _build_pr_body( + proposal: DatasetProposal, requested_by: str, dataset_dir: str +) -> str: + schema_lines = "\n".join( + f"- `{key}`: `{type_}`" for key, type_ in proposal.schema.items() + ) + if proposal.dependencies: + dep_lines = "\n".join( + f"- `{dep.name}/{dep.version}`" + + (f" (lookback `{dep.lookback}`)" if dep.lookback else "") + for dep in proposal.dependencies + ) + else: + dep_lines = "- none (root dataset)" + + body = f"""## Dataset proposal: `{proposal.name}/{proposal.version}` + +{proposal.description.strip()} + +**Proposed by:** {proposal.author_name.strip()} · team **{proposal.team.strip()}** \ +· discord `{proposal.discord_user.strip()}` · api key label `{requested_by}` + +| field | value | +|-------|-------| +| calendar | `{proposal.calendar}` | +| granularity | `{proposal.granularity}` | +| start-date | `{proposal.start_date}` | +| env-vars | `{str(proposal.env_vars).lower()}` | + +**Schema** +{schema_lines} + +**Dependencies** +{dep_lines} + +### Review checklist + +- [ ] builder logic reviewed — this code will run on the datastream server +- [ ] schema and dependencies make sense for the data +""" + if proposal.requirements_txt: + body += "- [ ] `requirements.txt` packages reviewed\n" + if proposal.env_vars: + body += ( + f"- [ ] **before first build**: place the real `.env` at " + f"`{dataset_dir}/.env` on the server " + f"(only `.env.template` is committed; see it for required vars)\n" + ) + return body + + +def propose_dataset( + proposal: DatasetProposal, + requested_by: str, + client: PullRequestOpener | None = None, +) -> ProposalResult: + """Validate a proposal and open a PR adding the dataset directory. + + Raises InvalidProposalError (bad submission), ProposalConflictError + (dataset or proposal branch already exists), or GitHubError (github + unreachable / misconfigured). + """ + if not _NAME_RE.fullmatch(proposal.name): + raise InvalidProposalError( + "dataset name must be lowercase alphanumeric with '-' or '_' " + "(it becomes a directory and branch name)" + ) + for field_name, value in ( + ("author_name", proposal.author_name), + ("team", proposal.team), + ("discord_user", proposal.discord_user), + ("description", proposal.description), + ): + if not value.strip(): + raise InvalidProposalError(f"{field_name} must not be empty") + try: + version = SemVer.parse(proposal.version) + except ValueError as e: + raise InvalidProposalError(f"invalid version: {e}") from e + + try: + registry.get_config(proposal.name, version) + except ValueError: + pass + else: + raise ProposalConflictError( + f"dataset {proposal.name}/{proposal.version} already exists" + ) + + # validate the exact bytes that will be committed, with the same code + # paths the server runs at startup + config_toml = generate_config_toml(proposal) + try: + raw = tomllib.loads(config_toml) + validate_config(raw, proposal.name, version) + normalize_config(raw) + cfg = DatasetConfig.from_raw(raw) + except ValueError as e: + raise InvalidProposalError(str(e)) from e + + _validate_against_registry(cfg) + _validate_builder_script(proposal.builder_script) + # commit the linted/formatted script so the pr passes repo ci + builder_script = _lint_builder_script(proposal.builder_script) + + dataset_dir = f"{SCRIPTS_PREFIX}/{proposal.name}/{proposal.version}" + files = { + f"{dataset_dir}/config.toml": config_toml, + f"{dataset_dir}/builder.py": _ensure_trailing_newline(builder_script), + } + if proposal.requirements_txt and proposal.requirements_txt.strip(): + files[f"{dataset_dir}/requirements.txt"] = _ensure_trailing_newline( + proposal.requirements_txt + ) + if proposal.env_template and proposal.env_template.strip(): + files[f"{dataset_dir}/.env.template"] = _ensure_trailing_newline( + proposal.env_template + ) + + branch = f"add-dataset/{proposal.name}-{proposal.version}" + title = f"feat: add dataset {proposal.name}/{proposal.version}" + body = _build_pr_body(proposal, requested_by, dataset_dir) + + github: PullRequestOpener = ( + client if client is not None else GitHubClient.from_env() + ) + reviewers = [ + login.strip() + for login in os.environ.get("GITHUB_REVIEWERS", DEFAULT_REVIEWERS).split(",") + if login.strip() + ] + try: + pr_url = github.open_pr_with_files( + branch=branch, + base="main", + title=title, + body=body, + commit_message=title, + files=files, + reviewers=reviewers, + ) + except BranchAlreadyExistsError as e: + raise ProposalConflictError( + f"a proposal for {proposal.name}/{proposal.version} is already open " + f"(branch '{branch}' exists)" + ) from e + + logger.info( + "dataset proposal submitted", + dataset=proposal.name, + version=proposal.version, + requested_by=requested_by, + pr_url=pr_url, + ) + return ProposalResult(pr_url=pr_url, branch=branch) + + +def _ensure_trailing_newline(text: str) -> str: + return text if text.endswith("\n") else text + "\n" diff --git a/builders/server/pyproject.toml b/builders/server/pyproject.toml index b264677..f46cb5d 100644 --- a/builders/server/pyproject.toml +++ b/builders/server/pyproject.toml @@ -12,6 +12,7 @@ dependencies = [ "exchange-calendars>=4.13", "python-dotenv>=1.0.0", "structlog>=24.0.0", + "ruff>=0.15.21", ] [dependency-groups] diff --git a/builders/server/tests/core/api/test_proposal_route.py b/builders/server/tests/core/api/test_proposal_route.py new file mode 100644 index 0000000..e0e30e3 --- /dev/null +++ b/builders/server/tests/core/api/test_proposal_route.py @@ -0,0 +1,91 @@ +import core.api.routes as routes +import pytest +from core.github.client import GitHubError +from core.service.proposals import ( + InvalidProposalError, + ProposalConflictError, + ProposalResult, +) +from fastapi.testclient import TestClient +from main import app + +client: TestClient = TestClient(app) + +PAYLOAD = { + "name": "my-dataset", + "version": "0.1.0", + "calendar": "everyday", + "granularity": "1d", + "start_date": "2022-01-01", + "schema": {"price": "float"}, + "builder_script": "def build(deps, ts):\n return []\n", + "author_name": "Kai Zhang", + "team": "quant", + "discord_user": "kai#1234", + "description": "test data", +} + + +def _propose_stub(result=None, error=None): + def stub(proposal, requested_by, client=None): + if error is not None: + raise error + return result + + return stub + + +def test_propose_returns_pr_url(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + routes, + "propose_dataset", + _propose_stub( + result=ProposalResult( + pr_url="https://github.com/acme/data/pull/42", + branch="add-dataset/my-dataset-0.1.0", + ) + ), + ) + res = client.post("/api/v1/datasets", json=PAYLOAD) + assert res.status_code == 200 + body = res.json() + assert body["pr_url"] == "https://github.com/acme/data/pull/42" + assert body["branch"] == "add-dataset/my-dataset-0.1.0" + assert body["dataset_name"] == "my-dataset" + + +def test_invalid_proposal_maps_to_400(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + routes, + "propose_dataset", + _propose_stub(error=InvalidProposalError("bad schema")), + ) + res = client.post("/api/v1/datasets", json=PAYLOAD) + assert res.status_code == 400 + assert "bad schema" in res.json()["detail"] + + +def test_conflict_maps_to_409(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + routes, + "propose_dataset", + _propose_stub(error=ProposalConflictError("already exists")), + ) + res = client.post("/api/v1/datasets", json=PAYLOAD) + assert res.status_code == 409 + + +def test_github_failure_maps_to_502(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + routes, + "propose_dataset", + _propose_stub(error=GitHubError(500, "boom")), + ) + res = client.post("/api/v1/datasets", json=PAYLOAD) + assert res.status_code == 502 + assert "github error" in res.json()["detail"] + + +def test_missing_fields_rejected_by_validation() -> None: + res = client.post("/api/v1/datasets", json={"name": "x"}) + assert res.status_code == 422 diff --git a/builders/server/tests/core/service/test_proposals.py b/builders/server/tests/core/service/test_proposals.py new file mode 100644 index 0000000..2fd1f9e --- /dev/null +++ b/builders/server/tests/core/service/test_proposals.py @@ -0,0 +1,349 @@ +import subprocess +import tomllib +from pathlib import Path +from typing import Any + +import core.runtime.registry as registry +import core.service.proposals as proposals +import pytest +from core.github.client import BranchAlreadyExistsError +from core.service.proposals import ( + DatasetProposal, + InvalidProposalError, + ProposalConflictError, + ProposedDependency, + generate_config_toml, + propose_dataset, +) + +_DEP_CONFIG = """\ +name = "mock-dep" +version = "0.1.0" +granularity = "1d" +start-date = "2021-06-01" +calendar = "everyday" + +[schema] +price = "int" +""" + +VALID_BUILDER = """\ +from datetime import datetime + + +def build(dependencies, timestamp: datetime) -> list[dict]: + return [{"ticker": "AAPL", "price": 1}] +""" + + +class FakeGitHub: + """stands in for GitHubClient; records the pr call or raises.""" + + def __init__(self, error: Exception | None = None): + self.error = error + self.calls: list[dict[str, Any]] = [] + + def open_pr_with_files( + self, + branch: str, + base: str, + title: str, + body: str, + commit_message: str, + files: dict[str, str], + reviewers: list[str] | None = None, + ) -> str: + if self.error is not None: + raise self.error + self.calls.append( + { + "branch": branch, + "base": base, + "title": title, + "body": body, + "commit_message": commit_message, + "files": files, + "reviewers": reviewers, + } + ) + return "https://github.com/acme/data/pull/42" + + +@pytest.fixture(autouse=True) +def _registry_with_dep(tmp_path: Path): + """populate the registry with mock-dep/0.1.0; reset afterwards.""" + dep_dir = tmp_path / "mock-dep" / "0.1.0" + dep_dir.mkdir(parents=True) + (dep_dir / "config.toml").write_text(_DEP_CONFIG) + registry.load_all_configs(tmp_path) + yield + registry._CONFIG_REGISTRY = {} + + +def _proposal(**overrides: Any) -> DatasetProposal: + defaults: dict[str, Any] = dict( + name="my-dataset", + version="0.1.0", + calendar="everyday", + granularity="1d", + start_date="2022-01-01", + schema={"ticker": "str", "price": "float"}, + builder_script=VALID_BUILDER, + author_name="Kai Zhang", + team="quant", + discord_user="kai#1234", + description="daily test data for the proposal flow", + ) + defaults.update(overrides) + return DatasetProposal(**defaults) + + +def test_happy_path_opens_pr_with_files() -> None: + github = FakeGitHub() + result = propose_dataset(_proposal(), requested_by="team-a", client=github) + + assert result.pr_url == "https://github.com/acme/data/pull/42" + assert result.branch == "add-dataset/my-dataset-0.1.0" + + call = github.calls[0] + assert call["base"] == "main" + assert call["title"] == "feat: add dataset my-dataset/0.1.0" + files = call["files"] + assert set(files) == { + "builders/scripts/my-dataset/0.1.0/config.toml", + "builders/scripts/my-dataset/0.1.0/builder.py", + } + assert "team-a" in call["body"] + # proposer identity and purpose are surfaced for reviewers + assert "Kai Zhang" in call["body"] + assert "quant" in call["body"] + assert "kai#1234" in call["body"] + assert "daily test data for the proposal flow" in call["body"] + + # the committed config parses and carries the submitted fields + raw = tomllib.loads(files["builders/scripts/my-dataset/0.1.0/config.toml"]) + assert raw["name"] == "my-dataset" + assert raw["schema"] == {"ticker": "str", "price": "float"} + + +def test_dependency_with_lookback_round_trips() -> None: + github = FakeGitHub() + proposal = _proposal( + dependencies=[ + ProposedDependency(name="mock-dep", version="0.1.0", lookback="5d") + ] + ) + propose_dataset(proposal, requested_by="team-a", client=github) + + config = github.calls[0]["files"]["builders/scripts/my-dataset/0.1.0/config.toml"] + raw = tomllib.loads(config) + assert raw["dependencies"]["mock-dep"] == {"version": "0.1.0", "lookback": "5d"} + + +def test_optional_files_included_when_present() -> None: + github = FakeGitHub() + proposal = _proposal( + env_vars=True, + requirements_txt="pandas>=2.0\n", + env_template="API_KEY=\n", + ) + propose_dataset(proposal, requested_by="team-a", client=github) + + call = github.calls[0] + files = call["files"] + assert "builders/scripts/my-dataset/0.1.0/requirements.txt" in files + assert "builders/scripts/my-dataset/0.1.0/.env.template" in files + # the env checklist reminds reviewers secrets are placed manually + assert ".env" in call["body"] + assert "before first build" in call["body"] + + +def test_env_file_itself_is_never_committed() -> None: + github = FakeGitHub() + propose_dataset( + _proposal(env_vars=True, env_template="API_KEY=\n"), + requested_by="team-a", + client=github, + ) + files = github.calls[0]["files"] + assert "builders/scripts/my-dataset/0.1.0/.env" not in files + + +def test_unknown_dependency_rejected() -> None: + proposal = _proposal( + dependencies=[ProposedDependency(name="nope", version="9.9.9")] + ) + with pytest.raises(InvalidProposalError, match="not a known dataset"): + propose_dataset(proposal, requested_by="t", client=FakeGitHub()) + + +def test_granularity_finer_than_dependency_rejected() -> None: + proposal = _proposal( + granularity="1h", + dependencies=[ProposedDependency(name="mock-dep", version="0.1.0")], + ) + with pytest.raises(InvalidProposalError, match="finer than dependency"): + propose_dataset(proposal, requested_by="t", client=FakeGitHub()) + + +def test_start_date_before_dependency_rejected() -> None: + proposal = _proposal( + start_date="2020-01-01", + dependencies=[ProposedDependency(name="mock-dep", version="0.1.0")], + ) + with pytest.raises(InvalidProposalError, match="start-date"): + propose_dataset(proposal, requested_by="t", client=FakeGitHub()) + + +def test_existing_dataset_conflicts() -> None: + proposal = _proposal(name="mock-dep", version="0.1.0", start_date="2021-06-01") + with pytest.raises(ProposalConflictError, match="already exists"): + propose_dataset(proposal, requested_by="t", client=FakeGitHub()) + + +def test_existing_branch_conflicts() -> None: + github = FakeGitHub(error=BranchAlreadyExistsError(422, "Reference already exists")) + with pytest.raises(ProposalConflictError, match="already open"): + propose_dataset(_proposal(), requested_by="t", client=github) + + +@pytest.mark.parametrize( + "bad_name", ["MyDataset", "my dataset", "../escape", "-leading", ""] +) +def test_invalid_names_rejected(bad_name: str) -> None: + with pytest.raises(InvalidProposalError, match="dataset name"): + propose_dataset(_proposal(name=bad_name), requested_by="t", client=FakeGitHub()) + + +@pytest.mark.parametrize( + "field_name", ["author_name", "team", "discord_user", "description"] +) +def test_blank_proposer_fields_rejected(field_name: str) -> None: + with pytest.raises(InvalidProposalError, match=field_name): + propose_dataset( + _proposal(**{field_name: " "}), requested_by="t", client=FakeGitHub() + ) + + +def test_invalid_version_rejected() -> None: + with pytest.raises(InvalidProposalError, match="version"): + propose_dataset( + _proposal(version="not-semver"), requested_by="t", client=FakeGitHub() + ) + + +def test_invalid_schema_type_rejected() -> None: + with pytest.raises(InvalidProposalError): + propose_dataset( + _proposal(schema={"price": "decimal"}), + requested_by="t", + client=FakeGitHub(), + ) + + +def test_unknown_calendar_rejected() -> None: + with pytest.raises(InvalidProposalError): + propose_dataset( + _proposal(calendar="lunar"), requested_by="t", client=FakeGitHub() + ) + + +def test_builder_syntax_error_rejected() -> None: + with pytest.raises(InvalidProposalError, match="syntax error"): + propose_dataset( + _proposal(builder_script="def build(:\n"), + requested_by="t", + client=FakeGitHub(), + ) + + +def test_builder_without_build_rejected() -> None: + with pytest.raises(InvalidProposalError, match="build\\(\\)"): + propose_dataset( + _proposal(builder_script="def make(a, b):\n return []\n"), + requested_by="t", + client=FakeGitHub(), + ) + + +def test_builder_wrong_arity_rejected() -> None: + with pytest.raises(InvalidProposalError, match="two arguments"): + propose_dataset( + _proposal(builder_script="def build(only_one):\n return []\n"), + requested_by="t", + client=FakeGitHub(), + ) + + +def test_default_reviewers_requested(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("GITHUB_REVIEWERS", raising=False) + github = FakeGitHub() + propose_dataset(_proposal(), requested_by="t", client=github) + assert github.calls[0]["reviewers"] == ["Blackgaurd", "Scr4tch587"] + + +def test_reviewers_env_override(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("GITHUB_REVIEWERS", "alice, bob") + github = FakeGitHub() + propose_dataset(_proposal(), requested_by="t", client=github) + assert github.calls[0]["reviewers"] == ["alice", "bob"] + + +def test_builder_script_is_autofixed_before_commit() -> None: + """unused imports and formatting are cleaned server-side so the pr passes ci.""" + github = FakeGitHub() + script = ( + "from datetime import datetime\n" + "from typing import Any\n\n\n" + "def build(dependencies, timestamp):\n" + " return [{'ticker': 'AAPL', 'price': 1.0}]\n" + ) + propose_dataset(_proposal(builder_script=script), requested_by="t", client=github) + committed = github.calls[0]["files"]["builders/scripts/my-dataset/0.1.0/builder.py"] + assert "from typing import Any" not in committed # unused import removed + assert "from datetime import datetime" not in committed + assert '"AAPL"' in committed # ruff format normalizes quotes + + +def test_unfixable_lint_error_rejected() -> None: + """violations ruff cannot autofix (e.g. undefined name) reject the proposal.""" + script = "def build(dependencies, timestamp):\n return [undefined_var]\n" + with pytest.raises(InvalidProposalError, match="fails lint"): + propose_dataset( + _proposal(builder_script=script), requested_by="t", client=FakeGitHub() + ) + + +def test_lint_error_surfaces_ruff_stderr(monkeypatch: pytest.MonkeyPatch) -> None: + """a ruff failure that only writes to stderr still reaches the caller.""" + + def fake_run(*_args: object, **_kwargs: object) -> subprocess.CompletedProcess[str]: + return subprocess.CompletedProcess( + args=[], returncode=2, stdout="", stderr="ruff failed: bad config" + ) + + monkeypatch.setattr(proposals.subprocess, "run", fake_run) + with pytest.raises(InvalidProposalError, match="ruff failed: bad config"): + propose_dataset(_proposal(), requested_by="t", client=FakeGitHub()) + + +def test_lint_error_reports_exit_code_when_ruff_is_silent( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """no output on either stream still produces a non-empty diagnostic.""" + + def fake_run(*_args: object, **_kwargs: object) -> subprocess.CompletedProcess[str]: + return subprocess.CompletedProcess( + args=[], returncode=101, stdout="", stderr="" + ) + + monkeypatch.setattr(proposals.subprocess, "run", fake_run) + with pytest.raises(InvalidProposalError, match="exited with code 101"): + propose_dataset(_proposal(), requested_by="t", client=FakeGitHub()) + + +def test_generated_toml_quotes_awkward_schema_keys() -> None: + """schema keys that aren't bare toml keys are quoted, not mangled.""" + proposal = _proposal(schema={"has space": "str", "ok_key": "int"}) + raw = tomllib.loads(generate_config_toml(proposal)) + assert raw["schema"] == {"has space": "str", "ok_key": "int"} diff --git a/dev-docs/SPEC-backend.md b/dev-docs/SPEC-backend.md index b72d042..02a838c 100644 --- a/dev-docs/SPEC-backend.md +++ b/dev-docs/SPEC-backend.md @@ -28,6 +28,10 @@ GET /data/{dataset_name}/{dataset_version}?start=&end=&bui DELETE /data/{dataset_name}/{dataset_version}?start=&end= ``` +``` +POST /datasets +``` + ### API authentication Every endpoint except `GET /status` requires a valid API key sent as a bearer token: @@ -179,6 +183,38 @@ Each entry in `rows` contains all data dicts for that timestamp (matching the DB | `404` | Dataset exists but has no rows in the requested range | | `500` | Unexpected failure (DB error) | +### Dataset proposals endpoint + +`POST /datasets` submits a new dataset for review. **The server never writes to its own scripts directory**: it validates the submission, generates the dataset files, and opens a GitHub pull request adding `builders/scripts///` to the repo. The dataset goes live only after the PR is reviewed, merged, and the server restarts. This keeps code review as the trust gate — a builder script is code that executes on the server, and the "internal users write trusted builders" assumption (see "Build behavior") is enforced by human review, not by the API key alone. + +**Request body** (JSON): `name`, `version`, `calendar`, `granularity`, `start_date`, `schema` (field → type map), `builder_script`, proposer identity (`author_name`, `team`, `discord_user`, `description` — all required, surfaced in the PR body), and optional `dependencies` (list of `{name, version, lookback?}`), `env_vars`, `requirements_txt`, `env_template`. + +**Validation** (`core/service/proposals.py`), in order: +1. name must match `^[a-z0-9][a-z0-9_-]*$` (it becomes a directory and branch name); version must parse as SemVer; proposer fields must be non-empty +2. `(name, version)` must not already exist in the config registry +3. the canonical `config.toml` is generated, then **those exact bytes** are re-parsed and run through the same `validate_config` used at startup — the committed config is byte-identical to what was validated +4. registry cross-checks against live configs: dependencies exist, granularity is not finer than any dependency's, start-date is not before any dependency's (a new dataset is a leaf nothing references, so cycles are impossible) +5. builder script: AST-parsed, must define a top-level `build(dependencies, timestamp)` with exactly two positional args +6. the script is then run through **ruff autofix + format** server-side (same rule selection as repo CI — kept in sync manually with the root `pyproject.toml`); unfixable violations reject the proposal with ruff's output. Proposal PRs therefore land pre-linted and cannot fail the CI lint gate. The rejection message combines **both** of ruff's streams (violations go to stdout, but a ruff-internal failure such as a bad config reports only on stderr) and falls back to the exit code if ruff printed nothing, so the caller and the server log never get a bare "fails lint:" with no diagnostic + +**PR shape**: branch `add-dataset/-` off `main`, one commit containing `config.toml`, `builder.py`, and (when provided) `requirements.txt` / `.env.template`. **The real `.env` is never committed** — for `env-vars = true` datasets the PR body carries a checklist item to place it on the server manually before the first build. Title: `feat: add dataset /`. Reviewers are auto-requested (best-effort: the PR author is excluded, and a failed batch falls back to per-reviewer requests). + +**Configuration** (env vars, documented in `infra/.env.template`): +- `GITHUB_TOKEN` (required for this endpoint only): PAT with Contents + Pull requests read/write on the repo. Without it the endpoint returns 502; everything else works. +- `GITHUB_REPO` (default `Wat-Street/datastream`), `GITHUB_API_URL` (default `https://api.github.com`; overridable for testing against a fake), `GITHUB_REVIEWERS` (default `Blackgaurd,Scr4tch587`). + +**Response** (200): `{"dataset_name", "dataset_version", "pr_url", "branch"}`. + +| Status | Meaning | +|--------|---------| +| `400` | Invalid submission (validation or lint failure; `detail` is safe to show in the UI) | +| `401` | Missing or invalid API key | +| `409` | Dataset already registered, or a proposal branch for it is already open | +| `422` | Malformed request body (missing/mistyped fields) | +| `502` | GitHub unreachable or `GITHUB_TOKEN` missing/invalid | + +The requesting API key's team label is logged and included in the PR body alongside the proposer fields. A CI test (`tests/core/runtime/test_real_scripts.py`) validates every real config in `builders/scripts/` with the startup validation path, so green CI on a proposal PR means the dataset cannot break server boot. + ### Build behavior - On a `POST /build` request, it builds missing data for the requested range and writes it to the database. diff --git a/dev-docs/SPEC-frontend.md b/dev-docs/SPEC-frontend.md index b5d7ae2..7ef940e 100644 --- a/dev-docs/SPEC-frontend.md +++ b/dev-docs/SPEC-frontend.md @@ -18,9 +18,10 @@ This is a purely internal tool. The goal is functional clarity, not aesthetics. ### Navigation -State-based navigation in `App.tsx`. Two views: +State-based navigation in `App.tsx`. Three views: - `'list'`: dataset catalog (landing page) - `'detail'`: dataset data viewer +- `'create'`: dataset proposal form (lazy-loaded via `React.lazy` — its CodeMirror chunk never loads during normal browsing) No routing library. A single discriminated-union `view` state controls which component renders. This also means GitHub Pages needs no SPA 404 fallback. @@ -48,11 +49,12 @@ frontend/src/ json-view.tsx # recursive jsx syntax highlighter (no innerHTML) json-modal.tsx # row details in a shadcn Dialog api-key-dialog.tsx # zod + react-hook-form key entry dialog + create-dataset.tsx # proposal form (lazy-loaded): schema/dep editors, CodeMirror ``` ### Dependencies -Runtime: React 19, TanStack Query (fetch state, caching, retries, central error handling), react-hook-form + zod + `@hookform/resolvers` (forms/validation), shadcn/ui's underlying packages (`radix-ui`, `lucide-react`, `sonner`, `class-variance-authority`, `clsx`, `tailwind-merge`). +Runtime: React 19, TanStack Query (fetch state, caching, retries, central error handling), react-hook-form + zod + `@hookform/resolvers` (forms/validation), shadcn/ui's underlying packages (`radix-ui`, `lucide-react`, `sonner`, `class-variance-authority`, `clsx`, `tailwind-merge`), and `@uiw/react-codemirror` + `@codemirror/lang-python` (builder-script editor, loaded only by the create view). ### Tooling and quality gates @@ -77,8 +79,9 @@ All API calls go through `lib/api.ts` and use the `/api/v1` prefix. - **Base URL**: `VITE_API_BASE_URL` (baked in at build time, set for Pages builds) or relative `/api` in dev, where Vite proxies to localhost:3000 - `fetchDatasets()` -- `GET /api/v1/datasets` - `fetchData(name, version, start, end)` -- `GET /api/v1/data/{name}/{version}?start=...&end=...&build-data=false` +- `proposeDataset(payload)` -- `POST /api/v1/datasets` (the only write; the backend opens a dataset-proposal PR, see "Dataset proposals endpoint" in `SPEC-backend.md`) -The frontend is read-only and never triggers builds (`build-data=false` always). Both 200 and 206 responses are treated as valid (206 indicates partial/incomplete data). Failures throw `ApiError` (carries the HTTP status) so the query layer can react per-status. +Browsing never triggers builds (`build-data=false` always). Both 200 and 206 responses are treated as valid (206 indicates partial/incomplete data). Failures throw `ApiError` carrying the HTTP status **and the backend's `detail` message** when present, so server-side validation errors render verbatim in forms. 401s from both queries and mutations funnel through one handler (`QueryCache` + `MutationCache` `onError`). ### API-key auth @@ -128,6 +131,19 @@ The backend requires `Authorization: Bearer ` on everything except `/st - Highlighting is done by `json-view.tsx`, a recursive JSX renderer (colors keys, strings, numbers, booleans, null) — no HTML strings, no `dangerouslySetInnerHTML` - Closes on: Escape key, backdrop click, or close button (built into the Dialog) -## Planned (not yet implemented) +### Create dataset (proposal form) -- **Create-dataset UI**: a form for name/version/calendar/granularity/start-date/schema/dependencies plus a builder-script editor (CodeMirror). Blocked on approval of the corresponding backend endpoint, which changes the builder trust model (script upload = code execution gated by API key instead of git review). +`create-dataset.tsx` — reachable via the "new dataset" header button — submits a dataset proposal to `POST /datasets`, which opens a GitHub PR (creation is **never direct**; review remains the gate for code that runs on the server). + +Form sections: + +- **Proposer** (all required): name, Wat Street team, Discord user, and a description of what the dataset is for — surfaced verbatim in the PR body for reviewers +- **Basics**: name, version, calendar + granularity selects (choices hardcoded as frontend constants; the server rejects unknowns), start date +- **Schema**: dynamic key/type rows (`str`/`int`/`float`/`bool`), minimum one. Field names are **trimmed** (so `"price "` can't become a different TOML key than `"price"`) and checked for **duplicates** — the schema is submitted as an object, so a repeated key would otherwise drop the earlier row silently. The duplicate issue is attached to the offending row's path, so the message renders on that row rather than at the section level +- **Dependencies**: picker populated from `GET /datasets`, optional lookback string (`5d`-style) +- **Builder script**: CodeMirror editor (Python highlighting, dark theme) pre-filled with a `build(dependencies, timestamp)` template that includes a commented example return value. A "reset to template" button in the section heading restores the template, so experimenting in the editor is recoverable +- **Extras**: optional `requirements.txt`; an env-vars checkbox reveals a `.env.template` field (the real `.env` never goes in the PR and must be placed on the server manually — the success screen and PR checklist both say so). Unchecking the box hides the field without clearing it, so `env_template` is only submitted when the checkbox is on — a generated config can never say `env-vars = false` while a `.env.template` still lands in the PR + +Validation is two-layered: zod mirrors the server's format rules (name regex, semver, `YYYY-MM-DD`, lookback format) for instant feedback, and any server-side 400/409 (`ApiError.message` carries the backend `detail`, e.g. dependency granularity violations) renders in an inline error box. The form runs in `mode: "onBlur"` rather than react-hook-form's `"onSubmit"` default, so hints appear as fields are filled instead of only after the first submit attempt. The error box preserves newlines (`whitespace-pre-wrap`), since ruff reports one violation per line. The server also lint-fixes the builder script with ruff before committing, so submitted scripts may differ cosmetically from what lands in the PR. + +On success the form is replaced by a panel linking the opened PR, with a "create another" reset. diff --git a/frontend/bun.lock b/frontend/bun.lock index 6303927..32bfe65 100644 --- a/frontend/bun.lock +++ b/frontend/bun.lock @@ -5,8 +5,10 @@ "": { "name": "frontend", "dependencies": { + "@codemirror/lang-python": "^6.2.1", "@hookform/resolvers": "^5.2.2", "@tanstack/react-query": "^5.90.0", + "@uiw/react-codemirror": "^4.25.11", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "lucide-react": "^1.23.0", @@ -72,12 +74,32 @@ "@babel/plugin-transform-react-jsx-source": ["@babel/plugin-transform-react-jsx-source@7.29.7", "", { "dependencies": { "@babel/helper-plugin-utils": "^7.29.7" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "sha512-06IyK09H3wi4cGbhDBwp5gUGo0IKtnYa8tyTiephirPCK6fbobVGiXMMI5zLQ4aKEYP3wZ3ArU44o+8KMrSG/Q=="], + "@babel/runtime": ["@babel/runtime@7.29.7", "", {}, "sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw=="], + "@babel/template": ["@babel/template@7.29.7", "", { "dependencies": { "@babel/code-frame": "^7.29.7", "@babel/parser": "^7.29.7", "@babel/types": "^7.29.7" } }, "sha512-puq+Gf35oI24FeN11LkoUQFqv9uwNeWpxXZi/Ji3rRIoKAzKnxRaZ+Gkj0vKS9ZCiTESfng1N9LyOyXvo+m+Gg=="], "@babel/traverse": ["@babel/traverse@7.29.7", "", { "dependencies": { "@babel/code-frame": "^7.29.7", "@babel/generator": "^7.29.7", "@babel/helper-globals": "^7.29.7", "@babel/parser": "^7.29.7", "@babel/template": "^7.29.7", "@babel/types": "^7.29.7", "debug": "^4.3.1" } }, "sha512-EhlfNQtZ+NK22w5BM61ciuiq1m58ed33Wr1Xan//ZRTy6hgjnwyCffRYwzsGXdASJSUJ1guZILsErh1eQcl+zw=="], "@babel/types": ["@babel/types@7.29.7", "", { "dependencies": { "@babel/helper-string-parser": "^7.29.7", "@babel/helper-validator-identifier": "^7.29.7" } }, "sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA=="], + "@codemirror/autocomplete": ["@codemirror/autocomplete@6.20.3", "", { "dependencies": { "@codemirror/language": "^6.0.0", "@codemirror/state": "^6.0.0", "@codemirror/view": "^6.17.0", "@lezer/common": "^1.0.0" } }, "sha512-tlosUqb+3BbxCxZdu4tKeRghPFC+QM7q4X5YhKV2eCmPG+1r2F3f4AaSz5sCrFqUtX4Jh20VFTKecl16MgiV9g=="], + + "@codemirror/commands": ["@codemirror/commands@6.10.4", "", { "dependencies": { "@codemirror/language": "^6.0.0", "@codemirror/state": "^6.7.0", "@codemirror/view": "^6.27.0", "@lezer/common": "^1.1.0" } }, "sha512-Ryk9y9T0FFVF0cUGhAknveAyUOl/A1qReTFi+qPKtOh2Z9F4AUBz3XOrYD4ZEgZirdugVzHvd/2/Wcwy5OliTg=="], + + "@codemirror/lang-python": ["@codemirror/lang-python@6.2.1", "", { "dependencies": { "@codemirror/autocomplete": "^6.3.2", "@codemirror/language": "^6.8.0", "@codemirror/state": "^6.0.0", "@lezer/common": "^1.2.1", "@lezer/python": "^1.1.4" } }, "sha512-IRjC8RUBhn9mGR9ywecNhB51yePWCGgvHfY1lWN/Mrp3cKuHr0isDKia+9HnvhiWNnMpbGhWrkhuWOc09exRyw=="], + + "@codemirror/language": ["@codemirror/language@6.12.4", "", { "dependencies": { "@codemirror/state": "^6.0.0", "@codemirror/view": "^6.23.0", "@lezer/common": "^1.5.0", "@lezer/highlight": "^1.0.0", "@lezer/lr": "^1.0.0", "style-mod": "^4.0.0" } }, "sha512-1q4PaT+o6PbgpkJt4Q8Fv5XJxTy4FUZ4MWETtyiDw3J0Pyr9E2vqcKL+k9wcvjNTIsauxvE7OfmWj3FRPHQ76A=="], + + "@codemirror/lint": ["@codemirror/lint@6.9.7", "", { "dependencies": { "@codemirror/state": "^6.0.0", "@codemirror/view": "^6.42.0", "crelt": "^1.0.5" } }, "sha512-28/+iWLYxKxsvGYhSYL7zaCZqLz5+FFFDq9tVsvGv9kv8RY4fFAchJ5WX9M3YrrRlTIsECjsXPqeNgnSmNP2dg=="], + + "@codemirror/search": ["@codemirror/search@6.7.1", "", { "dependencies": { "@codemirror/state": "^6.0.0", "@codemirror/view": "^6.37.0", "crelt": "^1.0.5" } }, "sha512-uMe5UO6PamJtSHrXhhHOzSX3ReWtiJrva6GnPMwSOrZtiExb5X5eExhr2OUZQVvdxPsKpY3Ro2mFbQadpPWmHA=="], + + "@codemirror/state": ["@codemirror/state@6.7.1", "", { "dependencies": { "@marijn/find-cluster-break": "^1.0.0" } }, "sha512-9QzNDgE4EYDnAHfrTlR2lwiPciiOymLtwKK+8yHQzCc7GXhAP9xdEbEJFy2IWB1j9UGUl9BsgMmTo/ImA02T7A=="], + + "@codemirror/theme-one-dark": ["@codemirror/theme-one-dark@6.1.3", "", { "dependencies": { "@codemirror/language": "^6.0.0", "@codemirror/state": "^6.0.0", "@codemirror/view": "^6.0.0", "@lezer/highlight": "^1.0.0" } }, "sha512-NzBdIvEJmx6fjeremiGp3t/okrLPYT0d9orIc7AFun8oZcRk58aejkqhv6spnz4MLAevrKNPMQYXEWMg4s+sKA=="], + + "@codemirror/view": ["@codemirror/view@6.43.6", "", { "dependencies": { "@codemirror/state": "^6.7.0", "crelt": "^1.0.6", "style-mod": "^4.1.0", "w3c-keyname": "^2.2.4" } }, "sha512-EVunGSYN1wz1p75WY1s3Xg7t3i8Yol0kGZGizNdX9BUFgMFILYVe8/u6EVpo7Ff5PwbZuILb4QAq7IZoKzIEQA=="], + "@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.28.1", "", { "os": "aix", "cpu": "ppc64" }, "sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ=="], "@esbuild/android-arm": ["@esbuild/android-arm@0.28.1", "", { "os": "android", "cpu": "arm" }, "sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ=="], @@ -178,6 +200,16 @@ "@jridgewell/trace-mapping": ["@jridgewell/trace-mapping@0.3.31", "", { "dependencies": { "@jridgewell/resolve-uri": "^3.1.0", "@jridgewell/sourcemap-codec": "^1.4.14" } }, "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw=="], + "@lezer/common": ["@lezer/common@1.5.2", "", {}, "sha512-sxQE460fPZyU3sdc8lafxiPwJHBzZRy/udNFynGQky1SePYBdhkBl1kOagA9uT3pxR8K09bOrmTUqA9wb/PjSQ=="], + + "@lezer/highlight": ["@lezer/highlight@1.2.3", "", { "dependencies": { "@lezer/common": "^1.3.0" } }, "sha512-qXdH7UqTvGfdVBINrgKhDsVTJTxactNNxLk7+UMwZhU13lMHaOBlJe9Vqp907ya56Y3+ed2tlqzys7jDkTmW0g=="], + + "@lezer/lr": ["@lezer/lr@1.4.10", "", { "dependencies": { "@lezer/common": "^1.0.0" } }, "sha512-rnCpTIBafOx4mRp43xOxDJbFipJm/c0cia/V5TiGlhmMa+wsSdoGmUN3w5Bqrks/09Q/D4tNAmWaT8p6NRi77A=="], + + "@lezer/python": ["@lezer/python@1.1.19", "", { "dependencies": { "@lezer/common": "^1.2.0", "@lezer/highlight": "^1.0.0", "@lezer/lr": "^1.0.0" } }, "sha512-MhQIURHRytsNzP/YXnqpYKW6la6voAH3kyplTOOiCdjyFY6cWWGFVmYVdHIPrElqSDf4iCDktQCockB9FxuhzQ=="], + + "@marijn/find-cluster-break": ["@marijn/find-cluster-break@1.0.3", "", {}, "sha512-FY+MKLBoTsLNJF/eLWaOsXGdz6uh3Iu1axjPf6TUq92IYumcTcXWHoS747JARLkcdlJ/Waiaxc5wQfFO8jC6NA=="], + "@radix-ui/number": ["@radix-ui/number@1.1.2", "", {}, "sha512-ceTwaxc4I5IOi97DgCotl3pqiyRGvffcc0oOsE2dQYaJOFIDsDt4VWG6xEbg1QePv9QWausCEIppud/tJ1wNig=="], "@radix-ui/primitive": ["@radix-ui/primitive@1.1.5", "", {}, "sha512-d86WIWFYNtGA0H/d8exstrTRTp7eWJYlYJbtNofxr/3ljupZYn6EFDG/Qgu/0Kc8v7yMUxySagqJsL1+PdYjWg=="], @@ -424,6 +456,10 @@ "@typescript-eslint/visitor-keys": ["@typescript-eslint/visitor-keys@8.63.0", "", { "dependencies": { "@typescript-eslint/types": "8.63.0", "eslint-visitor-keys": "^5.0.0" } }, "sha512-UexrHGnGTpbuQHct2ExOc2ZcFbGUS9FOesCxxqdBGcpI1BxYu/LZ6U8Aq6/72XtF/qRBk9nhuGHFJIXXMhPMdw=="], + "@uiw/codemirror-extensions-basic-setup": ["@uiw/codemirror-extensions-basic-setup@4.25.11", "", { "dependencies": { "@codemirror/autocomplete": "^6.0.0", "@codemirror/commands": "^6.0.0", "@codemirror/language": "^6.0.0", "@codemirror/lint": "^6.0.0", "@codemirror/search": "^6.0.0", "@codemirror/state": "^6.0.0", "@codemirror/view": "^6.0.0" } }, "sha512-otyFa+n9IOYtEjaKOxPedHkj15fTPUF21wdR9pv0GpZPfuGl27cvmcv6+tognbRu9VvEcsHKE+ESoszeo3KfTw=="], + + "@uiw/react-codemirror": ["@uiw/react-codemirror@4.25.11", "", { "dependencies": { "@babel/runtime": "^7.18.6", "@codemirror/commands": "^6.1.0", "@codemirror/state": "^6.1.1", "@codemirror/theme-one-dark": "^6.0.0", "@uiw/codemirror-extensions-basic-setup": "4.25.11", "codemirror": "^6.0.0" }, "peerDependencies": { "@codemirror/view": ">=6.0.0", "react": ">=17.0.0", "react-dom": ">=17.0.0" } }, "sha512-DYVFAKLX+F/4JS9N/7xexh+TICrlncwkX9HKKInrP1bwO0tSfc3k0GB6oawTYhelVKh20cX3TuRx+NJSkVXuMw=="], + "@vitejs/plugin-react": ["@vitejs/plugin-react@5.2.0", "", { "dependencies": { "@babel/core": "^7.29.0", "@babel/plugin-transform-react-jsx-self": "^7.27.1", "@babel/plugin-transform-react-jsx-source": "^7.27.1", "@rolldown/pluginutils": "1.0.0-rc.3", "@types/babel__core": "^7.20.5", "react-refresh": "^0.18.0" }, "peerDependencies": { "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0" } }, "sha512-YmKkfhOAi3wsB1PhJq5Scj3GXMn3WvtQ/JC0xoopuHoXSdmtdStOpFrYaT1kie2YgFBcIe64ROzMYRjCrYOdYw=="], "acorn": ["acorn@8.17.0", "", { "bin": { "acorn": "bin/acorn" } }, "sha512-xRQbDb9BnwDafYNn6Vwl839DYVjqXYb1XVGtWAZ1kcDc6iwAL4hg3B1dZlRiuENFeO2H53gFG3in621AdERVAg=="], @@ -456,6 +492,8 @@ "clsx": ["clsx@2.1.1", "", {}, "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA=="], + "codemirror": ["codemirror@6.0.2", "", { "dependencies": { "@codemirror/autocomplete": "^6.0.0", "@codemirror/commands": "^6.0.0", "@codemirror/language": "^6.0.0", "@codemirror/lint": "^6.0.0", "@codemirror/search": "^6.0.0", "@codemirror/state": "^6.0.0", "@codemirror/view": "^6.0.0" } }, "sha512-VhydHotNW5w1UGK0Qj96BwSk/Zqbp9WbnyK2W/eVMv4QyF41INRGpjUhFJY7/uDNuudSc33a/PKr4iDqRduvHw=="], + "color-convert": ["color-convert@2.0.1", "", { "dependencies": { "color-name": "~1.1.4" } }, "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ=="], "color-name": ["color-name@1.1.4", "", {}, "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="], @@ -464,6 +502,8 @@ "convert-source-map": ["convert-source-map@2.0.0", "", {}, "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg=="], + "crelt": ["crelt@1.0.7", "", {}, "sha512-aK6BbWfhf4U/wCcLHKPJl/xa6VkVstRaPywWtMKGwuOLc/wZTyQYuoxgvZnNsBvv7Kg3YTBQYYBCggcviQczuA=="], + "cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="], "csstype": ["csstype@3.2.3", "", {}, "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ=="], @@ -672,6 +712,8 @@ "strip-json-comments": ["strip-json-comments@3.1.1", "", {}, "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig=="], + "style-mod": ["style-mod@4.1.3", "", {}, "sha512-i/n8VsZydrugj3Iuzll8+x/00GH2vnYsk1eomD8QiRrSAeW6ItbCQDtfXCeJHd0iwiNagqjQkvpvREEPtW3IoQ=="], + "supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], "tailwind-merge": ["tailwind-merge@3.6.0", "", {}, "sha512-uxL7qAVQriqRQPAyK3pj66VqskWqoZ37PW94jwOTwNfq/z9oyu1V+eqrZqtR2+fCiXdYOZe/Modt8GtvqNzu+w=="], @@ -706,6 +748,8 @@ "vite": ["vite@7.3.6", "", { "dependencies": { "esbuild": "^0.27.0 || ^0.28.0", "fdir": "^6.5.0", "picomatch": "^4.0.3", "postcss": "^8.5.6", "rollup": "^4.43.0", "tinyglobby": "^0.2.15" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", "jiti": ">=1.21.0", "less": "^4.0.0", "lightningcss": "^1.21.0", "sass": "^1.70.0", "sass-embedded": "^1.70.0", "stylus": ">=0.54.8", "sugarss": "^5.0.0", "terser": "^5.16.0", "tsx": "^4.8.1", "yaml": "^2.4.2" }, "optionalPeers": ["@types/node", "jiti", "less", "lightningcss", "sass", "sass-embedded", "stylus", "sugarss", "terser", "tsx", "yaml"], "bin": { "vite": "bin/vite.js" } }, "sha512-4XP60spRGjSZFf1qYH+dJIkK2znL3zQfl9KkOV9MkkRR/3Dls0dxaBsQPTloEc5BLXWPL9vsOxopxyKoMmDueg=="], + "w3c-keyname": ["w3c-keyname@2.2.8", "", {}, "sha512-dpojBhNsCNN7T82Tm7k26A6G9ML3NkhDsnw9n/eoxSRlVBB4CEtIQ/KTCLI2Fwf3ataSXRhYFkQi3SlnFwPvPQ=="], + "which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="], "word-wrap": ["word-wrap@1.2.5", "", {}, "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA=="], diff --git a/frontend/package.json b/frontend/package.json index b47a13d..9e90df9 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -13,8 +13,10 @@ "typecheck": "tsc -b" }, "dependencies": { + "@codemirror/lang-python": "^6.2.1", "@hookform/resolvers": "^5.2.2", "@tanstack/react-query": "^5.90.0", + "@uiw/react-codemirror": "^4.25.11", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "lucide-react": "^1.23.0", diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 77756da..e21c89e 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -1,5 +1,5 @@ -import { KeyRoundIcon } from "lucide-react"; -import { useState } from "react"; +import { KeyRoundIcon, PlusIcon } from "lucide-react"; +import { lazy, Suspense, useState } from "react"; import type { DatasetSummary } from "@/lib/api"; @@ -9,7 +9,17 @@ import { DatasetList } from "@/components/dataset-list"; import { Button } from "@/components/ui/button"; import { useApiKey } from "@/hooks/use-api-key"; -type View = { view: "list" } | { view: "detail"; dataset: DatasetSummary }; +// lazy: the create view pulls in codemirror, which shouldn't weigh down browsing +const CreateDataset = lazy(() => + import("@/components/create-dataset").then((module) => ({ + default: module.CreateDataset, + })), +); + +type View = + | { view: "list" } + | { view: "detail"; dataset: DatasetSummary } + | { view: "create" }; function App() { const { setDialogOpen } = useApiKey(); @@ -19,27 +29,48 @@ function App() {

Datastream

- +
+ {view.view === "list" && ( + + )} + +
- {view.view === "list" ? ( + {view.view === "list" && ( setView({ view: "detail", dataset })} /> - ) : ( + )} + {view.view === "detail" && ( setView({ view: "list" })} /> )} + {view.view === "create" && ( + loading...

+ } + > + setView({ view: "list" })} /> + + )}
diff --git a/frontend/src/components/create-dataset.tsx b/frontend/src/components/create-dataset.tsx new file mode 100644 index 0000000..56ac379 --- /dev/null +++ b/frontend/src/components/create-dataset.tsx @@ -0,0 +1,741 @@ +import { python } from "@codemirror/lang-python"; +import { zodResolver } from "@hookform/resolvers/zod"; +import { useMutation } from "@tanstack/react-query"; +import CodeMirror from "@uiw/react-codemirror"; +import { + ArrowLeftIcon, + ExternalLinkIcon, + PlusIcon, + RotateCcwIcon, + XIcon, +} from "lucide-react"; +import { useFieldArray, useForm, useWatch } from "react-hook-form"; +import { z } from "zod"; + +import type { DatasetProposalPayload, ProposalResponse } from "@/lib/api"; + +import { Button } from "@/components/ui/button"; +import { Checkbox } from "@/components/ui/checkbox"; +import { + Form, + FormControl, + FormDescription, + FormField, + FormItem, + FormLabel, + FormMessage, +} from "@/components/ui/form"; +import { Input } from "@/components/ui/input"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Textarea } from "@/components/ui/textarea"; +import { useDatasets } from "@/hooks/use-datasets"; +import { proposeDataset } from "@/lib/api"; + +const SCHEMA_TYPES = ["str", "int", "float", "bool"] as const; +const CALENDARS = ["everyday", "weekday", "always-open", "nyse-daily"] as const; +const GRANULARITIES = ["1s", "1m", "1h", "1d"] as const; + +const DEFAULT_BUILDER = `from datetime import datetime +from typing import Any + + +def build( + dependencies: dict[str, dict[datetime, list[dict]]], + timestamp: datetime, +) -> list[dict[str, Any]]: + # return one dict per row for this timestamp, matching the schema + # + # e.g. with a schema of {"ticker": "str", "close": "float"}, a dataset with + # one row per ticker would return: + # + # return [ + # {"ticker": "AAPL", "close": 189.5}, + # {"ticker": "MSFT", "close": 402.1}, + # ] + # + # a single-row dataset returns a list of length 1. dependency data arrives + # as dependencies["dep-name"][timestamp] -> list[dict] + return [] +`; + +const formSchema = z.object({ + name: z + .string() + .regex( + /^[a-z0-9][a-z0-9_-]*$/, + "lowercase alphanumeric with - or _ (becomes the directory name)", + ), + version: z.string().regex(/^\d+\.\d+\.\d+$/, "semver like 0.1.0"), + calendar: z.enum(CALENDARS), + granularity: z.enum(GRANULARITIES), + startDate: z.string().regex(/^\d{4}-\d{2}-\d{2}$/, "YYYY-MM-DD"), + schemaFields: z + .array( + z.object({ + // trimmed so " price" and "price" can't become two different toml keys + key: z.string().trim().min(1, "field name required"), + type: z.enum(SCHEMA_TYPES), + }), + ) + .min(1, "at least one schema field") + // the schema becomes an object, so a duplicate key would silently drop the + // earlier row instead of reaching the server + .superRefine((fields, ctx) => { + const seen = new Set(); + fields.forEach((field, index) => { + if (!field.key) return; + if (seen.has(field.key)) { + ctx.addIssue({ + code: "custom", + message: "duplicate field name", + path: [index, "key"], + }); + return; + } + seen.add(field.key); + }); + }), + dependencies: z.array( + z.object({ + dataset: z.string().min(1, "pick a dataset"), + lookback: z + .string() + .regex(/^\d+[dhms]$/, "like 5d, 24h, 30m, 60s") + .or(z.literal("")), + }), + ), + builderScript: z + .string() + .refine( + (script) => /def\s+build\s*\(/.test(script), + "must define a build() function", + ), + authorName: z.string().trim().min(1, "your name is required"), + team: z.string().trim().min(1, "team is required"), + discordUser: z.string().trim().min(1, "discord username is required"), + description: z + .string() + .trim() + .min(10, "a sentence or two about what this dataset is for"), + envVars: z.boolean(), + requirementsTxt: z.string(), + envTemplate: z.string(), +}); + +type FormValues = z.infer; + +function toPayload(values: FormValues): DatasetProposalPayload { + return { + name: values.name, + version: values.version, + calendar: values.calendar, + granularity: values.granularity, + start_date: values.startDate, + schema: Object.fromEntries( + values.schemaFields.map((field) => [field.key, field.type]), + ), + dependencies: values.dependencies.map((dep) => { + const [name, version] = dep.dataset.split("@"); + return { name, version, lookback: dep.lookback || undefined }; + }), + builder_script: values.builderScript, + author_name: values.authorName, + team: values.team, + discord_user: values.discordUser, + description: values.description, + env_vars: values.envVars, + requirements_txt: values.requirementsTxt.trim() + ? values.requirementsTxt + : undefined, + // gated on envVars too: the textarea is only hidden when the box is + // unchecked, not cleared, so a stale template must not reach the pr + env_template: + values.envVars && values.envTemplate.trim() + ? values.envTemplate + : undefined, + }; +} + +function SectionHeading({ + children, + action, +}: { + children: React.ReactNode; + action?: React.ReactNode; +}) { + return ( +
+

{children}

+ {action} +
+ ); +} + +export function CreateDataset({ onBack }: { onBack: () => void }) { + const { data: datasets } = useDatasets(); + + const form = useForm({ + resolver: zodResolver(formSchema), + // default is "onSubmit", which hides every hint until the first submit + mode: "onBlur", + defaultValues: { + name: "", + version: "0.1.0", + calendar: "everyday", + granularity: "1d", + startDate: "", + authorName: "", + team: "", + discordUser: "", + description: "", + schemaFields: [{ key: "", type: "str" }], + dependencies: [], + builderScript: DEFAULT_BUILDER, + envVars: false, + requirementsTxt: "", + envTemplate: "", + }, + }); + + const schemaRows = useFieldArray({ + control: form.control, + name: "schemaFields", + }); + const depRows = useFieldArray({ + control: form.control, + name: "dependencies", + }); + + const envVarsEnabled = useWatch({ + control: form.control, + name: "envVars", + }); + + const mutation = useMutation({ + mutationFn: (values) => proposeDataset(toPayload(values)), + }); + + if (mutation.isSuccess) { + return ( +
+

Proposal opened

+

+ + {mutation.data.dataset_name}/{mutation.data.dataset_version} + {" "} + was submitted as a pull request. It goes live after review, merge, and + the next server restart. +

+

+ + {mutation.data.pr_url} + +

+ {form.getValues("envVars") && ( +

+ this dataset needs env vars: after merge, someone must place the + real .env on the server (see the + PR checklist) before it can build. +

+ )} +
+ + +
+
+ ); + } + + return ( +
+ +

New dataset

+

+ Submitting opens a pull request with the dataset files — nothing runs + until a reviewer merges it. +

+ +
+ mutation.mutate(values))} + className="space-y-8" + > +
+ Proposer +
+ ( + + Your name + + + + + + )} + /> + ( + + Wat Street team + + + + + + )} + /> + ( + + Discord user + + + + + + )} + /> +
+ ( + + What is this dataset for? + +