diff --git a/.github/workflows/deploy-pages.yml b/.github/workflows/deploy-pages.yml new file mode 100644 index 00000000..edfd38d2 --- /dev/null +++ b/.github/workflows/deploy-pages.yml @@ -0,0 +1,53 @@ +name: Deploy Frontend to Pages + +on: + push: + branches: [main] + paths: + - "frontend/**" + - ".github/workflows/deploy-pages.yml" + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: pages + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: oven-sh/setup-bun@v2 + + - name: Install dependencies + working-directory: frontend + run: bun install --frozen-lockfile + + - name: Build + working-directory: frontend + run: bun run build + env: + # repo actions variable: the https api origin (no trailing slash, no /api/v1) + VITE_API_BASE_URL: ${{ vars.VITE_API_BASE_URL }} + + - uses: actions/configure-pages@v5 + + - uses: actions/upload-pages-artifact@v3 + with: + path: frontend/dist + + deploy: + needs: build + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - id: deployment + uses: actions/deploy-pages@v4 diff --git a/builders/server/core/api/routes.py b/builders/server/core/api/routes.py index 9a2f8834..b55ed166 100644 --- a/builders/server/core/api/routes.py +++ b/builders/server/core/api/routes.py @@ -1,9 +1,12 @@ from datetime import datetime import structlog -from fastapi import APIRouter, HTTPException, Query +from fastapi import APIRouter, Depends, HTTPException, Query from fastapi.responses import JSONResponse +from pydantic import BaseModel, ConfigDict, Field +from core.auth import verify_api_key +from core.github import GitHubError from core.service.builder import ( DatasetNotFoundError, NoDataInRangeError, @@ -13,6 +16,13 @@ get_data, ) from core.service.catalog import list_datasets +from core.service.proposals import ( + DatasetProposal, + InvalidProposalError, + ProposalConflictError, + ProposedDependency, + propose_dataset, +) from core.utils.semver import SemVer logger = structlog.get_logger() @@ -45,6 +55,89 @@ def datasets_list() -> dict: } +class DependencyIn(BaseModel): + name: str + version: str + lookback: str | None = None + + +class DatasetProposalIn(BaseModel): + model_config = ConfigDict(populate_by_name=True) + + name: str + version: str + calendar: str + granularity: str + start_date: str + builder_script: str + author_name: str + team: str + discord_user: str + description: str + # 'schema' shadows a deprecated BaseModel attr, so alias it + data_schema: dict[str, str] = Field(alias="schema") + dependencies: list[DependencyIn] = Field(default_factory=list) + env_vars: bool = False + requirements_txt: str | None = None + env_template: str | None = None + + +@router.post("/datasets") +def datasets_propose( + payload: DatasetProposalIn, team: str = Depends(verify_api_key) +) -> dict: + """Propose a new dataset: validate the submission and open a GitHub PR. + + Nothing is written to the server; the dataset goes live only after the + PR is reviewed, merged, and the server restarts. + """ + structlog.contextvars.bind_contextvars( + dataset_name=payload.name, version=payload.version + ) + proposal = DatasetProposal( + name=payload.name, + version=payload.version, + calendar=payload.calendar, + granularity=payload.granularity, + start_date=payload.start_date, + schema=payload.data_schema, + builder_script=payload.builder_script, + author_name=payload.author_name, + team=payload.team, + discord_user=payload.discord_user, + description=payload.description, + dependencies=[ + ProposedDependency(name=d.name, version=d.version, lookback=d.lookback) + for d in payload.dependencies + ], + env_vars=payload.env_vars, + requirements_txt=payload.requirements_txt, + env_template=payload.env_template, + ) + + try: + result = propose_dataset(proposal, requested_by=team) + except InvalidProposalError as e: + logger.warning("proposal rejected", error=str(e)) + raise HTTPException(status_code=400, detail=str(e)) from e + except ProposalConflictError as e: + logger.warning("proposal conflict", error=str(e)) + raise HTTPException(status_code=409, detail=str(e)) from e + except GitHubError as e: + logger.exception("proposal github call failed") + raise HTTPException(status_code=502, detail=f"github error: {e.message}") from e + except Exception as e: + logger.exception("proposal failed") + raise HTTPException(status_code=500, detail=str(e)) from e + + return { + "dataset_name": payload.name, + "dataset_version": payload.version, + "pr_url": result.pr_url, + "branch": result.branch, + } + + @router.post("/build/{dataset_name}/{dataset_version}") def build( dataset_name: str, diff --git a/builders/server/core/github/__init__.py b/builders/server/core/github/__init__.py new file mode 100644 index 00000000..eb6ff356 --- /dev/null +++ b/builders/server/core/github/__init__.py @@ -0,0 +1,7 @@ +from core.github.client import ( + BranchAlreadyExistsError, + GitHubClient, + GitHubError, +) + +__all__ = ["BranchAlreadyExistsError", "GitHubClient", "GitHubError"] diff --git a/builders/server/core/github/client.py b/builders/server/core/github/client.py new file mode 100644 index 00000000..0d07d325 --- /dev/null +++ b/builders/server/core/github/client.py @@ -0,0 +1,209 @@ +"""Minimal GitHub REST client for opening dataset-proposal pull requests. + +The dataset-creation flow never writes to the server's own scripts directory; +it proposes new datasets as PRs so code review stays the gate for anything +that will execute on the server. This client holds the bot token and wraps +the handful of REST calls needed: read a branch sha, commit files to a new +branch (single commit via the git data api), and open the PR. +""" + +import os + +import requests +import structlog + +logger = structlog.get_logger() + +DEFAULT_API_ROOT = "https://api.github.com" +DEFAULT_REPO = "Wat-Street/datastream" +REQUEST_TIMEOUT_SECONDS = 15.0 + + +class GitHubError(Exception): + """Raised when a GitHub API call fails.""" + + def __init__(self, status: int, message: str) -> None: + super().__init__(f"github api error {status}: {message}") + self.status = status + self.message = message + + +class BranchAlreadyExistsError(GitHubError): + """Raised when the proposal branch already exists (open proposal).""" + + +class GitHubClient: + """Thin wrapper over the GitHub REST api, authenticated with a bot token.""" + + def __init__( + self, + token: str, + repo: str = DEFAULT_REPO, + session: requests.Session | None = None, + api_root: str = DEFAULT_API_ROOT, + ) -> None: + self.repo = repo + self._token = token + self._session = session or requests.Session() + self._api_root = api_root + + @classmethod + def from_env(cls) -> "GitHubClient": + """Build a client from GITHUB_TOKEN / GITHUB_REPO / GITHUB_API_URL env vars. + + Raises GitHubError if no token is configured. + """ + token = os.environ.get("GITHUB_TOKEN", "") + if not token: + raise GitHubError(0, "GITHUB_TOKEN is not configured on the server") + return cls( + token=token, + repo=os.environ.get("GITHUB_REPO", DEFAULT_REPO), + api_root=os.environ.get("GITHUB_API_URL", DEFAULT_API_ROOT), + ) + + def _request(self, method: str, path: str, json_body: dict | None = None) -> dict: + res = self._session.request( + method, + f"{self._api_root}{path}", + json=json_body, + headers={ + "Authorization": f"Bearer {self._token}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + }, + timeout=REQUEST_TIMEOUT_SECONDS, + ) + if res.status_code >= 400: + try: + message = str(res.json().get("message", res.text)) + except ValueError: + message = res.text + logger.error( + "github api call failed", + method=method, + path=path, + status=res.status_code, + message=message, + ) + # the git refs api reports an existing branch as a 422 + if res.status_code == 422 and "already exists" in message.lower(): + raise BranchAlreadyExistsError(res.status_code, message) + raise GitHubError(res.status_code, message) + return dict(res.json()) + + def get_branch_sha(self, branch: str) -> str: + """Return the commit sha a branch currently points at.""" + ref = self._request("GET", f"/repos/{self.repo}/git/ref/heads/{branch}") + return str(ref["object"]["sha"]) + + def commit_files_to_new_branch( + self, + branch: str, + base_sha: str, + message: str, + files: dict[str, str], + ) -> str: + """Create one commit containing `files` on a new branch off base_sha. + + `files` maps repo-relative paths to text content. Returns the commit + sha. Raises BranchAlreadyExistsError if the branch exists. + """ + tree = self._request( + "POST", + f"/repos/{self.repo}/git/trees", + { + "base_tree": base_sha, + "tree": [ + {"path": path, "mode": "100644", "type": "blob", "content": content} + for path, content in sorted(files.items()) + ], + }, + ) + commit = self._request( + "POST", + f"/repos/{self.repo}/git/commits", + {"message": message, "tree": tree["sha"], "parents": [base_sha]}, + ) + self._request( + "POST", + f"/repos/{self.repo}/git/refs", + {"ref": f"refs/heads/{branch}", "sha": commit["sha"]}, + ) + return str(commit["sha"]) + + def create_pull(self, title: str, body: str, head: str, base: str) -> dict: + """Open a PR and return the api response (html_url, number, ...).""" + return self._request( + "POST", + f"/repos/{self.repo}/pulls", + {"title": title, "body": body, "head": head, "base": base}, + ) + + def request_reviewers(self, pr_number: int, reviewers: list[str]) -> None: + """Best-effort reviewer assignment. + + github rejects the WHOLE batch if any single login is invalid (pr + author, non-collaborator), which would silently drop the valid + reviewers too — so on batch failure, retry each login individually. + Failures must not fail the proposal, so they are only logged. + """ + try: + self._request( + "POST", + f"/repos/{self.repo}/pulls/{pr_number}/requested_reviewers", + {"reviewers": reviewers}, + ) + return + except GitHubError as e: + logger.warning( + "batch reviewer request failed, retrying individually", + pr_number=pr_number, + reviewers=reviewers, + error=str(e), + ) + for reviewer in reviewers: + try: + self._request( + "POST", + f"/repos/{self.repo}/pulls/{pr_number}/requested_reviewers", + {"reviewers": [reviewer]}, + ) + except GitHubError as e: + logger.warning( + "reviewer request failed", + pr_number=pr_number, + reviewer=reviewer, + error=str(e), + ) + + def open_pr_with_files( + self, + branch: str, + base: str, + title: str, + body: str, + commit_message: str, + files: dict[str, str], + reviewers: list[str] | None = None, + ) -> str: + """Commit `files` to a new branch off `base` and open a PR. + + Returns the PR's html url. + """ + base_sha = self.get_branch_sha(base) + self.commit_files_to_new_branch(branch, base_sha, commit_message, files) + pr = self.create_pull(title=title, body=body, head=branch, base=base) + if reviewers: + # the token's own account authors the pr and cannot review it + author = str(pr.get("user", {}).get("login", "")) + eligible = [r for r in reviewers if r.lower() != author.lower()] + if eligible: + self.request_reviewers(int(pr["number"]), eligible) + logger.info( + "proposal pr opened", + repo=self.repo, + branch=branch, + pr_url=pr.get("html_url"), + ) + return str(pr["html_url"]) diff --git a/builders/server/core/service/proposals.py b/builders/server/core/service/proposals.py new file mode 100644 index 00000000..06052239 --- /dev/null +++ b/builders/server/core/service/proposals.py @@ -0,0 +1,419 @@ +"""Dataset proposal service: validate a submission and open a GitHub PR. + +Proposed datasets are never written to the server's scripts directory — code +review stays the gate for anything that will execute on the server. This +module turns a submission into the exact files that will land in +builders/scripts///, re-validates the generated config bytes +with the same checks the server runs at startup, and opens the PR via the +github client. +""" + +import ast +import json +import os +import re +import subprocess +import sys +import tempfile +import tomllib +from dataclasses import dataclass, field +from pathlib import Path +from typing import Protocol + +import structlog + +from core.github import BranchAlreadyExistsError, GitHubClient +from core.runtime import registry +from core.runtime.config import DatasetConfig, normalize_config, validate_config +from core.utils.semver import SemVer + +logger = structlog.get_logger() + +# dataset names become directory names, branch names, and toml keys +_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9_-]*$") +_BARE_TOML_KEY_RE = re.compile(r"^[A-Za-z0-9_-]+$") + +SCRIPTS_PREFIX = "builders/scripts" + +# keep in sync with [tool.ruff.lint] select in the repo-root pyproject.toml: +# proposal prs must pass the same ci lint that runs over builders/scripts/** +RUFF_SELECT = "B,E,F,I,PIE,SIM,T20,UP" +RUFF_TIMEOUT_SECONDS = 30 + +# github logins asked to review every proposal pr (comma-separated env override) +DEFAULT_REVIEWERS = "Blackgaurd,Scr4tch587" + + +class InvalidProposalError(ValueError): + """Submission failed validation; safe to show the message to the caller.""" + + +class ProposalConflictError(Exception): + """The dataset or its proposal branch already exists.""" + + +@dataclass(frozen=True) +class ProposedDependency: + name: str + version: str + lookback: str | None = None + + +@dataclass(frozen=True) +class DatasetProposal: + name: str + version: str + calendar: str + granularity: str + start_date: str + schema: dict[str, str] + builder_script: str + # who is proposing and why -- surfaced in the pr body for reviewers + author_name: str + team: str + discord_user: str + description: str + dependencies: list[ProposedDependency] = field(default_factory=list) + env_vars: bool = False + requirements_txt: str | None = None + env_template: str | None = None + + +@dataclass(frozen=True) +class ProposalResult: + pr_url: str + branch: str + + +class PullRequestOpener(Protocol): + """The slice of GitHubClient the proposal service needs (test-fakeable).""" + + def open_pr_with_files( + self, + branch: str, + base: str, + title: str, + body: str, + commit_message: str, + files: dict[str, str], + reviewers: list[str] | None = None, + ) -> str: ... + + +def _toml_str(value: str) -> str: + """quote a string as a toml basic string + + (json escaping is a valid subset). + """ + return json.dumps(value) + + +def _toml_key(key: str) -> str: + return key if _BARE_TOML_KEY_RE.fullmatch(key) else json.dumps(key) + + +def generate_config_toml(proposal: DatasetProposal) -> str: + """Render the canonical config.toml for a proposal. + + The returned text is exactly what gets committed; callers re-parse and + re-validate these bytes so the PR can never contain a config that fails + server startup. + """ + lines = [ + f"name = {_toml_str(proposal.name)}", + f"version = {_toml_str(proposal.version)}", + 'builder = "builder.py"', + f"calendar = {_toml_str(proposal.calendar)}", + f"granularity = {_toml_str(proposal.granularity)}", + f"start-date = {_toml_str(proposal.start_date)}", + ] + if proposal.env_vars: + lines.append("env-vars = true") + + lines += ["", "[schema]"] + lines += [ + f"{_toml_key(key)} = {_toml_str(type_)}" + for key, type_ in proposal.schema.items() + ] + + if proposal.dependencies: + lines += ["", "[dependencies]"] + for dep in proposal.dependencies: + if dep.lookback: + lines.append( + f"{_toml_key(dep.name)} = {{ version = {_toml_str(dep.version)}," + f" lookback = {_toml_str(dep.lookback)} }}" + ) + else: + lines.append(f"{_toml_key(dep.name)} = {_toml_str(dep.version)}") + + return "\n".join(lines) + "\n" + + +def _validate_builder_script(script: str) -> None: + """Same convention the ci gate enforces: + + parseable, top-level build(dependencies, timestamp). + """ + try: + tree = ast.parse(script) + except SyntaxError as e: + raise InvalidProposalError(f"builder script has a syntax error: {e}") from e + + build_fns = [ + node + for node in tree.body + if isinstance(node, ast.FunctionDef) and node.name == "build" + ] + if not build_fns: + raise InvalidProposalError( + "builder script must define a top-level build() function" + ) + args = build_fns[0].args + positional = args.posonlyargs + args.args + if len(positional) != 2: + raise InvalidProposalError( + "build() must take exactly two arguments (dependencies, timestamp), " + f"got {[a.arg for a in positional]}" + ) + + +def _ruff_output(result: subprocess.CompletedProcess[str]) -> str: + """Combines a ruff run's stdout and stderr into one diagnostic string. + + Violations land on stdout, but a ruff-internal failure (bad config, + unreadable file) reports only on stderr, so reading either stream alone can + produce an empty — and useless — error message. + """ + parts = [part for part in (result.stdout.strip(), result.stderr.strip()) if part] + if not parts: + # ruff produced nothing at all, so at least report the exit code + return f"ruff exited with code {result.returncode} and no output" + return "\n".join(parts) + + +def _lint_builder_script(script: str) -> str: + """Autofix + format the script with ruff so the proposal pr passes repo ci. + + Returns the cleaned script. Raises InvalidProposalError when violations + remain after autofix (the message includes ruff's output). + """ + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "builder.py" + path.write_text(script) + common = [sys.executable, "-m", "ruff"] + check = subprocess.run( + [ + *common, + "check", + "--isolated", + "--select", + RUFF_SELECT, + "--fix", + str(path), + ], + capture_output=True, + text=True, + timeout=RUFF_TIMEOUT_SECONDS, + ) + if check.returncode != 0: + raise InvalidProposalError( + f"builder script fails lint:\n{_ruff_output(check)}" + ) + fmt = subprocess.run( + [*common, "format", "--isolated", str(path)], + capture_output=True, + text=True, + timeout=RUFF_TIMEOUT_SECONDS, + ) + if fmt.returncode != 0: + raise InvalidProposalError( + f"builder script fails formatting:\n{_ruff_output(fmt)}" + ) + return path.read_text() + + +def _validate_against_registry(cfg: DatasetConfig) -> None: + """Cross-checks against currently registered datasets. + + A new dataset is a leaf no existing config references, so it cannot + introduce a cycle — only dep existence, granularity, and start-date + ordering need checking (mirrors registry startup validation). + """ + for dep_name, dep_info in cfg.dependencies.items(): + try: + dep_cfg = registry.get_config(dep_name, dep_info.version) + except ValueError as e: + raise InvalidProposalError( + f"dependency {dep_name}/{dep_info.version} is not a known dataset" + ) from e + if cfg.granularity < dep_cfg.granularity: + raise InvalidProposalError( + f"granularity is finer than dependency {dep_name}/{dep_info.version}" + ) + if cfg.start_date < dep_cfg.start_date: + raise InvalidProposalError( + f"start-date is before dependency {dep_name}/{dep_info.version}'s " + f"start-date ({dep_cfg.start_date.date()})" + ) + + +def _build_pr_body( + proposal: DatasetProposal, requested_by: str, dataset_dir: str +) -> str: + schema_lines = "\n".join( + f"- `{key}`: `{type_}`" for key, type_ in proposal.schema.items() + ) + if proposal.dependencies: + dep_lines = "\n".join( + f"- `{dep.name}/{dep.version}`" + + (f" (lookback `{dep.lookback}`)" if dep.lookback else "") + for dep in proposal.dependencies + ) + else: + dep_lines = "- none (root dataset)" + + body = f"""## Dataset proposal: `{proposal.name}/{proposal.version}` + +{proposal.description.strip()} + +**Proposed by:** {proposal.author_name.strip()} · team **{proposal.team.strip()}** \ +· discord `{proposal.discord_user.strip()}` · api key label `{requested_by}` + +| field | value | +|-------|-------| +| calendar | `{proposal.calendar}` | +| granularity | `{proposal.granularity}` | +| start-date | `{proposal.start_date}` | +| env-vars | `{str(proposal.env_vars).lower()}` | + +**Schema** +{schema_lines} + +**Dependencies** +{dep_lines} + +### Review checklist + +- [ ] builder logic reviewed — this code will run on the datastream server +- [ ] schema and dependencies make sense for the data +""" + if proposal.requirements_txt: + body += "- [ ] `requirements.txt` packages reviewed\n" + if proposal.env_vars: + body += ( + f"- [ ] **before first build**: place the real `.env` at " + f"`{dataset_dir}/.env` on the server " + f"(only `.env.template` is committed; see it for required vars)\n" + ) + return body + + +def propose_dataset( + proposal: DatasetProposal, + requested_by: str, + client: PullRequestOpener | None = None, +) -> ProposalResult: + """Validate a proposal and open a PR adding the dataset directory. + + Raises InvalidProposalError (bad submission), ProposalConflictError + (dataset or proposal branch already exists), or GitHubError (github + unreachable / misconfigured). + """ + if not _NAME_RE.fullmatch(proposal.name): + raise InvalidProposalError( + "dataset name must be lowercase alphanumeric with '-' or '_' " + "(it becomes a directory and branch name)" + ) + for field_name, value in ( + ("author_name", proposal.author_name), + ("team", proposal.team), + ("discord_user", proposal.discord_user), + ("description", proposal.description), + ): + if not value.strip(): + raise InvalidProposalError(f"{field_name} must not be empty") + try: + version = SemVer.parse(proposal.version) + except ValueError as e: + raise InvalidProposalError(f"invalid version: {e}") from e + + try: + registry.get_config(proposal.name, version) + except ValueError: + pass + else: + raise ProposalConflictError( + f"dataset {proposal.name}/{proposal.version} already exists" + ) + + # validate the exact bytes that will be committed, with the same code + # paths the server runs at startup + config_toml = generate_config_toml(proposal) + try: + raw = tomllib.loads(config_toml) + validate_config(raw, proposal.name, version) + normalize_config(raw) + cfg = DatasetConfig.from_raw(raw) + except ValueError as e: + raise InvalidProposalError(str(e)) from e + + _validate_against_registry(cfg) + _validate_builder_script(proposal.builder_script) + # commit the linted/formatted script so the pr passes repo ci + builder_script = _lint_builder_script(proposal.builder_script) + + dataset_dir = f"{SCRIPTS_PREFIX}/{proposal.name}/{proposal.version}" + files = { + f"{dataset_dir}/config.toml": config_toml, + f"{dataset_dir}/builder.py": _ensure_trailing_newline(builder_script), + } + if proposal.requirements_txt and proposal.requirements_txt.strip(): + files[f"{dataset_dir}/requirements.txt"] = _ensure_trailing_newline( + proposal.requirements_txt + ) + if proposal.env_template and proposal.env_template.strip(): + files[f"{dataset_dir}/.env.template"] = _ensure_trailing_newline( + proposal.env_template + ) + + branch = f"add-dataset/{proposal.name}-{proposal.version}" + title = f"feat: add dataset {proposal.name}/{proposal.version}" + body = _build_pr_body(proposal, requested_by, dataset_dir) + + github: PullRequestOpener = ( + client if client is not None else GitHubClient.from_env() + ) + reviewers = [ + login.strip() + for login in os.environ.get("GITHUB_REVIEWERS", DEFAULT_REVIEWERS).split(",") + if login.strip() + ] + try: + pr_url = github.open_pr_with_files( + branch=branch, + base="main", + title=title, + body=body, + commit_message=title, + files=files, + reviewers=reviewers, + ) + except BranchAlreadyExistsError as e: + raise ProposalConflictError( + f"a proposal for {proposal.name}/{proposal.version} is already open " + f"(branch '{branch}' exists)" + ) from e + + logger.info( + "dataset proposal submitted", + dataset=proposal.name, + version=proposal.version, + requested_by=requested_by, + pr_url=pr_url, + ) + return ProposalResult(pr_url=pr_url, branch=branch) + + +def _ensure_trailing_newline(text: str) -> str: + return text if text.endswith("\n") else text + "\n" diff --git a/builders/server/pyproject.toml b/builders/server/pyproject.toml index b2646772..f46cb5dc 100644 --- a/builders/server/pyproject.toml +++ b/builders/server/pyproject.toml @@ -12,6 +12,7 @@ dependencies = [ "exchange-calendars>=4.13", "python-dotenv>=1.0.0", "structlog>=24.0.0", + "ruff>=0.15.21", ] [dependency-groups] diff --git a/builders/server/tests/core/api/test_proposal_route.py b/builders/server/tests/core/api/test_proposal_route.py new file mode 100644 index 00000000..e0e30e30 --- /dev/null +++ b/builders/server/tests/core/api/test_proposal_route.py @@ -0,0 +1,91 @@ +import core.api.routes as routes +import pytest +from core.github.client import GitHubError +from core.service.proposals import ( + InvalidProposalError, + ProposalConflictError, + ProposalResult, +) +from fastapi.testclient import TestClient +from main import app + +client: TestClient = TestClient(app) + +PAYLOAD = { + "name": "my-dataset", + "version": "0.1.0", + "calendar": "everyday", + "granularity": "1d", + "start_date": "2022-01-01", + "schema": {"price": "float"}, + "builder_script": "def build(deps, ts):\n return []\n", + "author_name": "Kai Zhang", + "team": "quant", + "discord_user": "kai#1234", + "description": "test data", +} + + +def _propose_stub(result=None, error=None): + def stub(proposal, requested_by, client=None): + if error is not None: + raise error + return result + + return stub + + +def test_propose_returns_pr_url(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + routes, + "propose_dataset", + _propose_stub( + result=ProposalResult( + pr_url="https://github.com/acme/data/pull/42", + branch="add-dataset/my-dataset-0.1.0", + ) + ), + ) + res = client.post("/api/v1/datasets", json=PAYLOAD) + assert res.status_code == 200 + body = res.json() + assert body["pr_url"] == "https://github.com/acme/data/pull/42" + assert body["branch"] == "add-dataset/my-dataset-0.1.0" + assert body["dataset_name"] == "my-dataset" + + +def test_invalid_proposal_maps_to_400(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + routes, + "propose_dataset", + _propose_stub(error=InvalidProposalError("bad schema")), + ) + res = client.post("/api/v1/datasets", json=PAYLOAD) + assert res.status_code == 400 + assert "bad schema" in res.json()["detail"] + + +def test_conflict_maps_to_409(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + routes, + "propose_dataset", + _propose_stub(error=ProposalConflictError("already exists")), + ) + res = client.post("/api/v1/datasets", json=PAYLOAD) + assert res.status_code == 409 + + +def test_github_failure_maps_to_502(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + routes, + "propose_dataset", + _propose_stub(error=GitHubError(500, "boom")), + ) + res = client.post("/api/v1/datasets", json=PAYLOAD) + assert res.status_code == 502 + assert "github error" in res.json()["detail"] + + +def test_missing_fields_rejected_by_validation() -> None: + res = client.post("/api/v1/datasets", json={"name": "x"}) + assert res.status_code == 422 diff --git a/builders/server/tests/core/github/__init__.py b/builders/server/tests/core/github/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/builders/server/tests/core/github/test_client.py b/builders/server/tests/core/github/test_client.py new file mode 100644 index 00000000..39b82264 --- /dev/null +++ b/builders/server/tests/core/github/test_client.py @@ -0,0 +1,264 @@ +import json + +import pytest +import requests +from core.github.client import ( + BranchAlreadyExistsError, + GitHubClient, + GitHubError, +) +from requests.adapters import BaseAdapter + + +class FakeAdapter(BaseAdapter): + """records outgoing requests and plays back canned (status, body) responses.""" + + def __init__(self, responses: list[tuple[int, dict]]): + super().__init__() + self.responses = list(responses) + self.sent: list[requests.PreparedRequest] = [] + + def send(self, request, **kwargs): # noqa: ANN001 -- test double + self.sent.append(request) + status, body = self.responses.pop(0) + res = requests.Response() + res.status_code = status + res._content = json.dumps(body).encode() + res.headers["Content-Type"] = "application/json" + res.request = request + return res + + def close(self): + pass + + +def _client( + responses: list[tuple[int, dict]], +) -> tuple[GitHubClient, FakeAdapter]: + session = requests.Session() + adapter = FakeAdapter(responses) + session.mount("https://", adapter) + return GitHubClient(token="tok", repo="acme/data", session=session), adapter + + +def _body(request: requests.PreparedRequest) -> dict: + assert request.body is not None + return dict(json.loads(request.body)) + + +def test_open_pr_with_files_happy_path() -> None: + """the full flow: read base sha, tree, commit, branch ref, pr.""" + client, adapter = _client( + [ + (200, {"object": {"sha": "base-sha"}}), + (201, {"sha": "tree-sha"}), + (201, {"sha": "commit-sha"}), + (201, {"ref": "refs/heads/add-dataset/foo-0.1.0"}), + (201, {"html_url": "https://github.com/acme/data/pull/7", "number": 7}), + ] + ) + + url = client.open_pr_with_files( + branch="add-dataset/foo-0.1.0", + base="main", + title="feat: add dataset foo/0.1.0", + body="body", + commit_message="feat: add dataset foo/0.1.0", + files={"b/two.py": "print(2)\n", "a/one.toml": "x = 1\n"}, + ) + + assert url == "https://github.com/acme/data/pull/7" + paths = [(req.method, req.path_url) for req in adapter.sent] + assert paths == [ + ("GET", "/repos/acme/data/git/ref/heads/main"), + ("POST", "/repos/acme/data/git/trees"), + ("POST", "/repos/acme/data/git/commits"), + ("POST", "/repos/acme/data/git/refs"), + ("POST", "/repos/acme/data/pulls"), + ] + + assert adapter.sent[0].headers["Authorization"] == "Bearer tok" + + tree_body = _body(adapter.sent[1]) + assert tree_body["base_tree"] == "base-sha" + # files are sorted by path for deterministic trees + assert [entry["path"] for entry in tree_body["tree"]] == ["a/one.toml", "b/two.py"] + + commit_body = _body(adapter.sent[2]) + assert commit_body == { + "message": "feat: add dataset foo/0.1.0", + "tree": "tree-sha", + "parents": ["base-sha"], + } + + ref_body = _body(adapter.sent[3]) + assert ref_body == { + "ref": "refs/heads/add-dataset/foo-0.1.0", + "sha": "commit-sha", + } + + pr_body = _body(adapter.sent[4]) + assert pr_body["head"] == "add-dataset/foo-0.1.0" + assert pr_body["base"] == "main" + + +_PR_CREATED = ( + 201, + { + "html_url": "https://github.com/acme/data/pull/7", + "number": 7, + "user": {"login": "bot-account"}, + }, +) + + +def test_reviewers_requested_after_pr_created() -> None: + client, adapter = _client( + [ + (200, {"object": {"sha": "base-sha"}}), + (201, {"sha": "tree-sha"}), + (201, {"sha": "commit-sha"}), + (201, {"ref": "refs/heads/b"}), + _PR_CREATED, + (201, {"requested_reviewers": []}), + ] + ) + client.open_pr_with_files( + branch="b", + base="main", + title="t", + body="b", + commit_message="m", + files={"a": "1"}, + reviewers=["alice", "bob"], + ) + assert adapter.sent[-1].path_url == "/repos/acme/data/pulls/7/requested_reviewers" + assert _body(adapter.sent[-1]) == {"reviewers": ["alice", "bob"]} + + +def test_pr_author_excluded_from_reviewers() -> None: + """the token's account cannot review its own pr; it is filtered out.""" + client, adapter = _client( + [ + (200, {"object": {"sha": "base-sha"}}), + (201, {"sha": "tree-sha"}), + (201, {"sha": "commit-sha"}), + (201, {"ref": "refs/heads/b"}), + _PR_CREATED, + (201, {"requested_reviewers": []}), + ] + ) + client.open_pr_with_files( + branch="b", + base="main", + title="t", + body="b", + commit_message="m", + files={"a": "1"}, + reviewers=["alice", "Bot-Account"], + ) + assert _body(adapter.sent[-1]) == {"reviewers": ["alice"]} + + +def test_author_only_reviewer_list_skips_request() -> None: + client, adapter = _client( + [ + (200, {"object": {"sha": "base-sha"}}), + (201, {"sha": "tree-sha"}), + (201, {"sha": "commit-sha"}), + (201, {"ref": "refs/heads/b"}), + _PR_CREATED, + ] + ) + client.open_pr_with_files( + branch="b", + base="main", + title="t", + body="b", + commit_message="m", + files={"a": "1"}, + reviewers=["bot-account"], + ) + # no reviewer call was made: the pr creation is the last request + assert adapter.sent[-1].path_url == "/repos/acme/data/pulls" + + +def test_batch_reviewer_failure_retries_individually() -> None: + """one invalid login must not drop the valid reviewers with it.""" + client, adapter = _client( + [ + (200, {"object": {"sha": "base-sha"}}), + (201, {"sha": "tree-sha"}), + (201, {"sha": "commit-sha"}), + (201, {"ref": "refs/heads/b"}), + _PR_CREATED, + (422, {"message": "Reviews may only be requested from collaborators."}), + (201, {"requested_reviewers": []}), + (422, {"message": "Reviews may only be requested from collaborators."}), + ] + ) + url = client.open_pr_with_files( + branch="b", + base="main", + title="t", + body="b", + commit_message="m", + files={"a": "1"}, + reviewers=["alice", "not-a-collaborator"], + ) + assert url == "https://github.com/acme/data/pull/7" + # batch failed, then each reviewer was retried on its own + reviewer_calls = [ + _body(req) + for req in adapter.sent + if req.path_url.endswith("/requested_reviewers") + ] + assert reviewer_calls == [ + {"reviewers": ["alice", "not-a-collaborator"]}, + {"reviewers": ["alice"]}, + {"reviewers": ["not-a-collaborator"]}, + ] + + +def test_api_error_carries_status_and_message() -> None: + client, _ = _client([(404, {"message": "Not Found"})]) + with pytest.raises(GitHubError) as exc_info: + client.get_branch_sha("main") + assert exc_info.value.status == 404 + assert "Not Found" in exc_info.value.message + + +def test_existing_branch_raises_specific_error() -> None: + """the git refs api reports an existing branch as a 422.""" + client, _ = _client( + [ + (200, {"object": {"sha": "base-sha"}}), + (201, {"sha": "tree-sha"}), + (201, {"sha": "commit-sha"}), + (422, {"message": "Reference already exists"}), + ] + ) + with pytest.raises(BranchAlreadyExistsError): + client.open_pr_with_files( + branch="add-dataset/foo-0.1.0", + base="main", + title="t", + body="b", + commit_message="m", + files={"a": "1"}, + ) + + +def test_from_env_requires_token(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("GITHUB_TOKEN", raising=False) + with pytest.raises(GitHubError): + GitHubClient.from_env() + + +def test_from_env_reads_repo_and_api_root(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("GITHUB_TOKEN", "tok") + monkeypatch.setenv("GITHUB_REPO", "acme/data") + monkeypatch.setenv("GITHUB_API_URL", "http://localhost:9999") + client = GitHubClient.from_env() + assert client.repo == "acme/data" + assert client._api_root == "http://localhost:9999" diff --git a/builders/server/tests/core/service/test_proposals.py b/builders/server/tests/core/service/test_proposals.py new file mode 100644 index 00000000..2fd1f9e4 --- /dev/null +++ b/builders/server/tests/core/service/test_proposals.py @@ -0,0 +1,349 @@ +import subprocess +import tomllib +from pathlib import Path +from typing import Any + +import core.runtime.registry as registry +import core.service.proposals as proposals +import pytest +from core.github.client import BranchAlreadyExistsError +from core.service.proposals import ( + DatasetProposal, + InvalidProposalError, + ProposalConflictError, + ProposedDependency, + generate_config_toml, + propose_dataset, +) + +_DEP_CONFIG = """\ +name = "mock-dep" +version = "0.1.0" +granularity = "1d" +start-date = "2021-06-01" +calendar = "everyday" + +[schema] +price = "int" +""" + +VALID_BUILDER = """\ +from datetime import datetime + + +def build(dependencies, timestamp: datetime) -> list[dict]: + return [{"ticker": "AAPL", "price": 1}] +""" + + +class FakeGitHub: + """stands in for GitHubClient; records the pr call or raises.""" + + def __init__(self, error: Exception | None = None): + self.error = error + self.calls: list[dict[str, Any]] = [] + + def open_pr_with_files( + self, + branch: str, + base: str, + title: str, + body: str, + commit_message: str, + files: dict[str, str], + reviewers: list[str] | None = None, + ) -> str: + if self.error is not None: + raise self.error + self.calls.append( + { + "branch": branch, + "base": base, + "title": title, + "body": body, + "commit_message": commit_message, + "files": files, + "reviewers": reviewers, + } + ) + return "https://github.com/acme/data/pull/42" + + +@pytest.fixture(autouse=True) +def _registry_with_dep(tmp_path: Path): + """populate the registry with mock-dep/0.1.0; reset afterwards.""" + dep_dir = tmp_path / "mock-dep" / "0.1.0" + dep_dir.mkdir(parents=True) + (dep_dir / "config.toml").write_text(_DEP_CONFIG) + registry.load_all_configs(tmp_path) + yield + registry._CONFIG_REGISTRY = {} + + +def _proposal(**overrides: Any) -> DatasetProposal: + defaults: dict[str, Any] = dict( + name="my-dataset", + version="0.1.0", + calendar="everyday", + granularity="1d", + start_date="2022-01-01", + schema={"ticker": "str", "price": "float"}, + builder_script=VALID_BUILDER, + author_name="Kai Zhang", + team="quant", + discord_user="kai#1234", + description="daily test data for the proposal flow", + ) + defaults.update(overrides) + return DatasetProposal(**defaults) + + +def test_happy_path_opens_pr_with_files() -> None: + github = FakeGitHub() + result = propose_dataset(_proposal(), requested_by="team-a", client=github) + + assert result.pr_url == "https://github.com/acme/data/pull/42" + assert result.branch == "add-dataset/my-dataset-0.1.0" + + call = github.calls[0] + assert call["base"] == "main" + assert call["title"] == "feat: add dataset my-dataset/0.1.0" + files = call["files"] + assert set(files) == { + "builders/scripts/my-dataset/0.1.0/config.toml", + "builders/scripts/my-dataset/0.1.0/builder.py", + } + assert "team-a" in call["body"] + # proposer identity and purpose are surfaced for reviewers + assert "Kai Zhang" in call["body"] + assert "quant" in call["body"] + assert "kai#1234" in call["body"] + assert "daily test data for the proposal flow" in call["body"] + + # the committed config parses and carries the submitted fields + raw = tomllib.loads(files["builders/scripts/my-dataset/0.1.0/config.toml"]) + assert raw["name"] == "my-dataset" + assert raw["schema"] == {"ticker": "str", "price": "float"} + + +def test_dependency_with_lookback_round_trips() -> None: + github = FakeGitHub() + proposal = _proposal( + dependencies=[ + ProposedDependency(name="mock-dep", version="0.1.0", lookback="5d") + ] + ) + propose_dataset(proposal, requested_by="team-a", client=github) + + config = github.calls[0]["files"]["builders/scripts/my-dataset/0.1.0/config.toml"] + raw = tomllib.loads(config) + assert raw["dependencies"]["mock-dep"] == {"version": "0.1.0", "lookback": "5d"} + + +def test_optional_files_included_when_present() -> None: + github = FakeGitHub() + proposal = _proposal( + env_vars=True, + requirements_txt="pandas>=2.0\n", + env_template="API_KEY=\n", + ) + propose_dataset(proposal, requested_by="team-a", client=github) + + call = github.calls[0] + files = call["files"] + assert "builders/scripts/my-dataset/0.1.0/requirements.txt" in files + assert "builders/scripts/my-dataset/0.1.0/.env.template" in files + # the env checklist reminds reviewers secrets are placed manually + assert ".env" in call["body"] + assert "before first build" in call["body"] + + +def test_env_file_itself_is_never_committed() -> None: + github = FakeGitHub() + propose_dataset( + _proposal(env_vars=True, env_template="API_KEY=\n"), + requested_by="team-a", + client=github, + ) + files = github.calls[0]["files"] + assert "builders/scripts/my-dataset/0.1.0/.env" not in files + + +def test_unknown_dependency_rejected() -> None: + proposal = _proposal( + dependencies=[ProposedDependency(name="nope", version="9.9.9")] + ) + with pytest.raises(InvalidProposalError, match="not a known dataset"): + propose_dataset(proposal, requested_by="t", client=FakeGitHub()) + + +def test_granularity_finer_than_dependency_rejected() -> None: + proposal = _proposal( + granularity="1h", + dependencies=[ProposedDependency(name="mock-dep", version="0.1.0")], + ) + with pytest.raises(InvalidProposalError, match="finer than dependency"): + propose_dataset(proposal, requested_by="t", client=FakeGitHub()) + + +def test_start_date_before_dependency_rejected() -> None: + proposal = _proposal( + start_date="2020-01-01", + dependencies=[ProposedDependency(name="mock-dep", version="0.1.0")], + ) + with pytest.raises(InvalidProposalError, match="start-date"): + propose_dataset(proposal, requested_by="t", client=FakeGitHub()) + + +def test_existing_dataset_conflicts() -> None: + proposal = _proposal(name="mock-dep", version="0.1.0", start_date="2021-06-01") + with pytest.raises(ProposalConflictError, match="already exists"): + propose_dataset(proposal, requested_by="t", client=FakeGitHub()) + + +def test_existing_branch_conflicts() -> None: + github = FakeGitHub(error=BranchAlreadyExistsError(422, "Reference already exists")) + with pytest.raises(ProposalConflictError, match="already open"): + propose_dataset(_proposal(), requested_by="t", client=github) + + +@pytest.mark.parametrize( + "bad_name", ["MyDataset", "my dataset", "../escape", "-leading", ""] +) +def test_invalid_names_rejected(bad_name: str) -> None: + with pytest.raises(InvalidProposalError, match="dataset name"): + propose_dataset(_proposal(name=bad_name), requested_by="t", client=FakeGitHub()) + + +@pytest.mark.parametrize( + "field_name", ["author_name", "team", "discord_user", "description"] +) +def test_blank_proposer_fields_rejected(field_name: str) -> None: + with pytest.raises(InvalidProposalError, match=field_name): + propose_dataset( + _proposal(**{field_name: " "}), requested_by="t", client=FakeGitHub() + ) + + +def test_invalid_version_rejected() -> None: + with pytest.raises(InvalidProposalError, match="version"): + propose_dataset( + _proposal(version="not-semver"), requested_by="t", client=FakeGitHub() + ) + + +def test_invalid_schema_type_rejected() -> None: + with pytest.raises(InvalidProposalError): + propose_dataset( + _proposal(schema={"price": "decimal"}), + requested_by="t", + client=FakeGitHub(), + ) + + +def test_unknown_calendar_rejected() -> None: + with pytest.raises(InvalidProposalError): + propose_dataset( + _proposal(calendar="lunar"), requested_by="t", client=FakeGitHub() + ) + + +def test_builder_syntax_error_rejected() -> None: + with pytest.raises(InvalidProposalError, match="syntax error"): + propose_dataset( + _proposal(builder_script="def build(:\n"), + requested_by="t", + client=FakeGitHub(), + ) + + +def test_builder_without_build_rejected() -> None: + with pytest.raises(InvalidProposalError, match="build\\(\\)"): + propose_dataset( + _proposal(builder_script="def make(a, b):\n return []\n"), + requested_by="t", + client=FakeGitHub(), + ) + + +def test_builder_wrong_arity_rejected() -> None: + with pytest.raises(InvalidProposalError, match="two arguments"): + propose_dataset( + _proposal(builder_script="def build(only_one):\n return []\n"), + requested_by="t", + client=FakeGitHub(), + ) + + +def test_default_reviewers_requested(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("GITHUB_REVIEWERS", raising=False) + github = FakeGitHub() + propose_dataset(_proposal(), requested_by="t", client=github) + assert github.calls[0]["reviewers"] == ["Blackgaurd", "Scr4tch587"] + + +def test_reviewers_env_override(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("GITHUB_REVIEWERS", "alice, bob") + github = FakeGitHub() + propose_dataset(_proposal(), requested_by="t", client=github) + assert github.calls[0]["reviewers"] == ["alice", "bob"] + + +def test_builder_script_is_autofixed_before_commit() -> None: + """unused imports and formatting are cleaned server-side so the pr passes ci.""" + github = FakeGitHub() + script = ( + "from datetime import datetime\n" + "from typing import Any\n\n\n" + "def build(dependencies, timestamp):\n" + " return [{'ticker': 'AAPL', 'price': 1.0}]\n" + ) + propose_dataset(_proposal(builder_script=script), requested_by="t", client=github) + committed = github.calls[0]["files"]["builders/scripts/my-dataset/0.1.0/builder.py"] + assert "from typing import Any" not in committed # unused import removed + assert "from datetime import datetime" not in committed + assert '"AAPL"' in committed # ruff format normalizes quotes + + +def test_unfixable_lint_error_rejected() -> None: + """violations ruff cannot autofix (e.g. undefined name) reject the proposal.""" + script = "def build(dependencies, timestamp):\n return [undefined_var]\n" + with pytest.raises(InvalidProposalError, match="fails lint"): + propose_dataset( + _proposal(builder_script=script), requested_by="t", client=FakeGitHub() + ) + + +def test_lint_error_surfaces_ruff_stderr(monkeypatch: pytest.MonkeyPatch) -> None: + """a ruff failure that only writes to stderr still reaches the caller.""" + + def fake_run(*_args: object, **_kwargs: object) -> subprocess.CompletedProcess[str]: + return subprocess.CompletedProcess( + args=[], returncode=2, stdout="", stderr="ruff failed: bad config" + ) + + monkeypatch.setattr(proposals.subprocess, "run", fake_run) + with pytest.raises(InvalidProposalError, match="ruff failed: bad config"): + propose_dataset(_proposal(), requested_by="t", client=FakeGitHub()) + + +def test_lint_error_reports_exit_code_when_ruff_is_silent( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """no output on either stream still produces a non-empty diagnostic.""" + + def fake_run(*_args: object, **_kwargs: object) -> subprocess.CompletedProcess[str]: + return subprocess.CompletedProcess( + args=[], returncode=101, stdout="", stderr="" + ) + + monkeypatch.setattr(proposals.subprocess, "run", fake_run) + with pytest.raises(InvalidProposalError, match="exited with code 101"): + propose_dataset(_proposal(), requested_by="t", client=FakeGitHub()) + + +def test_generated_toml_quotes_awkward_schema_keys() -> None: + """schema keys that aren't bare toml keys are quoted, not mangled.""" + proposal = _proposal(schema={"has space": "str", "ok_key": "int"}) + raw = tomllib.loads(generate_config_toml(proposal)) + assert raw["schema"] == {"has space": "str", "ok_key": "int"} diff --git a/dev-docs/SPEC-backend.md b/dev-docs/SPEC-backend.md index eb8869c9..02a838cc 100644 --- a/dev-docs/SPEC-backend.md +++ b/dev-docs/SPEC-backend.md @@ -28,6 +28,10 @@ GET /data/{dataset_name}/{dataset_version}?start=&end=&bui DELETE /data/{dataset_name}/{dataset_version}?start=&end= ``` +``` +POST /datasets +``` + ### API authentication Every endpoint except `GET /status` requires a valid API key sent as a bearer token: @@ -58,7 +62,16 @@ This prints a `dsk_`-prefixed raw key (hand to the client) and the `label:hash` |--------|---------| | `401` | Missing or invalid API key (on any endpoint except `/status`) | -**Clients.** The Python SDK sends the header automatically when given a key: pass `DatastreamClient(api_key=...)` (or the module-level `get_data(..., api_key=...)`), set it globally via `configure(api_key=...)`, or export `DATASTREAM_API_KEY`. The browser frontend does **not** yet send a key, so its requests currently return `401`; browser auth is handled in a later change (see `SPEC-frontend.md`). +**Clients.** The Python SDK sends the header automatically when given a key: pass `DatastreamClient(api_key=...)` (or the module-level `get_data(..., api_key=...)`), set it globally via `configure(api_key=...)`, or export `DATASTREAM_API_KEY`. The browser frontend prompts the user for a key and sends it on every request (see `SPEC-frontend.md`). + +### CORS + +The browser frontend is served from GitHub Pages — a different origin than the API — so `main.py` adds Starlette's `CORSMiddleware`: + +- **Allowed origins** come from the `CORS_ALLOW_ORIGINS` env var (comma-separated exact origins). Default: `https://wat-street.github.io,http://localhost:5173` (the Pages origin and the local Vite dev server). Origins never include a path — the Pages origin is `https://wat-street.github.io`, not `.../datastream`. +- The middleware is added last, so it wraps outermost: preflight `OPTIONS` requests short-circuit before auth and request-context logging. +- `allow_credentials` stays off (auth is a bearer header, not cookies); allowed headers are `Authorization` and `Content-Type`; all methods are allowed. +- **Auth is unchanged.** CORS only permits browsers to send the `Authorization` header cross-origin; every non-public route still requires a valid API key. ### Datasets endpoint @@ -170,6 +183,38 @@ Each entry in `rows` contains all data dicts for that timestamp (matching the DB | `404` | Dataset exists but has no rows in the requested range | | `500` | Unexpected failure (DB error) | +### Dataset proposals endpoint + +`POST /datasets` submits a new dataset for review. **The server never writes to its own scripts directory**: it validates the submission, generates the dataset files, and opens a GitHub pull request adding `builders/scripts///` to the repo. The dataset goes live only after the PR is reviewed, merged, and the server restarts. This keeps code review as the trust gate — a builder script is code that executes on the server, and the "internal users write trusted builders" assumption (see "Build behavior") is enforced by human review, not by the API key alone. + +**Request body** (JSON): `name`, `version`, `calendar`, `granularity`, `start_date`, `schema` (field → type map), `builder_script`, proposer identity (`author_name`, `team`, `discord_user`, `description` — all required, surfaced in the PR body), and optional `dependencies` (list of `{name, version, lookback?}`), `env_vars`, `requirements_txt`, `env_template`. + +**Validation** (`core/service/proposals.py`), in order: +1. name must match `^[a-z0-9][a-z0-9_-]*$` (it becomes a directory and branch name); version must parse as SemVer; proposer fields must be non-empty +2. `(name, version)` must not already exist in the config registry +3. the canonical `config.toml` is generated, then **those exact bytes** are re-parsed and run through the same `validate_config` used at startup — the committed config is byte-identical to what was validated +4. registry cross-checks against live configs: dependencies exist, granularity is not finer than any dependency's, start-date is not before any dependency's (a new dataset is a leaf nothing references, so cycles are impossible) +5. builder script: AST-parsed, must define a top-level `build(dependencies, timestamp)` with exactly two positional args +6. the script is then run through **ruff autofix + format** server-side (same rule selection as repo CI — kept in sync manually with the root `pyproject.toml`); unfixable violations reject the proposal with ruff's output. Proposal PRs therefore land pre-linted and cannot fail the CI lint gate. The rejection message combines **both** of ruff's streams (violations go to stdout, but a ruff-internal failure such as a bad config reports only on stderr) and falls back to the exit code if ruff printed nothing, so the caller and the server log never get a bare "fails lint:" with no diagnostic + +**PR shape**: branch `add-dataset/-` off `main`, one commit containing `config.toml`, `builder.py`, and (when provided) `requirements.txt` / `.env.template`. **The real `.env` is never committed** — for `env-vars = true` datasets the PR body carries a checklist item to place it on the server manually before the first build. Title: `feat: add dataset /`. Reviewers are auto-requested (best-effort: the PR author is excluded, and a failed batch falls back to per-reviewer requests). + +**Configuration** (env vars, documented in `infra/.env.template`): +- `GITHUB_TOKEN` (required for this endpoint only): PAT with Contents + Pull requests read/write on the repo. Without it the endpoint returns 502; everything else works. +- `GITHUB_REPO` (default `Wat-Street/datastream`), `GITHUB_API_URL` (default `https://api.github.com`; overridable for testing against a fake), `GITHUB_REVIEWERS` (default `Blackgaurd,Scr4tch587`). + +**Response** (200): `{"dataset_name", "dataset_version", "pr_url", "branch"}`. + +| Status | Meaning | +|--------|---------| +| `400` | Invalid submission (validation or lint failure; `detail` is safe to show in the UI) | +| `401` | Missing or invalid API key | +| `409` | Dataset already registered, or a proposal branch for it is already open | +| `422` | Malformed request body (missing/mistyped fields) | +| `502` | GitHub unreachable or `GITHUB_TOKEN` missing/invalid | + +The requesting API key's team label is logged and included in the PR body alongside the proposer fields. A CI test (`tests/core/runtime/test_real_scripts.py`) validates every real config in `builders/scripts/` with the startup validation path, so green CI on a proposal PR means the dataset cannot break server boot. + ### Build behavior - On a `POST /build` request, it builds missing data for the requested range and writes it to the database. diff --git a/dev-docs/SPEC-frontend.md b/dev-docs/SPEC-frontend.md index a3a3dd5c..7ef940e8 100644 --- a/dev-docs/SPEC-frontend.md +++ b/dev-docs/SPEC-frontend.md @@ -1,105 +1,124 @@ # Datastream Frontend -A lightweight internal UI built with Svelte 5 + Vite, using Bun as the package manager. +A lightweight internal UI built with React 19 + TypeScript + Vite, using Bun as the package manager and JS runtime (`bunx --bun vite`, so the installer and runtime always agree on native-binding architecture). - **Local dev**: `just frontend-dev` starts the Vite dev server on port 5173. The Vite config proxies `/api` to `http://localhost:3000` (the Python FastAPI server) to avoid CORS issues. -- **Docker**: the frontend is built as static files (`bun run build`) and served by nginx on port 80. nginx proxies `/api` to `http://builder:3000` via Docker internal DNS. -- The frontend is not containerized -- it runs locally via `just frontend-dev` and proxies to the backend container on port 3000. +- **Production**: deployed to **GitHub Pages** at `https://wat-street.github.io/datastream/` by `.github/workflows/deploy-pages.yml` (see "Deployment" below). The frontend is not containerized and nothing in `infra/` serves it — Caddy only proxies the API. ## Design This is a purely internal tool. The goal is functional clarity, not aesthetics. -- **No CSS framework or component library** -- scoped Svelte ` diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx new file mode 100644 index 00000000..e21c89e8 --- /dev/null +++ b/frontend/src/App.tsx @@ -0,0 +1,80 @@ +import { KeyRoundIcon, PlusIcon } from "lucide-react"; +import { lazy, Suspense, useState } from "react"; + +import type { DatasetSummary } from "@/lib/api"; + +import { ApiKeyDialog } from "@/components/api-key-dialog"; +import { DatasetDetail } from "@/components/dataset-detail"; +import { DatasetList } from "@/components/dataset-list"; +import { Button } from "@/components/ui/button"; +import { useApiKey } from "@/hooks/use-api-key"; + +// lazy: the create view pulls in codemirror, which shouldn't weigh down browsing +const CreateDataset = lazy(() => + import("@/components/create-dataset").then((module) => ({ + default: module.CreateDataset, + })), +); + +type View = + | { view: "list" } + | { view: "detail"; dataset: DatasetSummary } + | { view: "create" }; + +function App() { + const { setDialogOpen } = useApiKey(); + const [view, setView] = useState({ view: "list" }); + + return ( +
+
+

Datastream

+
+ {view.view === "list" && ( + + )} + +
+
+
+ {view.view === "list" && ( + setView({ view: "detail", dataset })} + /> + )} + {view.view === "detail" && ( + setView({ view: "list" })} + /> + )} + {view.view === "create" && ( + loading...

+ } + > + setView({ view: "list" })} /> + + )} +
+ +
+ ); +} + +export default App; diff --git a/frontend/src/app.css b/frontend/src/app.css deleted file mode 100644 index b31d73fa..00000000 --- a/frontend/src/app.css +++ /dev/null @@ -1,60 +0,0 @@ -/* reset */ -*, -*::before, -*::after { - box-sizing: border-box; - margin: 0; - padding: 0; -} - -:root { - /* colors */ - --color-bg: #0f1117; - --color-surface: #1a1d27; - --color-surface-hover: #242836; - --color-border: #2e3345; - --color-text: #e1e4ed; - --color-text-muted: #8b90a0; - --color-accent: #6c8cff; - --color-accent-hover: #8da6ff; - --color-success: #4caf7c; - --color-error: #e05555; - - /* json highlighting */ - --json-key: #6c8cff; - --json-string: #4caf7c; - --json-number: #e0a356; - --json-boolean: #c084fc; - --json-null: #8b90a0; - - /* typography */ - --font-sans: - "Inter", -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; - --font-mono: "JetBrains Mono", "Fira Code", "Consolas", monospace; - - /* spacing */ - --space-xs: 0.25rem; - --space-sm: 0.5rem; - --space-md: 1rem; - --space-lg: 1.5rem; - --space-xl: 2rem; - - /* radii */ - --radius-sm: 4px; - --radius-md: 8px; - --radius-lg: 12px; -} - -body { - font-family: var(--font-sans); - background: var(--color-bg); - color: var(--color-text); - line-height: 1.5; - -webkit-font-smoothing: antialiased; -} - -#app { - max-width: 1200px; - margin: 0 auto; - padding: var(--space-lg) var(--space-xl); -} diff --git a/frontend/src/components/DataTable.svelte b/frontend/src/components/DataTable.svelte deleted file mode 100644 index 6ac30bd6..00000000 --- a/frontend/src/components/DataTable.svelte +++ /dev/null @@ -1,98 +0,0 @@ - - -{#if !rows || rows.length === 0} -

no data for this range

-{:else} -
- - - - - {#each columns() as col (col)} - - {/each} - - - - {#each rows as row (row.timestamp)} - {#each row.data as entry, i (i)} - onrowclick({ timestamp: row.timestamp, ...entry })} - > - {#if i === 0} - - {/if} - {#each columns() as col (col)} - - {/each} - - {/each} - {/each} - -
timestamp{col}
{row.timestamp}{entry[col] ?? ""}
-
-{/if} - - diff --git a/frontend/src/components/DatasetDetail.svelte b/frontend/src/components/DatasetDetail.svelte deleted file mode 100644 index 1a76f4ad..00000000 --- a/frontend/src/components/DatasetDetail.svelte +++ /dev/null @@ -1,186 +0,0 @@ - - -
- - -

{name} {version}

- - {#if loading} -

loading...

- {:else if error} -
-

{error}

- -
- {:else if result} -

- {result.returned_timestamps} timestamps · page {page + 1} of {totalPages} -

- - - - {#if totalPages > 1} - - {/if} - {/if} - - {#if modalRow} - (modalRow = null)} /> - {/if} -
- - diff --git a/frontend/src/components/DatasetList.svelte b/frontend/src/components/DatasetList.svelte deleted file mode 100644 index 0f60dcb7..00000000 --- a/frontend/src/components/DatasetList.svelte +++ /dev/null @@ -1,145 +0,0 @@ - - -
- {#if loading} -

loading datasets...

- {:else if error} -
-

{error}

- -
- {:else if datasets.length === 0} -

no datasets found

- {:else} - - - - - - - - - - {#each datasets as dataset (`${dataset.name}@${dataset.version}`)} - onselect(dataset)}> - - - - - {/each} - -
nameversiondata
{dataset.name}{dataset.version} - -
- {/if} -
- - diff --git a/frontend/src/components/JsonModal.svelte b/frontend/src/components/JsonModal.svelte deleted file mode 100644 index bac71d18..00000000 --- a/frontend/src/components/JsonModal.svelte +++ /dev/null @@ -1,98 +0,0 @@ - - - - - -
-
- - 
{@html html}
-
-
- - diff --git a/frontend/src/components/api-key-dialog.tsx b/frontend/src/components/api-key-dialog.tsx new file mode 100644 index 00000000..b999d17e --- /dev/null +++ b/frontend/src/components/api-key-dialog.tsx @@ -0,0 +1,107 @@ +import { zodResolver } from "@hookform/resolvers/zod"; +import { useQueryClient } from "@tanstack/react-query"; +import { useForm } from "react-hook-form"; +import { z } from "zod"; + +import { Button } from "@/components/ui/button"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { + Form, + FormControl, + FormField, + FormItem, + FormLabel, + FormMessage, +} from "@/components/ui/form"; +import { Input } from "@/components/ui/input"; +import { useApiKey } from "@/hooks/use-api-key"; +import { maskApiKey } from "@/lib/api-key"; + +const formSchema = z.object({ + apiKey: z + .string() + .trim() + .regex(/^dsk_/, "keys start with dsk_") + .min(8, "key looks too short"), +}); + +type FormValues = z.infer; + +export function ApiKeyDialog() { + const { apiKey, saveApiKey, clearApiKey, dialogOpen, setDialogOpen } = + useApiKey(); + const queryClient = useQueryClient(); + + const form = useForm({ + resolver: zodResolver(formSchema), + defaultValues: { apiKey: "" }, + }); + + function onSubmit(values: FormValues) { + saveApiKey(values.apiKey.trim()); + form.reset(); + setDialogOpen(false); + // refetch everything that failed without a key + void queryClient.invalidateQueries(); + } + + return ( + + + + API key + + Requests are authenticated with a bearer key. Paste your{" "} + dsk_ key — it is stored only in + this browser. + + + {apiKey && ( +
+ {maskApiKey(apiKey)} + +
+ )} +
+ + ( + + {apiKey ? "Replace key" : "Key"} + + + + + + )} + /> +
+ +
+ + +
+ + ); +} diff --git a/frontend/src/components/create-dataset.tsx b/frontend/src/components/create-dataset.tsx new file mode 100644 index 00000000..56ac379b --- /dev/null +++ b/frontend/src/components/create-dataset.tsx @@ -0,0 +1,741 @@ +import { python } from "@codemirror/lang-python"; +import { zodResolver } from "@hookform/resolvers/zod"; +import { useMutation } from "@tanstack/react-query"; +import CodeMirror from "@uiw/react-codemirror"; +import { + ArrowLeftIcon, + ExternalLinkIcon, + PlusIcon, + RotateCcwIcon, + XIcon, +} from "lucide-react"; +import { useFieldArray, useForm, useWatch } from "react-hook-form"; +import { z } from "zod"; + +import type { DatasetProposalPayload, ProposalResponse } from "@/lib/api"; + +import { Button } from "@/components/ui/button"; +import { Checkbox } from "@/components/ui/checkbox"; +import { + Form, + FormControl, + FormDescription, + FormField, + FormItem, + FormLabel, + FormMessage, +} from "@/components/ui/form"; +import { Input } from "@/components/ui/input"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Textarea } from "@/components/ui/textarea"; +import { useDatasets } from "@/hooks/use-datasets"; +import { proposeDataset } from "@/lib/api"; + +const SCHEMA_TYPES = ["str", "int", "float", "bool"] as const; +const CALENDARS = ["everyday", "weekday", "always-open", "nyse-daily"] as const; +const GRANULARITIES = ["1s", "1m", "1h", "1d"] as const; + +const DEFAULT_BUILDER = `from datetime import datetime +from typing import Any + + +def build( + dependencies: dict[str, dict[datetime, list[dict]]], + timestamp: datetime, +) -> list[dict[str, Any]]: + # return one dict per row for this timestamp, matching the schema + # + # e.g. with a schema of {"ticker": "str", "close": "float"}, a dataset with + # one row per ticker would return: + # + # return [ + # {"ticker": "AAPL", "close": 189.5}, + # {"ticker": "MSFT", "close": 402.1}, + # ] + # + # a single-row dataset returns a list of length 1. dependency data arrives + # as dependencies["dep-name"][timestamp] -> list[dict] + return [] +`; + +const formSchema = z.object({ + name: z + .string() + .regex( + /^[a-z0-9][a-z0-9_-]*$/, + "lowercase alphanumeric with - or _ (becomes the directory name)", + ), + version: z.string().regex(/^\d+\.\d+\.\d+$/, "semver like 0.1.0"), + calendar: z.enum(CALENDARS), + granularity: z.enum(GRANULARITIES), + startDate: z.string().regex(/^\d{4}-\d{2}-\d{2}$/, "YYYY-MM-DD"), + schemaFields: z + .array( + z.object({ + // trimmed so " price" and "price" can't become two different toml keys + key: z.string().trim().min(1, "field name required"), + type: z.enum(SCHEMA_TYPES), + }), + ) + .min(1, "at least one schema field") + // the schema becomes an object, so a duplicate key would silently drop the + // earlier row instead of reaching the server + .superRefine((fields, ctx) => { + const seen = new Set(); + fields.forEach((field, index) => { + if (!field.key) return; + if (seen.has(field.key)) { + ctx.addIssue({ + code: "custom", + message: "duplicate field name", + path: [index, "key"], + }); + return; + } + seen.add(field.key); + }); + }), + dependencies: z.array( + z.object({ + dataset: z.string().min(1, "pick a dataset"), + lookback: z + .string() + .regex(/^\d+[dhms]$/, "like 5d, 24h, 30m, 60s") + .or(z.literal("")), + }), + ), + builderScript: z + .string() + .refine( + (script) => /def\s+build\s*\(/.test(script), + "must define a build() function", + ), + authorName: z.string().trim().min(1, "your name is required"), + team: z.string().trim().min(1, "team is required"), + discordUser: z.string().trim().min(1, "discord username is required"), + description: z + .string() + .trim() + .min(10, "a sentence or two about what this dataset is for"), + envVars: z.boolean(), + requirementsTxt: z.string(), + envTemplate: z.string(), +}); + +type FormValues = z.infer; + +function toPayload(values: FormValues): DatasetProposalPayload { + return { + name: values.name, + version: values.version, + calendar: values.calendar, + granularity: values.granularity, + start_date: values.startDate, + schema: Object.fromEntries( + values.schemaFields.map((field) => [field.key, field.type]), + ), + dependencies: values.dependencies.map((dep) => { + const [name, version] = dep.dataset.split("@"); + return { name, version, lookback: dep.lookback || undefined }; + }), + builder_script: values.builderScript, + author_name: values.authorName, + team: values.team, + discord_user: values.discordUser, + description: values.description, + env_vars: values.envVars, + requirements_txt: values.requirementsTxt.trim() + ? values.requirementsTxt + : undefined, + // gated on envVars too: the textarea is only hidden when the box is + // unchecked, not cleared, so a stale template must not reach the pr + env_template: + values.envVars && values.envTemplate.trim() + ? values.envTemplate + : undefined, + }; +} + +function SectionHeading({ + children, + action, +}: { + children: React.ReactNode; + action?: React.ReactNode; +}) { + return ( +
+

{children}

+ {action} +
+ ); +} + +export function CreateDataset({ onBack }: { onBack: () => void }) { + const { data: datasets } = useDatasets(); + + const form = useForm({ + resolver: zodResolver(formSchema), + // default is "onSubmit", which hides every hint until the first submit + mode: "onBlur", + defaultValues: { + name: "", + version: "0.1.0", + calendar: "everyday", + granularity: "1d", + startDate: "", + authorName: "", + team: "", + discordUser: "", + description: "", + schemaFields: [{ key: "", type: "str" }], + dependencies: [], + builderScript: DEFAULT_BUILDER, + envVars: false, + requirementsTxt: "", + envTemplate: "", + }, + }); + + const schemaRows = useFieldArray({ + control: form.control, + name: "schemaFields", + }); + const depRows = useFieldArray({ + control: form.control, + name: "dependencies", + }); + + const envVarsEnabled = useWatch({ + control: form.control, + name: "envVars", + }); + + const mutation = useMutation({ + mutationFn: (values) => proposeDataset(toPayload(values)), + }); + + if (mutation.isSuccess) { + return ( +
+

Proposal opened

+

+ + {mutation.data.dataset_name}/{mutation.data.dataset_version} + {" "} + was submitted as a pull request. It goes live after review, merge, and + the next server restart. +

+

+ + {mutation.data.pr_url} + +

+ {form.getValues("envVars") && ( +

+ this dataset needs env vars: after merge, someone must place the + real .env on the server (see the + PR checklist) before it can build. +

+ )} +
+ + +
+
+ ); + } + + return ( +
+ +

New dataset

+

+ Submitting opens a pull request with the dataset files — nothing runs + until a reviewer merges it. +

+ +
+ mutation.mutate(values))} + className="space-y-8" + > +
+ Proposer +
+ ( + + Your name + + + + + + )} + /> + ( + + Wat Street team + + + + + + )} + /> + ( + + Discord user + + + + + + )} + /> +
+ ( + + What is this dataset for? + +