diff --git a/api/main.go b/api/main.go index 164cc4ac5..58b738b89 100644 --- a/api/main.go +++ b/api/main.go @@ -199,6 +199,14 @@ func spaHandler(staticDir, assetBucketURL string) http.HandlerFunc { return } + // /.well-known/* are agent/bot discovery endpoints (RFC 8615), never + // SPA routes — 404 when missing instead of returning the SPA shell. + if strings.HasPrefix(r.URL.Path, "/.well-known/") { + setNoCacheHeaders(w) + http.NotFound(w, r) + return + } + // Fallback to root index.html for SPA routing setNoCacheHeaders(w) http.ServeFile(w, r, filepath.Join(staticDir, "index.html")) diff --git a/api/main_test.go b/api/main_test.go new file mode 100644 index 000000000..b4a707a0a --- /dev/null +++ b/api/main_test.go @@ -0,0 +1,62 @@ +package main + +import ( + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" +) + +func TestSpaHandler(t *testing.T) { + dir := t.TempDir() + + mustWrite := func(rel, body string) { + full := filepath.Join(dir, rel) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(full, []byte(body), 0o644); err != nil { + t.Fatalf("write: %v", err) + } + } + + mustWrite("index.html", "spa") + mustWrite("robots.txt", "User-agent: *\n") + mustWrite("assets/app.js", "console.log(1)") + mustWrite(".well-known/mcp/server-card.json", `{"ok":true}`) + + h := spaHandler(dir, "") + + tests := []struct { + name string + path string + wantStatus int + wantBody string + }{ + {"existing file: robots.txt", "/robots.txt", 200, "User-agent: *\n"}, + {"existing file: hashed asset", "/assets/app.js", 200, "console.log(1)"}, + {"existing file: well-known JSON", "/.well-known/mcp/server-card.json", 200, `{"ok":true}`}, + {"SPA route falls back to index.html", "/status", 200, "spa"}, + {"deep SPA route falls back to index.html", "/dz/devices/ABC123", 200, "spa"}, + {"missing whitelisted asset 404s", "/assets/missing.js", 404, ""}, + {"missing well-known with extension 404s", "/.well-known/agent-skills/index.json", 404, ""}, + {"missing well-known WITHOUT extension 404s", "/.well-known/api-catalog", 404, ""}, + {"missing extensionless well-known sub-path 404s", "/.well-known/openid-configuration", 404, ""}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, tt.path, nil) + rec := httptest.NewRecorder() + h(rec, req) + + if rec.Code != tt.wantStatus { + t.Fatalf("status = %d, want %d (body: %q)", rec.Code, tt.wantStatus, rec.Body.String()) + } + if tt.wantBody != "" && rec.Body.String() != tt.wantBody { + t.Fatalf("body = %q, want %q", rec.Body.String(), tt.wantBody) + } + }) + } +} diff --git a/k8s/docker/nginx.conf b/k8s/docker/nginx.conf index 91d952423..4115e2aa2 100644 --- a/k8s/docker/nginx.conf +++ b/k8s/docker/nginx.conf @@ -5,15 +5,24 @@ server { root /usr/share/nginx/html; index index.html; - # SPA fallback — serve index.html for all non-file routes + # Paths that look like files (have an extension) must resolve to a real + # file — otherwise return a proper 404. This prevents soft-404s where + # missing assets like /robots.txt or /sitemap.xml would fall through to + # index.html with a 200 status. + location ~ \.[^/]+$ { + try_files $uri =404; + } + + # SPA fallback — serve index.html for client-side routes (no extension). location / { try_files $uri $uri/ /index.html; } - # Proxy API requests to the api service + # Proxy API requests to the api service. `^~` stops nginx from evaluating + # the regex locations above, so API paths always proxy regardless of shape. # Using a variable forces runtime DNS resolution so nginx doesn't # crash if the api service isn't up yet. - location /api/ { + location ^~ /api/ { set $api_upstream http://api.lake-dev.svc.cluster.local:8080; proxy_pass $api_upstream; proxy_set_header Host $host; diff --git a/web/public/.well-known/mcp/server-card.json b/web/public/.well-known/mcp/server-card.json new file mode 100644 index 000000000..f08fd1d56 --- /dev/null +++ b/web/public/.well-known/mcp/server-card.json @@ -0,0 +1,16 @@ +{ + "serverInfo": { + "name": "doublezero", + "version": "1.0.0" + }, + "description": "DoubleZero Data MCP server. Query the DoubleZero network's ClickHouse analytics database via SQL, explore network topology via Cypher (mainnet-beta only), fetch the live database schema, and read DoubleZero documentation.", + "url": "https://data.malbeclabs.com/api/mcp", + "transport": { + "type": "streamable-http" + }, + "capabilities": { + "tools": true, + "resources": true, + "prompts": true + } +} diff --git a/web/public/llms.txt b/web/public/llms.txt new file mode 100644 index 000000000..75922ec98 --- /dev/null +++ b/web/public/llms.txt @@ -0,0 +1,48 @@ +# DoubleZero Data + +> Analytics platform for the DoubleZero network. Query network telemetry and Solana validator data via SQL or Cypher, explore topology, and inspect network health and performance. + +The primary agent interface is the MCP server described below. The human-facing web app at provides the same data interactively. + +## MCP Server + +- Endpoint: `https://data.malbeclabs.com/api/mcp` +- Transport: streamable-http (stateless) +- Discovery: [/.well-known/mcp/server-card.json](https://data.malbeclabs.com/.well-known/mcp/server-card.json) +- Docs: [/docs/mcp](https://data.malbeclabs.com/docs/mcp) + +All tools are read-only and can be called concurrently. Always call `get_schema` before writing SQL or Cypher — do not assume table or column names. + +### Tools + +- **execute_sql** — Run a read-only SQL query against the DoubleZero ClickHouse analytics database. Accepts `SELECT`, `WITH`, `SHOW`, `DESCRIBE`, `EXPLAIN`. Returns columns, rows, row count, and elapsed ms. +- **execute_cypher** — Run a read-only Cypher query against the DoubleZero Neo4j topology graph. Good for path-finding, reachability, and metro-to-metro latency (SQL only has directly-connected pairs). Available on mainnet-beta only. +- **get_schema** — Return the live ClickHouse schema (tables, columns, types, view definitions) for the current environment. +- **read_docs** — Fetch a DoubleZero documentation page by slug (e.g. `index`, `architecture`, `setup`, `troubleshooting`, `connect`, `connect-multicast`, `contribute`, `paying-fees`). + +### Resources + +- `doublezero://schema` — Live ClickHouse schema for the active environment. +- `doublezero://sql-context` — SQL patterns, business rules, and ClickHouse-specific guidance. +- `doublezero://cypher-context` — Cypher patterns and Neo4j guidance (mainnet-beta only). + +### Prompts + +- **analyze_data** — Ask a natural-language question about the DoubleZero network. Composes the right resource reads and tool calls to answer data questions. + +## Web app sections + +- [Status](https://data.malbeclabs.com/status) — Network, link, device, and metro status. +- [Topology](https://data.malbeclabs.com/topology/map) — Map, graph, and globe views; path calculator; redundancy. +- [Performance](https://data.malbeclabs.com/performance/dz-vs-internet) — DZ vs. internet latency, link latency, path latency. +- [Traffic](https://data.malbeclabs.com/traffic/overview) — Traffic dashboards and per-interface views. +- [Incidents](https://data.malbeclabs.com/incidents/links) — Link and device incidents. +- [DZ entities](https://data.malbeclabs.com/dz/devices) — Devices, links, metros, contributors, tenants, users, multicast groups. +- [Shreds](https://data.malbeclabs.com/dz/shreds/scoreboard) — Scoreboard, publishers, subscribers, funders, devices, activity, economics. +- [Solana](https://data.malbeclabs.com/solana/overview) — Validators and gossip nodes. +- [Changelog](https://data.malbeclabs.com/changelog) — Platform changes. + +## External + +- DoubleZero docs: +- DoubleZero website: diff --git a/web/public/robots.txt b/web/public/robots.txt new file mode 100644 index 000000000..01b8eadda --- /dev/null +++ b/web/public/robots.txt @@ -0,0 +1,8 @@ +User-agent: * +Content-Signal: search=yes, ai-input=yes, ai-train=yes +Allow: / +Disallow: /query/ +Disallow: /chat/ +Disallow: /settings + +Sitemap: https://data.malbeclabs.com/sitemap.xml diff --git a/web/public/sitemap.xml b/web/public/sitemap.xml new file mode 100644 index 000000000..3d451dec1 --- /dev/null +++ b/web/public/sitemap.xml @@ -0,0 +1,44 @@ + + + https://data.malbeclabs.com/chat + https://data.malbeclabs.com/status + https://data.malbeclabs.com/status/links + https://data.malbeclabs.com/status/devices + https://data.malbeclabs.com/status/metros + https://data.malbeclabs.com/status/methodology + https://data.malbeclabs.com/incidents/links + https://data.malbeclabs.com/incidents/devices + https://data.malbeclabs.com/topology/map + https://data.malbeclabs.com/topology/graph + https://data.malbeclabs.com/topology/globe + https://data.malbeclabs.com/topology/path-calculator + https://data.malbeclabs.com/topology/redundancy + https://data.malbeclabs.com/topology/metro-connectivity + https://data.malbeclabs.com/topology/maintenance + https://data.malbeclabs.com/performance/dz-vs-internet + https://data.malbeclabs.com/performance/link-latency + https://data.malbeclabs.com/performance/path-latency + https://data.malbeclabs.com/traffic/overview + https://data.malbeclabs.com/traffic/interfaces + https://data.malbeclabs.com/dz/devices + https://data.malbeclabs.com/dz/links + https://data.malbeclabs.com/dz/metros + https://data.malbeclabs.com/dz/contributors + https://data.malbeclabs.com/dz/tenants + https://data.malbeclabs.com/dz/users + https://data.malbeclabs.com/dz/multicast-groups + https://data.malbeclabs.com/dz/shreds/scoreboard + https://data.malbeclabs.com/dz/shreds/publishers + https://data.malbeclabs.com/dz/shreds/subscribers + https://data.malbeclabs.com/dz/shreds/devices + https://data.malbeclabs.com/dz/shreds/activity + https://data.malbeclabs.com/dz/shreds/economics + https://data.malbeclabs.com/dz/geoloc/probes + https://data.malbeclabs.com/dz/geoloc/users + https://data.malbeclabs.com/solana/overview + https://data.malbeclabs.com/solana/validators + https://data.malbeclabs.com/solana/gossip-nodes + https://data.malbeclabs.com/changelog + https://data.malbeclabs.com/docs/mcp + https://data.malbeclabs.com/terms +