Skip to content

Commit ffe6158

Browse files
ZeynelKocaZeynelKoca
committed
Add file-based dashboard provisioner
1 parent c70429e commit ffe6158

File tree

6 files changed

+585
-0
lines changed

6 files changed

+585
-0
lines changed

‎.changeset/dashboard-provisioner.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"@hyperdx/api": minor
3+
---
4+
5+
feat: add file-based dashboard provisioner that watches a directory for JSON files and upserts dashboards into MongoDB

‎packages/api/docs/auto_provision/AUTO_PROVISION.md‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -163,6 +163,62 @@ services:
163163
For more complex configurations, you can use environment files or Docker secrets
164164
to manage these values.
165165
166+
## Dashboard Provisioning
167+
168+
HyperDX supports file-based dashboard provisioning, similar to Grafana's
169+
provisioning system. A background process watches a directory for `.json` files
170+
and upserts dashboards into MongoDB, matched by name for idempotency.
171+
172+
### Environment Variables
173+
174+
| Variable | Required | Default | Description |
175+
| ----------------------------------- | -------- | ------- | --------------------------------------------------------------- |
176+
| `DASHBOARD_PROVISIONER_DIR` | Yes | — | Directory to watch for `.json` dashboard files |
177+
| `DASHBOARD_PROVISIONER_INTERVAL` | No | `30000` | Sync interval in milliseconds (minimum 1000) |
178+
| `DASHBOARD_PROVISIONER_TEAM_ID` | No\* | — | Scope provisioning to a specific team ID |
179+
| `DASHBOARD_PROVISIONER_ALL_TEAMS` | No\* | `false` | Set to `true` to provision dashboards to all teams |
180+
181+
\*One of `DASHBOARD_PROVISIONER_TEAM_ID` or `DASHBOARD_PROVISIONER_ALL_TEAMS=true`
182+
is required when `DASHBOARD_PROVISIONER_DIR` is set.
183+
184+
### Dashboard JSON Format
185+
186+
Each `.json` file in the provisioner directory should contain a dashboard object
187+
with at minimum a `name` and `tiles` array:
188+
189+
```json
190+
{
191+
"name": "My Dashboard",
192+
"tiles": [
193+
{
194+
"id": "tile-1",
195+
"x": 0,
196+
"y": 0,
197+
"w": 6,
198+
"h": 4,
199+
"config": {
200+
"name": "Request Count",
201+
"source": "Metrics",
202+
"displayType": "line",
203+
"select": [{ "aggFn": "count" }]
204+
}
205+
}
206+
],
207+
"tags": ["provisioned"]
208+
}
209+
```
210+
211+
### Behavior
212+
213+
- Dashboards are matched by name and team for idempotency
214+
- Provisioned dashboards are flagged with `provisioned: true` so they never
215+
overwrite user-created dashboards with the same name
216+
- Removing a file from the directory does **not** delete the dashboard from
217+
MongoDB (safe by default)
218+
- Files are validated against the `DashboardWithoutIdSchema` Zod schema; invalid
219+
files are skipped with a warning
220+
221+
166222
## Note on Security
167223

168224
While this feature is convenient for development and initial setup, be careful

‎packages/api/src/__tests__/dashboardProvisioner.test.ts‎

Lines changed: 314 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,314 @@
1+
import fs from 'fs';
2+
import os from 'os';
3+
import path from 'path';
4+
5+
import { createTeam } from '@/controllers/team';
6+
import {
7+
readDashboardFiles,
8+
startDashboardProvisioner,
9+
stopDashboardProvisioner,
10+
syncDashboards,
11+
} from '@/dashboardProvisioner';
12+
import { clearDBCollections, closeDB, connectDB, makeTile } from '@/fixtures';
13+
import Dashboard from '@/models/dashboard';
14+
15+
describe('dashboardProvisioner', () => {
16+
let tmpDir: string;
17+
18+
beforeAll(async () => {
19+
await connectDB();
20+
});
21+
22+
beforeEach(() => {
23+
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'hdx-dash-test-'));
24+
});
25+
26+
afterEach(async () => {
27+
fs.rmSync(tmpDir, { recursive: true, force: true });
28+
await clearDBCollections();
29+
});
30+
31+
afterAll(async () => {
32+
await closeDB();
33+
});
34+
35+
describe('readDashboardFiles', () => {
36+
it('returns empty array for non-existent directory', () => {
37+
const result = readDashboardFiles('/non/existent/path');
38+
expect(result).toEqual([]);
39+
});
40+
41+
it('returns empty array for directory with no json files', () => {
42+
fs.writeFileSync(path.join(tmpDir, 'readme.txt'), 'not a dashboard');
43+
const result = readDashboardFiles(tmpDir);
44+
expect(result).toEqual([]);
45+
});
46+
47+
it('skips invalid JSON files', () => {
48+
fs.writeFileSync(path.join(tmpDir, 'bad.json'), '{invalid json');
49+
const result = readDashboardFiles(tmpDir);
50+
expect(result).toEqual([]);
51+
});
52+
53+
it('skips files that fail schema validation', () => {
54+
fs.writeFileSync(
55+
path.join(tmpDir, 'no-name.json'),
56+
JSON.stringify({ tiles: [] }),
57+
);
58+
const result = readDashboardFiles(tmpDir);
59+
expect(result).toEqual([]);
60+
});
61+
62+
it('parses valid dashboard files', () => {
63+
fs.writeFileSync(
64+
path.join(tmpDir, 'test.json'),
65+
JSON.stringify({ name: 'Test', tiles: [makeTile()], tags: [] }),
66+
);
67+
const result = readDashboardFiles(tmpDir);
68+
expect(result).toHaveLength(1);
69+
expect(result[0].name).toBe('Test');
70+
});
71+
});
72+
73+
describe('syncDashboards', () => {
74+
it('creates a new dashboard', async () => {
75+
const team = await createTeam({ name: 'My Team' });
76+
fs.writeFileSync(
77+
path.join(tmpDir, 'test.json'),
78+
JSON.stringify({
79+
name: 'New Dashboard',
80+
tiles: [makeTile()],
81+
tags: [],
82+
}),
83+
);
84+
85+
await syncDashboards(team._id.toString(), tmpDir);
86+
87+
const count = await Dashboard.countDocuments({ team: team._id });
88+
expect(count).toBe(1);
89+
});
90+
91+
it('updates an existing dashboard by name', async () => {
92+
const team = await createTeam({ name: 'My Team' });
93+
const tile = makeTile();
94+
await new Dashboard({
95+
name: 'Existing',
96+
tiles: [tile],
97+
tags: [],
98+
team: team._id,
99+
provisioned: true,
100+
}).save();
101+
102+
const newTile = makeTile();
103+
fs.writeFileSync(
104+
path.join(tmpDir, 'existing.json'),
105+
JSON.stringify({
106+
name: 'Existing',
107+
tiles: [newTile],
108+
tags: ['updated'],
109+
}),
110+
);
111+
112+
await syncDashboards(team._id.toString(), tmpDir);
113+
114+
const dashboard = (await Dashboard.findOne({
115+
name: 'Existing',
116+
team: team._id,
117+
})) as any;
118+
expect(dashboard.tiles[0].id).toBe(newTile.id);
119+
expect(dashboard.tags).toEqual(['updated']);
120+
});
121+
122+
it('does not create duplicates on repeated sync', async () => {
123+
const team = await createTeam({ name: 'My Team' });
124+
fs.writeFileSync(
125+
path.join(tmpDir, 'test.json'),
126+
JSON.stringify({ name: 'Dashboard', tiles: [makeTile()], tags: [] }),
127+
);
128+
129+
await syncDashboards(team._id.toString(), tmpDir);
130+
await syncDashboards(team._id.toString(), tmpDir);
131+
await syncDashboards(team._id.toString(), tmpDir);
132+
133+
const count = await Dashboard.countDocuments({ team: team._id });
134+
expect(count).toBe(1);
135+
});
136+
137+
it('provisions to multiple teams', async () => {
138+
const teamA = await createTeam({ name: 'Team A' });
139+
const teamB = await createTeam({ name: 'Team B' });
140+
fs.writeFileSync(
141+
path.join(tmpDir, 'shared.json'),
142+
JSON.stringify({
143+
name: 'Shared Dashboard',
144+
tiles: [makeTile()],
145+
tags: [],
146+
}),
147+
);
148+
149+
await syncDashboards(teamA._id.toString(), tmpDir);
150+
await syncDashboards(teamB._id.toString(), tmpDir);
151+
152+
expect(await Dashboard.countDocuments({ team: teamA._id })).toBe(1);
153+
expect(await Dashboard.countDocuments({ team: teamB._id })).toBe(1);
154+
});
155+
156+
it('does not overwrite user-created dashboards', async () => {
157+
const team = await createTeam({ name: 'My Team' });
158+
const userTile = makeTile();
159+
await new Dashboard({
160+
name: 'My Dashboard',
161+
tiles: [userTile],
162+
tags: ['user-tag'],
163+
team: team._id,
164+
// provisioned defaults to false
165+
}).save();
166+
167+
fs.writeFileSync(
168+
path.join(tmpDir, 'my-dashboard.json'),
169+
JSON.stringify({
170+
name: 'My Dashboard',
171+
tiles: [makeTile()],
172+
tags: ['provisioned-tag'],
173+
}),
174+
);
175+
176+
await syncDashboards(team._id.toString(), tmpDir);
177+
178+
// User-created dashboard is untouched
179+
const userDashboard = (await Dashboard.findOne({
180+
name: 'My Dashboard',
181+
team: team._id,
182+
provisioned: { $ne: true },
183+
})) as any;
184+
expect(userDashboard).toBeTruthy();
185+
expect(userDashboard.tiles[0].id).toBe(userTile.id);
186+
expect(userDashboard.tags).toEqual(['user-tag']);
187+
188+
// Provisioned copy was created alongside it
189+
const provisionedDashboard = await Dashboard.findOne({
190+
name: 'My Dashboard',
191+
team: team._id,
192+
provisioned: true,
193+
});
194+
expect(provisionedDashboard).toBeTruthy();
195+
});
196+
});
197+
198+
describe('startDashboardProvisioner', () => {
199+
const originalEnv = process.env;
200+
201+
beforeEach(() => {
202+
process.env = { ...originalEnv };
203+
stopDashboardProvisioner();
204+
});
205+
206+
afterEach(() => {
207+
stopDashboardProvisioner();
208+
process.env = originalEnv;
209+
});
210+
211+
it('is a no-op when DASHBOARD_PROVISIONER_DIR is not set', async () => {
212+
const team = await createTeam({ name: 'My Team' });
213+
fs.writeFileSync(
214+
path.join(tmpDir, 'test.json'),
215+
JSON.stringify({ name: 'Noop', tiles: [makeTile()], tags: [] }),
216+
);
217+
delete process.env.DASHBOARD_PROVISIONER_DIR;
218+
process.env.DASHBOARD_PROVISIONER_ALL_TEAMS = 'true';
219+
await startDashboardProvisioner();
220+
expect(await Dashboard.countDocuments({ team: team._id })).toBe(0);
221+
});
222+
223+
it('rejects invalid interval', async () => {
224+
const team = await createTeam({ name: 'My Team' });
225+
fs.writeFileSync(
226+
path.join(tmpDir, 'test.json'),
227+
JSON.stringify({ name: 'Noop', tiles: [makeTile()], tags: [] }),
228+
);
229+
process.env.DASHBOARD_PROVISIONER_DIR = tmpDir;
230+
process.env.DASHBOARD_PROVISIONER_ALL_TEAMS = 'true';
231+
process.env.DASHBOARD_PROVISIONER_INTERVAL = 'abc';
232+
await startDashboardProvisioner();
233+
expect(await Dashboard.countDocuments({ team: team._id })).toBe(0);
234+
});
235+
236+
it('rejects interval below 1000ms', async () => {
237+
const team = await createTeam({ name: 'My Team' });
238+
fs.writeFileSync(
239+
path.join(tmpDir, 'test.json'),
240+
JSON.stringify({ name: 'Noop', tiles: [makeTile()], tags: [] }),
241+
);
242+
process.env.DASHBOARD_PROVISIONER_DIR = tmpDir;
243+
process.env.DASHBOARD_PROVISIONER_ALL_TEAMS = 'true';
244+
process.env.DASHBOARD_PROVISIONER_INTERVAL = '500';
245+
await startDashboardProvisioner();
246+
expect(await Dashboard.countDocuments({ team: team._id })).toBe(0);
247+
});
248+
249+
it('requires team ID or all-teams flag', async () => {
250+
const team = await createTeam({ name: 'My Team' });
251+
fs.writeFileSync(
252+
path.join(tmpDir, 'test.json'),
253+
JSON.stringify({ name: 'Noop', tiles: [makeTile()], tags: [] }),
254+
);
255+
process.env.DASHBOARD_PROVISIONER_DIR = tmpDir;
256+
delete process.env.DASHBOARD_PROVISIONER_TEAM_ID;
257+
delete process.env.DASHBOARD_PROVISIONER_ALL_TEAMS;
258+
await startDashboardProvisioner();
259+
expect(await Dashboard.countDocuments({ team: team._id })).toBe(0);
260+
});
261+
262+
it('rejects invalid team ID format', async () => {
263+
const team = await createTeam({ name: 'My Team' });
264+
fs.writeFileSync(
265+
path.join(tmpDir, 'test.json'),
266+
JSON.stringify({ name: 'Noop', tiles: [makeTile()], tags: [] }),
267+
);
268+
process.env.DASHBOARD_PROVISIONER_DIR = tmpDir;
269+
process.env.DASHBOARD_PROVISIONER_TEAM_ID = 'not-an-objectid';
270+
await startDashboardProvisioner();
271+
expect(await Dashboard.countDocuments({ team: team._id })).toBe(0);
272+
});
273+
274+
it('provisions dashboards for all teams', async () => {
275+
const team = await createTeam({ name: 'My Team' });
276+
fs.writeFileSync(
277+
path.join(tmpDir, 'test.json'),
278+
JSON.stringify({
279+
name: 'All Teams Dash',
280+
tiles: [makeTile()],
281+
tags: [],
282+
}),
283+
);
284+
process.env.DASHBOARD_PROVISIONER_DIR = tmpDir;
285+
process.env.DASHBOARD_PROVISIONER_ALL_TEAMS = 'true';
286+
process.env.DASHBOARD_PROVISIONER_INTERVAL = '60000';
287+
288+
await startDashboardProvisioner();
289+
290+
const count = await Dashboard.countDocuments({ team: team._id });
291+
expect(count).toBe(1);
292+
});
293+
294+
it('provisions dashboards for a specific team', async () => {
295+
const team = await createTeam({ name: 'My Team' });
296+
fs.writeFileSync(
297+
path.join(tmpDir, 'test.json'),
298+
JSON.stringify({
299+
name: 'Team Specific Dash',
300+
tiles: [makeTile()],
301+
tags: [],
302+
}),
303+
);
304+
process.env.DASHBOARD_PROVISIONER_DIR = tmpDir;
305+
process.env.DASHBOARD_PROVISIONER_TEAM_ID = team._id.toString();
306+
process.env.DASHBOARD_PROVISIONER_INTERVAL = '60000';
307+
308+
await startDashboardProvisioner();
309+
310+
const count = await Dashboard.countDocuments({ team: team._id });
311+
expect(count).toBe(1);
312+
});
313+
});
314+
});

0 commit comments

Comments
 (0)