hearthforge

hearthforge/tests/e2e.ci.test.ts
e2e.ci.test.ts
⎇
Raw
1/**
2 * CI pipeline E2E tests.
3 *
4 * Uses a mock Docker API server (Bun.serve over a Unix socket) so no real
5 * Docker/Podman installation is required. The mock handles every endpoint
6 * the CI service calls and lets individual tests queue custom exec responses
7 * (output + exit code) to simulate success, failure, and specific log output.
8 */
9import { describe, test, expect, beforeAll, afterAll, beforeEach } from "bun:test";
10import { chromium, type Browser, type BrowserContext } from "playwright";
11import { existsSync, rmSync, writeFileSync } from "node:fs";
12import { spawnSync } from "node:child_process";
13import path from "node:path";
14import {
15 BASE,
16 ADMIN_PASS,
17 DATA_DIR,
18 setupTestEnv,
19 spawnServer,
20 killServer,
21 seedRepo,
22 login,
23} from "./helpers.ts";
24import { db } from "../src/db/index.ts";
25import config from "../src/config.ts";
26import {
27 resetDockerSocket,
28 triggerRun,
29} from "../src/services/ci.ts";
30import { paths } from "../src/constants.ts";
31
32// ── Mock Docker server ────────────────────────────────────────────────────────
33
34const SOCKET_PATH = `/tmp/test-docker-ci-${process.pid}.sock`;
35
36interface ExecResp {
37 output: string;
38 exitCode: number;
39}
40
41// Per-exec-ID response map, populated when exec is created
42const execMap = new Map<string, ExecResp>();
43// Queue consumed in order when execs are created — allows tests to pre-program
44// specific step responses
45const execQueue: ExecResp[] = [];
46let execCounter = 0;
47
48function queueExec(resp: ExecResp) {
49 execQueue.push(resp);
50}
51
52function resetMock() {
53 execMap.clear();
54 execQueue.length = 0;
55 execCounter = 0;
56}
57
58/** Build a Docker multiplexed stream frame from a string. */
59function muxFrame(text: string, stream = 1): Uint8Array {
60 const payload = Buffer.from(text, "utf-8");
61 const hdr = Buffer.alloc(8);
62 hdr[0] = stream;
63 hdr.writeUInt32BE(payload.length, 4);
64 return Buffer.concat([hdr, payload]);
65}
66
67/** Build a minimal tar archive containing one file. */
68function makeTar(filename: string, content: string): Uint8Array {
69 const data = Buffer.from(content, "utf-8");
70 const hdr = Buffer.alloc(512);
71 hdr.write(path.basename(filename).slice(0, 100), 0, "ascii");
72 hdr.write("0000644\0", 100, "ascii"); // mode
73 hdr.write("0000000\0", 108, "ascii"); // uid
74 hdr.write("0000000\0", 116, "ascii"); // gid
75 hdr.write(data.length.toString(8).padStart(11, "0") + "\0", 124, "ascii");
76 hdr.write("00000000000\0", 136, "ascii"); // mtime
77 hdr[156] = 0x30; // type flag: regular file
78 // Checksum: fill with spaces, compute, write back
79 hdr.fill(0x20, 148, 156);
80 let sum = 0;
81 for (let i = 0; i < 512; i++) sum += hdr[i]!;
82 hdr.write(sum.toString(8).padStart(6, "0") + "\0 ", 148, "ascii");
83 // Pad file content to 512-byte block
84 const paddedLen = Math.ceil(Math.max(data.length, 1) / 512) * 512;
85 const padded = Buffer.alloc(paddedLen);
86 data.copy(padded);
87 return Buffer.concat([hdr, padded]);
88}
89
90let mockServer: ReturnType<typeof Bun.serve>;
91
92function startMockDocker() {
93 rmSync(SOCKET_PATH, { force: true });
94 mockServer = Bun.serve({
95 unix: SOCKET_PATH,
96 fetch(req: Request): Response {
97 const p = new URL(req.url).pathname;
98 const qs = new URL(req.url).searchParams;
99
100 // Health check
101 if (req.method === "GET" && p === "/v1.47/info") {
102 return Response.json({ ServerVersion: "mock" });
103 }
104 // Pull image (streaming, just needs to resolve)
105 if (req.method === "POST" && p.startsWith("/v1.47/images/create")) {
106 return new Response('{"status":"Pull complete"}\n');
107 }
108 // Create container
109 if (req.method === "POST" && /\/containers\/create/.test(p)) {
110 return Response.json({ Id: "mock-ctr-001" });
111 }
112 // Start container
113 if (
114 req.method === "POST" &&
115 /\/containers\/[^/]+\/start$/.test(p)
116 ) {
117 return new Response(null, { status: 204 });
118 }
119 // Create exec — pop next queued response and assign to this exec ID
120 if (
121 req.method === "POST" &&
122 /\/containers\/[^/]+\/exec$/.test(p)
123 ) {
124 execCounter++;
125 const execId = `mock-exec-${execCounter}`;
126 execMap.set(
127 execId,
128 execQueue.shift() ?? { output: "", exitCode: 0 },
129 );
130 return Response.json({ Id: execId });
131 }
132 // Start exec — return queued output as mux stream
133 if (req.method === "POST" && /\/exec\/[^/]+\/start$/.test(p)) {
134 const id = p.match(/\/exec\/([^/]+)\/start/)![1]!;
135 const resp = execMap.get(id) ?? { output: "", exitCode: 0 };
136 return new Response(
137 resp.output ? muxFrame(resp.output) : new Uint8Array(0),
138 );
139 }
140 // Inspect exec — return exit code
141 if (req.method === "GET" && /\/exec\/[^/]+\/json$/.test(p)) {
142 const id = p.match(/\/exec\/([^/]+)\/json/)![1]!;
143 const resp = execMap.get(id) ?? { output: "", exitCode: 0 };
144 return Response.json({ ExitCode: resp.exitCode });
145 }
146 // Archive (used by publish_file artifact collection)
147 if (
148 req.method === "GET" &&
149 /\/containers\/[^/]+\/archive/.test(p)
150 ) {
151 const filePath = qs.get("path") ?? "file.txt";
152 return new Response(
153 makeTar(path.basename(filePath), "artifact-content-123"),
154 { headers: { "Content-Type": "application/x-tar" } },
155 );
156 }
157 // Delete container
158 if (req.method === "DELETE" && /\/containers\//.test(p)) {
159 return new Response(null, { status: 204 });
160 }
161 return new Response("Not found", { status: 404 });
162 },
163 });
164}
165
166// ── Helpers ───────────────────────────────────────────────────────────────────
167
168/** Push a .hearthforge-ci.toml into an existing repo; return the commit SHA. */
169function seedCiToml(repoName: string, toml: string): string {
170 const repoDir = path.join(process.cwd(), DATA_DIR, "repos", `${repoName}.git`);
171 const tmp = `/tmp/hf-ci-seed-${Date.now()}`;
172 try {
173 spawnSync("git", ["clone", repoDir, tmp], { stdio: "ignore" });
174 spawnSync("git", ["-C", tmp, "config", "user.email", "ci@test.com"], {
175 stdio: "ignore",
176 });
177 spawnSync("git", ["-C", tmp, "config", "user.name", "CI Test"], {
178 stdio: "ignore",
179 });
180 writeFileSync(path.join(tmp, ".hearthforge-ci.toml"), toml);
181 spawnSync("git", ["-C", tmp, "add", ".hearthforge-ci.toml"], {
182 stdio: "ignore",
183 });
184 spawnSync("git", ["-C", tmp, "commit", "-m", "Add CI config"], {
185 stdio: "ignore",
186 });
187 spawnSync("git", ["-C", tmp, "push", "origin", "HEAD:main"], {
188 stdio: "ignore",
189 });
190 const r = spawnSync(
191 "git",
192 ["-C", tmp, "rev-parse", "HEAD"],
193 { stdio: ["ignore", "pipe", "ignore"] },
194 );
195 return r.stdout.toString().trim();
196 } finally {
197 rmSync(tmp, { recursive: true, force: true });
198 }
199}
200
201/** Poll until a CI run leaves pending/running state, then return its status. */
202async function waitForRun(runId: number, timeoutMs = 10_000): Promise<string> {
203 const deadline = Date.now() + timeoutMs;
204 while (Date.now() < deadline) {
205 const row = await db
206 .selectFrom("ci_runs")
207 .select("status")
208 .where("id", "=", runId)
209 .executeTakeFirst();
210 if (row && row.status !== "pending" && row.status !== "running") {
211 return row.status;
212 }
213 await Bun.sleep(100);
214 }
215 throw new Error(`Run ${runId} did not complete within ${timeoutMs}ms`);
216}
217
218async function loggedInContext(
219 browser: Browser,
220 username = "admin",
221 password = ADMIN_PASS,
222): Promise<BrowserContext> {
223 const ctx = await browser.newContext();
224 const page = await ctx.newPage();
225 await login(page, username, password);
226 await page.close();
227 return ctx;
228}
229
230// ── Test setup ────────────────────────────────────────────────────────────────
231
232let browser: Browser;
233let server: Awaited<ReturnType<typeof spawnServer>>;
234let adminCtx: BrowserContext;
235let adminUserId: number;
236let ciRepoSha: string; // SHA of commit with .hearthforge-ci.toml
237
238const SIMPLE_TOML = `
239image = "debian:latest"
240
241[on]
242manual = true
243push = ["main"]
244
245[hello]
246run_sh = "echo hello"
247`;
248
249const ARTIFACT_TOML = `
250image = "debian:latest"
251work_dir = "/ci"
252
253[on]
254manual = true
255
256[build]
257run_sh = "echo building"
258publish_file = ["/ci/output.txt"]
259`;
260
261beforeAll(async () => {
262 await setupTestEnv();
263
264 // Point CI service at mock socket BEFORE starting any runs
265 config.CI_DOCKER_SOCKET = SOCKET_PATH;
266 resetDockerSocket();
267 startMockDocker();
268
269 server = await spawnServer();
270 browser = await chromium.launch();
271 adminCtx = await loggedInContext(browser);
272
273 // Get admin user ID
274 const row = await db
275 .selectFrom("users")
276 .select("id")
277 .where("username", "=", "admin")
278 .executeTakeFirst();
279 adminUserId = row!.id;
280
281 // Create ci-repo via UI and seed it
282 const page = await adminCtx.newPage();
283 try {
284 await page.goto(`${BASE}/new`);
285 await page.fill("[name=name]", "ci-repo");
286 await page.click('form[action="/new"] button[type=submit]');
287 await page.waitForURL(`${BASE}/ci-repo`);
288 } finally {
289 await page.close();
290 }
291 seedRepo("ci-repo");
292 ciRepoSha = seedCiToml("ci-repo", SIMPLE_TOML);
293});
294
295afterAll(async () => {
296 await adminCtx.close();
297 await browser.close();
298 await killServer(server);
299 mockServer.stop(true);
300 rmSync(SOCKET_PATH, { force: true });
301});
302
303beforeEach(() => {
304 resetMock();
305});
306
307// ── Tests ─────────────────────────────────────────────────────────────────────
308
309describe("pipelines tab", () => {
310 test("tab is visible in repo nav", async () => {
311 const page = await adminCtx.newPage();
312 try {
313 await page.goto(`${BASE}/ci-repo`);
314 const tab = page.locator('.repo-tab', { hasText: 'Pipelines' });
315 expect(await tab.isVisible()).toBe(true);
316 } finally {
317 await page.close();
318 }
319 });
320
321 test("history page shows empty state when no runs", async () => {
322 // Use a separate repo that has never had a run
323 const page = await adminCtx.newPage();
324 try {
325 await page.goto(`${BASE}/ci-repo/ci`);
326 expect(await page.locator(".empty-state").isVisible()).toBe(true);
327 expect(await page.locator(".empty-state").textContent()).toContain(
328 "No pipeline runs yet",
329 );
330 } finally {
331 await page.close();
332 }
333 });
334
335 test("help section is collapsible and contains template download", async () => {
336 const page = await adminCtx.newPage();
337 try {
338 await page.goto(`${BASE}/ci-repo/ci`);
339 const help = page.locator("details.ci-help");
340 expect(await help.isVisible()).toBe(true);
341 await help.locator("summary").click();
342 const dlLink = page.locator('a[download=".hearthforge-ci.toml"]');
343 expect(await dlLink.isVisible()).toBe(true);
344 } finally {
345 await page.close();
346 }
347 });
348});
349
350describe("successful run", () => {
351 let runId: number;
352
353 beforeAll(async () => {
354 queueExec({ output: "hello from mock CI\n", exitCode: 0 });
355 runId = await triggerRun("ci-repo", {
356 triggerSource: "manual",
357 commitSha: ciRepoSha,
358 commitBranch: "main",
359 triggeredBy: adminUserId,
360 });
361 await waitForRun(runId);
362 });
363
364 test("run status is success", async () => {
365 const run = await db
366 .selectFrom("ci_runs")
367 .select("status")
368 .where("id", "=", runId)
369 .executeTakeFirst();
370 expect(run?.status).toBe("success");
371 });
372
373 test("step status is success and log is captured", async () => {
374 const step = await db
375 .selectFrom("ci_steps")
376 .select(["status", "log"])
377 .where("run_id", "=", runId)
378 .where("name", "=", "hello")
379 .executeTakeFirst();
380 expect(step?.status).toBe("success");
381 expect(step?.log).toContain("hello from mock CI");
382 });
383
384 test("history page shows the completed run", async () => {
385 const page = await adminCtx.newPage();
386 try {
387 await page.goto(`${BASE}/ci-repo/ci`);
388 expect(
389 await page.locator(".ci-status-pill.ci-status-success").count(),
390 ).toBeGreaterThan(0);
391 } finally {
392 await page.close();
393 }
394 });
395
396 test("run detail page shows step and log", async () => {
397 const page = await adminCtx.newPage();
398 try {
399 await page.goto(`${BASE}/ci-repo/ci/${runId}`);
400 expect(
401 await page.locator(".ci-step").first().textContent(),
402 ).toContain("hello");
403 // Open step details to see log
404 await page.locator(".ci-step").first().click();
405 expect(await page.locator(".ci-step-log").textContent()).toContain(
406 "hello from mock CI",
407 );
408 } finally {
409 await page.close();
410 }
411 });
412
413 test("retry creates a new run", async () => {
414 const page = await adminCtx.newPage();
415 try {
416 await page.goto(`${BASE}/ci-repo/ci/${runId}`);
417 await page.click('button:text("Retry")');
418 // Should redirect to the new run
419 await page.waitForURL(/\/ci-repo\/ci\/\d+/);
420 const newRunId = Number(
421 page.url().split("/ci/")[1],
422 );
423 expect(newRunId).toBeGreaterThan(runId);
424 // Wait for new run to complete (uses default exit 0)
425 const status = await waitForRun(newRunId);
426 expect(status).toBe("success");
427 } finally {
428 await page.close();
429 }
430 });
431});
432
433describe("failing run", () => {
434 let runId: number;
435
436 beforeAll(async () => {
437 // Step exec: non-zero exit code
438 queueExec({ output: "build error: file not found\n", exitCode: 1 });
439 runId = await triggerRun("ci-repo", {
440 triggerSource: "manual",
441 commitSha: ciRepoSha,
442 commitBranch: "main",
443 triggeredBy: adminUserId,
444 });
445 await waitForRun(runId);
446 });
447
448 test("run status is failure", async () => {
449 const run = await db
450 .selectFrom("ci_runs")
451 .select("status")
452 .where("id", "=", runId)
453 .executeTakeFirst();
454 expect(run?.status).toBe("failure");
455 });
456
457 test("step status is failure and error log captured", async () => {
458 const step = await db
459 .selectFrom("ci_steps")
460 .select(["status", "log"])
461 .where("run_id", "=", runId)
462 .where("name", "=", "hello")
463 .executeTakeFirst();
464 expect(step?.status).toBe("failure");
465 expect(step?.log).toContain("build error");
466 });
467
468 test("run detail page shows failure status", async () => {
469 const page = await adminCtx.newPage();
470 try {
471 await page.goto(`${BASE}/ci-repo/ci/${runId}`);
472 expect(
473 await page.locator(".ci-status-pill.ci-status-failure").count(),
474 ).toBeGreaterThan(0);
475 } finally {
476 await page.close();
477 }
478 });
479});
480
481describe("cancel", () => {
482 test("cancelling a pending run marks it cancelled", async () => {
483 // Trigger without queuing — run will start and eventually succeed,
484 // but we cancel immediately before it gets far
485 const runId = await triggerRun("ci-repo", {
486 triggerSource: "manual",
487 commitSha: ciRepoSha,
488 commitBranch: "main",
489 triggeredBy: adminUserId,
490 });
491 // Cancel via API before it completes
492 const resp = await fetch(`${BASE}/ci-repo/ci/${runId}/cancel`, {
493 method: "POST",
494 redirect: "manual",
495 });
496 expect(resp.status).toBe(302);
497
498 // Wait and check final status
499 const status = await waitForRun(runId);
500 expect(["cancelled", "success", "failure"]).toContain(status);
501
502 // If we got there first, it's cancelled
503 if (status === "cancelled") {
504 const run = await db
505 .selectFrom("ci_runs")
506 .select("status")
507 .where("id", "=", runId)
508 .executeTakeFirst();
509 expect(run?.status).toBe("cancelled");
510 }
511 });
512});
513
514describe("artifacts", () => {
515 let runId: number;
516 let artifactId: number;
517
518 beforeAll(async () => {
519 // Seed repo with artifact TOML
520 const sha = seedCiToml("ci-repo", ARTIFACT_TOML);
521 // work_dir causes 1 mkdir exec before the step
522 // defaults: {output:'', exitCode:0} for both
523 runId = await triggerRun("ci-repo", {
524 triggerSource: "manual",
525 commitSha: sha,
526 commitBranch: "main",
527 triggeredBy: adminUserId,
528 });
529 await waitForRun(runId);
530
531 const artifact = await db
532 .selectFrom("ci_artifacts")
533 .select("id")
534 .where("run_id", "=", runId)
535 .executeTakeFirst();
536 artifactId = artifact?.id ?? 0;
537 });
538
539 test("artifact row created in DB", async () => {
540 const artifacts = await db
541 .selectFrom("ci_artifacts")
542 .selectAll()
543 .where("run_id", "=", runId)
544 .execute();
545 expect(artifacts.length).toBe(1);
546 expect(artifacts[0]!.filename).toBe("output.txt");
547 });
548
549 test("artifact is downloadable via HTTP", async () => {
550 expect(artifactId).toBeGreaterThan(0);
551 const resp = await fetch(
552 `${BASE}/ci-repo/ci/${runId}/artifacts/${artifactId}`,
553 );
554 expect(resp.status).toBe(200);
555 const body = await resp.text();
556 expect(body).toBe("artifact-content-123");
557 });
558
559 test("run detail page shows artifact list", async () => {
560 const page = await adminCtx.newPage();
561 try {
562 await page.goto(`${BASE}/ci-repo/ci/${runId}`);
563 expect(
564 await page.locator(".ci-artifact-item").count(),
565 ).toBeGreaterThan(0);
566 expect(
567 await page.locator(".ci-artifact-name").textContent(),
568 ).toContain("output.txt");
569 } finally {
570 await page.close();
571 }
572 });
573});
574
575describe("badge", () => {
576 test("badge SVG returns success status after successful run", async () => {
577 const resp = await fetch(`${BASE}/ci-repo/ci/badge.svg`);
578 expect(resp.status).toBe(200);
579 expect(resp.headers.get("Content-Type")).toContain("image/svg+xml");
580 const body = await resp.text();
581 expect(body).toContain("<svg");
582 expect(body).toContain("success");
583 });
584
585 test("badge returns 404 for private repo when not logged in", async () => {
586 // Create a private repo
587 const page = await adminCtx.newPage();
588 try {
589 await page.goto(`${BASE}/new`);
590 await page.fill("[name=name]", "private-ci-repo");
591 await page.check("[name=is_private]");
592 await page.click('form[action="/new"] button[type=submit]');
593 await page.waitForURL(`${BASE}/private-ci-repo`);
594 } finally {
595 await page.close();
596 }
597 const resp = await fetch(`${BASE}/private-ci-repo/ci/badge.svg`);
598 expect(resp.status).toBe(404);
599 });
600});
601
602describe("secrets", () => {
603 test("can add, list, and delete a secret via settings", async () => {
604 const page = await adminCtx.newPage();
605 try {
606 await page.goto(`${BASE}/ci-repo/settings`);
607 // Add secret — scope to the CI secrets form
608 const secretsForm = page.locator('form[action$="/settings/ci-secrets"]');
609 await secretsForm.locator('[name=name]').fill("MY_SECRET");
610 await secretsForm.locator('[name=value]').fill("super-secret-value");
611 await secretsForm.locator('[name=description]').fill("A test secret");
612 await secretsForm.locator('button[type=submit]').click();
613 await page.waitForURL(/settings/);
614 // Secret name is shown, value masked
615 expect(await page.locator('code:text("MY_SECRET")').count()).toBe(1);
616 expect(await page.getByText("●●●●●●").count()).toBeGreaterThan(0);
617
618 // Delete it
619 const deleteBtn = page
620 .locator(".label-settings-item")
621 .filter({ hasText: "MY_SECRET" })
622 .locator('button:text("Delete")');
623 await deleteBtn.click();
624 await page.waitForURL(/settings/);
625 expect(await page.locator('code:text("MY_SECRET")').count()).toBe(0);
626 } finally {
627 await page.close();
628 }
629 });
630
631 test("secret value is masked in step logs", async () => {
632 // Add secret
633 await db
634 .insertInto("ci_secrets")
635 .values({
636 repo_id: (await db
637 .selectFrom("repositories")
638 .select("id")
639 .where("name", "=", "ci-repo")
640 .executeTakeFirstOrThrow()).id,
641 name: "MASK_ME",
642 value: "s3cr3t-p4ssw0rd",
643 })
644 .execute();
645
646 // Step echoes the secret value; mock returns it as output
647 queueExec({ output: "s3cr3t-p4ssw0rd is the value\n", exitCode: 0 });
648 const runId = await triggerRun("ci-repo", {
649 triggerSource: "manual",
650 commitSha: ciRepoSha,
651 commitBranch: "main",
652 triggeredBy: adminUserId,
653 });
654 await waitForRun(runId);
655
656 const step = await db
657 .selectFrom("ci_steps")
658 .select("log")
659 .where("run_id", "=", runId)
660 .where("name", "=", "hello")
661 .executeTakeFirst();
662
663 expect(step?.log).not.toContain("s3cr3t-p4ssw0rd");
664 expect(step?.log).toContain("[MASKED]");
665
666 // Cleanup
667 await db
668 .deleteFrom("ci_secrets")
669 .where("name", "=", "MASK_ME")
670 .execute();
671 });
672});
673