Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 9 additions & 41 deletions kiloclaw/controller/src/routes/health.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,15 @@ import { Hono } from 'hono';
import { registerHealthRoute } from './health';
import type { Supervisor } from '../supervisor';

function createMockSupervisor(state: string): Supervisor {
function createMockSupervisor(): Supervisor {
return {
start: async () => true,
stop: async () => true,
restart: async () => true,
shutdown: async () => undefined,
getState: () => state as ReturnType<Supervisor['getState']>,
getState: () => 'running',
getStats: () => ({
state: state as ReturnType<Supervisor['getState']>,
state: 'running',
pid: 42,
uptime: 123,
restarts: 2,
Expand All @@ -21,55 +21,23 @@ function createMockSupervisor(state: string): Supervisor {
}

describe('GET /_kilo/health', () => {
it('returns 200 with status fields when gateway is running', async () => {
it('returns 200 with minimal payload', async () => {
const app = new Hono();
registerHealthRoute(app, createMockSupervisor('running'));
registerHealthRoute(app, createMockSupervisor());

const resp = await app.request('/_kilo/health');
expect(resp.status).toBe(200);
expect(await resp.json()).toEqual({
status: 'ok',
gateway: 'running',
uptime: 123,
restarts: 2,
});
});

it('returns 503 when gateway is not running', async () => {
const app = new Hono();
registerHealthRoute(app, createMockSupervisor('crashed'));

const resp = await app.request('/_kilo/health');
expect(resp.status).toBe(503);
const body = (await resp.json()) as { status: string; gateway: string };
expect(body.status).toBe('starting');
expect(body.gateway).toBe('crashed');
});

it('returns 503 when gateway is starting', async () => {
const app = new Hono();
registerHealthRoute(app, createMockSupervisor('starting'));

const resp = await app.request('/_kilo/health');
expect(resp.status).toBe(503);
const body = (await resp.json()) as { status: string; gateway: string };
expect(body.status).toBe('starting');
expect(body.gateway).toBe('starting');
expect(await resp.json()).toEqual({ status: 'ok' });
});
});

describe('GET /health (compatibility alias)', () => {
it('returns 200 with status fields', async () => {
it('returns 200 with minimal payload', async () => {
const app = new Hono();
registerHealthRoute(app, createMockSupervisor('running'));
registerHealthRoute(app, createMockSupervisor());

const resp = await app.request('/health');
expect(resp.status).toBe(200);
expect(await resp.json()).toEqual({
status: 'ok',
gateway: 'running',
uptime: 123,
restarts: 2,
});
expect(await resp.json()).toEqual({ status: 'ok' });
});
});
19 changes: 4 additions & 15 deletions kiloclaw/controller/src/routes/health.ts
Original file line number Diff line number Diff line change
@@ -1,22 +1,11 @@
import type { Context, Hono } from 'hono';
import type { Supervisor } from '../supervisor';

export function registerHealthRoute(app: Hono, supervisor: Supervisor): void {
const handler = (c: Context) => {
const stats = supervisor.getStats();
const ready = stats.state === 'running';
return c.json(
{
status: ready ? 'ok' : 'starting',
gateway: stats.state,
uptime: stats.uptime,
restarts: stats.restarts,
},
ready ? 200 : 503
);
};
export function registerHealthRoute(app: Hono, _supervisor: Supervisor): void {
const handler = (c: Context) => c.json({ status: 'ok' });
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

WARNING: Health endpoint now always reports healthy, masking startup/crash states

This route used to return 503 when the gateway was not running. Returning 200/{ status: 'ok' } unconditionally can make readiness checks and external monitors treat unhealthy instances as healthy, which risks routing traffic to instances that are still starting or have crashed.


// Public Fly health probe endpoint. Keep response intentionally minimal.
app.get('/_kilo/health', handler);
// Compatibility alias for machines still configured with legacy health path.
// Compatibility alias to match the same minimal, public health response.
app.get('/health', handler);
}
17 changes: 12 additions & 5 deletions kiloclaw/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import { authMiddleware, internalApiMiddleware } from './auth';
import { sandboxIdFromUserId } from './auth/sandbox-id';
import { registerVersionIfNeeded } from './lib/image-version';
import { startingUpPage } from './pages/starting-up';
import { buildForwardHeaders } from './utils/proxy-headers';

// Export DOs (match wrangler.jsonc class_name bindings)
export { KiloClawInstance } from './durable-objects/kiloclaw-instance';
Expand Down Expand Up @@ -249,11 +250,17 @@ app.all('*', async c => {

const isWebSocketRequest = request.headers.get('Upgrade')?.toLowerCase() === 'websocket';

// Build headers to forward, adding the fly-force-instance-id header
const forwardHeaders = new Headers(request.headers);
forwardHeaders.set('fly-force-instance-id', machineId);
// Remove hop-by-hop headers that shouldn't be forwarded
forwardHeaders.delete('host');
if (!c.env.GATEWAY_TOKEN_SECRET) {
console.error('[CONFIG] Missing required environment variables: GATEWAY_TOKEN_SECRET');
return c.json({ error: 'Configuration error' }, 503);
}

const forwardHeaders = await buildForwardHeaders({
requestHeaders: request.headers,
machineId,
sandboxId,
gatewayTokenSecret: c.env.GATEWAY_TOKEN_SECRET,
});

// WebSocket proxy
if (isWebSocketRequest) {
Expand Down
42 changes: 42 additions & 0 deletions kiloclaw/src/utils/proxy-headers.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
import { describe, expect, it } from 'vitest';
import { deriveGatewayToken } from '../auth/gateway-token';
import { buildForwardHeaders } from './proxy-headers';

describe('buildForwardHeaders', () => {
it('adds routing and proxy token headers', async () => {
const headers = new Headers({
host: 'example.com',
'x-request-id': 'req-123',
});

const result = await buildForwardHeaders({
requestHeaders: headers,
machineId: 'machine-123',
sandboxId: 'sandbox-abc',
gatewayTokenSecret: 'secret-123',
});

const expectedToken = await deriveGatewayToken('sandbox-abc', 'secret-123');

expect(result.get('fly-force-instance-id')).toBe('machine-123');
expect(result.get('x-kiloclaw-proxy-token')).toBe(expectedToken);
expect(result.get('host')).toBeNull();
expect(result.get('x-request-id')).toBe('req-123');
});

it('overwrites inbound proxy token header', async () => {
const headers = new Headers({
'x-kiloclaw-proxy-token': 'old-token',
});

const result = await buildForwardHeaders({
requestHeaders: headers,
machineId: 'machine-123',
sandboxId: 'sandbox-abc',
gatewayTokenSecret: 'secret-123',
});

const expectedToken = await deriveGatewayToken('sandbox-abc', 'secret-123');
expect(result.get('x-kiloclaw-proxy-token')).toBe(expectedToken);
});
});
18 changes: 18 additions & 0 deletions kiloclaw/src/utils/proxy-headers.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
import { deriveGatewayToken } from '../auth/gateway-token';

export async function buildForwardHeaders(params: {
requestHeaders: Headers;
machineId: string;
sandboxId: string;
gatewayTokenSecret: string;
}): Promise<Headers> {
const { requestHeaders, machineId, sandboxId, gatewayTokenSecret } = params;
const forwardHeaders = new Headers(requestHeaders);

const gatewayToken = await deriveGatewayToken(sandboxId, gatewayTokenSecret);
forwardHeaders.set('x-kiloclaw-proxy-token', gatewayToken);
forwardHeaders.set('fly-force-instance-id', machineId);
forwardHeaders.delete('host');
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

WARNING: Incomplete hop-by-hop header stripping when forwarding request headers

Only host is removed, but connection and headers listed by connection (for example upgrade, te, proxy-connection, keep-alive, transfer-encoding) should also be stripped before proxying. Forwarding hop-by-hop headers can cause protocol inconsistencies and request smuggling-style edge cases across intermediaries.

Copy link
Contributor Author

@pandemicsyn pandemicsyn Feb 26, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is kinda true kinda not, for WS Connection/Upgrade are part of the handshake which is why haven't done strict hop-by-hop stripping.


return forwardHeaders;
}
1 change: 1 addition & 0 deletions kiloclaw/wrangler.jsonc
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@
"FLY_IMAGE_TAG": "latest",
"FLY_REGION": "us,eu",
"OPENCLAW_ALLOWED_ORIGINS": "https://claw.kilosessions.ai,https://kilo.ai,https://www.kilo.ai",
"REQUIRE_PROXY_TOKEN": "true",
// Defaults to "production". For local dev, override to "development" in .dev.vars.
"WORKER_ENV": "production",
},
Expand Down