Skip to content

kiloclaw: remove IPv4 anycast address provisioning for Fly apps #482

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
kiloconnect wants to merge 2 commits into
base: main
Choose a base branch
from
Open

kiloclaw: remove IPv4 anycast address provisioning for Fly apps #482

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2b55c93
kiloclaw: remove IPv4 anycast address provisioning for Fly apps
kiloconnect[bot] Feb 23, 2026
9e1696f
release ipv4 scripts
pandemicsyn Feb 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion kiloclaw/AGENTS.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ src/
│ ├── gateway-token.ts # HMAC-SHA256 derivation for per-sandbox tokens
│ └── sandbox-id.ts # userId <-> sandboxId (base64url, reversible)
├── durable-objects/
│ ├── kiloclaw-app.ts # DO: per-user Fly App lifecycle (create app, allocate IPs, env key)
│ ├── kiloclaw-app.ts # DO: per-user Fly App lifecycle (create app, allocate IPv6, env key)
│ └── kiloclaw-instance.ts # DO: lifecycle state machine, reconciliation, two-phase destroy
├── fly/
│ ├── apps.ts # Fly Apps + IP allocation REST API (per-user apps)
Expand Down
100 changes: 100 additions & 0 deletions kiloclaw/scripts/release-ipv4.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
#!/usr/bin/env bash
# Release IPv4 addresses from all kiloclaw per-user Fly apps.
#
# Iterates over apps matching the acct-* (production) and dev-* (development)
# naming conventions, finds any v4/shared_v4 IP assignments, and releases them.
#
# Usage:
# ./scripts/release-ipv4.sh # dry-run (default)
# ./scripts/release-ipv4.sh --apply # actually release IPs
# ./scripts/release-ipv4.sh --org kilo-679 # override org slug
#
# Prerequisites:
# - flyctl authenticated (`fly auth login`)
# - jq installed

set -euo pipefail

ORG="kilo-679"
DRY_RUN=true

while [[ $# -gt 0 ]]; do
case "$1" in
--apply) DRY_RUN=false; shift ;;
--org) ORG="$2"; shift 2 ;;
-h|--help)
echo "Usage: $0 [--apply] [--org <slug>]"
echo " --apply Actually release IPs (default is dry-run)"
echo " --org Fly org slug (default: kilo-679)"
exit 0
;;
*) echo "Unknown option: $1"; exit 1 ;;
esac
done

command -v jq >/dev/null 2>&1 || { echo "ERROR: jq is required but not installed."; exit 1; }
command -v fly >/dev/null 2>&1 || { echo "ERROR: flyctl (fly) is required but not installed."; exit 1; }

if $DRY_RUN; then
echo "[DRY RUN] Pass --apply to actually release IPs."
echo ""
fi

# List all apps in the org, filter to kiloclaw per-user apps (acct-* and dev-*)
APPS=$(fly apps list -o "$ORG" --json 2>/dev/null \
| jq -r '.[].Name // empty' \
| grep -E '^(acct|dev)-[0-9a-f]{20}$') || true

if [[ -z "$APPS" ]]; then
echo "No kiloclaw apps found in org $ORG."
exit 0
fi

APP_COUNT=$(echo "$APPS" | wc -l | tr -d ' ')
echo "Found $APP_COUNT kiloclaw app(s) in org $ORG."
echo ""

RELEASED=0
SKIPPED=0
ERRORS=0

while IFS= read -r APP; do
# Get IPs as JSON; skip app on failure (e.g. app in bad state)
IP_JSON=$(fly ips list -a "$APP" --json 2>/dev/null) || {
echo " [$APP] WARN: failed to list IPs, skipping"
ERRORS=$((ERRORS + 1))
continue
}

# Extract IPv4 addresses (Type contains "v4": covers "v4" and "shared_v4")
IPV4_ADDRS=$(echo "$IP_JSON" \
| jq -r '.[] | select(.Type | test("v4")) | .Address') || true

if [[ -z "$IPV4_ADDRS" ]]; then
SKIPPED=$((SKIPPED + 1))
continue
fi

while IFS= read -r IP; do
if $DRY_RUN; then
echo " [$APP] Would release IPv4: $IP"
else
if fly ips release "$IP" -a "$APP" 2>/dev/null; then
echo " [$APP] Released IPv4: $IP"
else
echo " [$APP] ERROR: failed to release $IP"
ERRORS=$((ERRORS + 1))
continue
fi
fi
RELEASED=$((RELEASED + 1))
done <<< "$IPV4_ADDRS"
done <<< "$APPS"

echo ""
if $DRY_RUN; then
echo "Dry run complete: $RELEASED IP(s) would be released, $SKIPPED app(s) had no IPv4, $ERRORS error(s)."
echo "Run with --apply to execute."
else
echo "Done: $RELEASED IP(s) released, $SKIPPED app(s) had no IPv4, $ERRORS error(s)."
fi
75 changes: 15 additions & 60 deletions kiloclaw/src/durable-objects/kiloclaw-app.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ vi.mock('../fly/apps', async () => {
createApp: vi.fn().mockResolvedValue({ id: 'app-id', created_at: 1234567890 }),
getApp: vi.fn().mockResolvedValue(null), // default: app doesn't exist yet
deleteApp: vi.fn().mockResolvedValue(undefined),
allocateIP: vi.fn().mockResolvedValue({ address: '::1', type: 'v6' }),
allocateIPv6: vi.fn().mockResolvedValue({ address: '::1', type: 'v6' }),
};
});

Expand Down Expand Up @@ -115,11 +115,11 @@ beforeEach(() => {
// Reset default mock behaviors
(appsClient.getApp as Mock).mockResolvedValue(null);
(appsClient.createApp as Mock).mockResolvedValue({ id: 'app-id', created_at: 1234567890 });
(appsClient.allocateIP as Mock).mockResolvedValue({ address: '::1', type: 'v6' });
(appsClient.allocateIPv6 as Mock).mockResolvedValue({ address: '::1', type: 'v6' });
});

describe('ensureApp', () => {
it('creates app, allocates both IPs, and sets env key on first call', async () => {
it('creates app, allocates IPv6, and sets env key on first call', async () => {
const { appDO, storage } = createAppDO();

const result = await appDO.ensureApp('user-1');
Expand All @@ -132,10 +132,8 @@ describe('ensureApp', () => {
'user-1',
'kiloclaw_user_id'
);
expect(appsClient.allocateIP).toHaveBeenCalledTimes(2);
expect(appsClient.allocateIP).toHaveBeenCalledWith('test-token', result.appName, 'v6');
expect(appsClient.allocateIP).toHaveBeenCalledWith('test-token', result.appName, 'shared_v4');
expect(storage._store.get('ipv4Allocated')).toBe(true);
expect(appsClient.allocateIPv6).toHaveBeenCalledTimes(1);
expect(appsClient.allocateIPv6).toHaveBeenCalledWith('test-token', result.appName);
expect(storage._store.get('ipv6Allocated')).toBe(true);
// Env key was set
expect(secretsClient.setAppSecret).toHaveBeenCalledWith(
Expand All @@ -154,7 +152,7 @@ describe('ensureApp', () => {
await appDO.ensureApp('user-1');

expect(appsClient.createApp).not.toHaveBeenCalled();
expect(appsClient.allocateIP).toHaveBeenCalledTimes(2);
expect(appsClient.allocateIPv6).toHaveBeenCalledTimes(1);
});

it('is idempotent — second call is a no-op', async () => {
Expand All @@ -167,60 +165,22 @@ describe('ensureApp', () => {
expect(result1.appName).toBe(result2.appName);
expect(appsClient.createApp).not.toHaveBeenCalled();
expect(appsClient.getApp).not.toHaveBeenCalled();
expect(appsClient.allocateIP).not.toHaveBeenCalled();
expect(appsClient.allocateIPv6).not.toHaveBeenCalled();
expect(secretsClient.setAppSecret).not.toHaveBeenCalled();
});

it('resumes from partial state — IPv6 done, IPv4 pending', async () => {
const storage = createFakeStorage();
storage._store.set('userId', 'user-1');
storage._store.set('flyAppName', 'acct-test');
storage._store.set('ipv6Allocated', true);
storage._store.set('ipv4Allocated', false);

// App already exists (since we already created it)
(appsClient.getApp as Mock).mockResolvedValue({ id: 'existing', created_at: 100 });

const { appDO } = createAppDO(storage);
await appDO.ensureApp('user-1');

// Should only allocate IPv4
expect(appsClient.allocateIP).toHaveBeenCalledTimes(1);
expect(appsClient.allocateIP).toHaveBeenCalledWith('test-token', 'acct-test', 'shared_v4');
});

it('arms retry alarm and rethrows on partial failure', async () => {
// IPv6 allocation will fail
(appsClient.allocateIP as Mock).mockRejectedValue(new FlyApiError('timeout', 503, 'retry'));
(appsClient.allocateIPv6 as Mock).mockRejectedValue(new FlyApiError('timeout', 503, 'retry'));

const { appDO, storage } = createAppDO();

await expect(appDO.ensureApp('user-1')).rejects.toThrow('timeout');

// App name was persisted (partial state saved before failure)
expect(storage._store.get('flyAppName')).toMatch(/^acct-/);
// IPs NOT allocated
// IPv6 NOT allocated
expect(storage._store.get('ipv6Allocated')).not.toBe(true);
expect(storage._store.get('ipv4Allocated')).not.toBe(true);
// Retry alarm armed
expect(storage._getAlarm()).not.toBeNull();
});

it('arms retry alarm when IPv4 allocation fails after IPv6 succeeds', async () => {
let callCount = 0;
(appsClient.allocateIP as Mock).mockImplementation(() => {
callCount++;
if (callCount === 1) return Promise.resolve({ address: '::1', type: 'v6' });
return Promise.reject(new FlyApiError('rate limit', 429, 'slow down'));
});

const { appDO, storage } = createAppDO();

await expect(appDO.ensureApp('user-1')).rejects.toThrow('rate limit');

// IPv6 was persisted, IPv4 was not
expect(storage._store.get('ipv6Allocated')).toBe(true);
expect(storage._store.get('ipv4Allocated')).not.toBe(true);
// Retry alarm armed
expect(storage._getAlarm()).not.toBeNull();
});
Expand Down Expand Up @@ -299,29 +259,27 @@ describe('destroyApp', () => {
});

describe('alarm retry', () => {
it('retries incomplete IP allocation on alarm', async () => {
it('retries incomplete IPv6 allocation on alarm', async () => {
const storage = createFakeStorage();
storage._store.set('userId', 'user-1');
storage._store.set('flyAppName', 'acct-test');
storage._store.set('ipv6Allocated', true);
storage._store.set('ipv4Allocated', false);
storage._store.set('ipv6Allocated', false);

// App exists
(appsClient.getApp as Mock).mockResolvedValue({ id: 'existing', created_at: 100 });

const { appDO } = createAppDO(storage);
await appDO.alarm();

expect(appsClient.allocateIP).toHaveBeenCalledWith('test-token', 'acct-test', 'shared_v4');
expect(storage._store.get('ipv4Allocated')).toBe(true);
expect(appsClient.allocateIPv6).toHaveBeenCalledWith('test-token', 'acct-test');
expect(storage._store.get('ipv6Allocated')).toBe(true);
});

it('reschedules alarm if retry fails', async () => {
const storage = createFakeStorage();
storage._store.set('userId', 'user-1');
storage._store.set('flyAppName', 'acct-test');
storage._store.set('ipv6Allocated', false);
storage._store.set('ipv4Allocated', false);
storage._store.set('envKeySet', false);

(appsClient.getApp as Mock).mockRejectedValue(new FlyApiError('timeout', 503, 'retry'));
Expand All @@ -337,15 +295,14 @@ describe('alarm retry', () => {
storage._store.set('userId', 'user-1');
storage._store.set('flyAppName', 'acct-test');
storage._store.set('ipv6Allocated', true);
storage._store.set('ipv4Allocated', true);
storage._store.set('envKeySet', true);
storage._store.set('envKey', 'test-key');

const { appDO } = createAppDO(storage);
await appDO.alarm();

expect(appsClient.getApp).not.toHaveBeenCalled();
expect(appsClient.allocateIP).not.toHaveBeenCalled();
expect(appsClient.allocateIPv6).not.toHaveBeenCalled();
expect(secretsClient.setAppSecret).not.toHaveBeenCalled();
});

Expand All @@ -354,7 +311,6 @@ describe('alarm retry', () => {
storage._store.set('userId', 'user-1');
storage._store.set('flyAppName', 'acct-test');
storage._store.set('ipv6Allocated', true);
storage._store.set('ipv4Allocated', true);
storage._store.set('envKeySet', false);

// App exists
Expand All @@ -364,7 +320,7 @@ describe('alarm retry', () => {
await appDO.alarm();

// Should only set the env key, not allocate IPs
expect(appsClient.allocateIP).not.toHaveBeenCalled();
expect(appsClient.allocateIPv6).not.toHaveBeenCalled();
expect(secretsClient.setAppSecret).toHaveBeenCalled();
expect(storage._store.get('envKeySet')).toBe(true);
});
Expand Down Expand Up @@ -405,7 +361,6 @@ describe('ensureEnvKey', () => {
const storage = createFakeStorage();
storage._store.set('userId', 'user-1');
storage._store.set('flyAppName', 'acct-test');
storage._store.set('ipv4Allocated', true);
storage._store.set('ipv6Allocated', true);
storage._store.set('envKeySet', false);

Expand Down
28 changes: 9 additions & 19 deletions kiloclaw/src/durable-objects/kiloclaw-app.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@
* Separate from KiloClawInstance to support future multi-instance per user,
* where one Fly App contains multiple instances (machines + volumes).
*
* The App DO ensures that each user has a Fly App with allocated IPs and
* an encryption key before any machines are created. ensureApp() is idempotent:
* safe to call multiple times, only creates the app + IPs + key on first call.
* The App DO ensures that each user has a Fly App with an allocated IPv6 address
* and an encryption key before any machines are created. ensureApp() is idempotent:
* safe to call multiple times, only creates the app + IP + key on first call.
*
* If setup partially fails, the alarm retries.
*/
Expand All @@ -27,6 +27,7 @@ import { METADATA_KEY_USER_ID } from './kiloclaw-instance';
const AppStateSchema = z.object({
userId: z.string().default(''),
flyAppName: z.string().nullable().default(null),
// Legacy: ipv4Allocated may still exist in storage for old DOs; parsed but ignored.
ipv4Allocated: z.boolean().default(false),
ipv6Allocated: z.boolean().default(false),
envKeySet: z.boolean().default(false),
Expand All @@ -49,7 +50,6 @@ export class KiloClawApp extends DurableObject<KiloClawEnv> {
private loaded = false;
private userId: string | null = null;
private flyAppName: string | null = null;
private ipv4Allocated = false;
private ipv6Allocated = false;
private envKeySet = false;
private envKey: string | null = null;
Expand All @@ -65,7 +65,6 @@ export class KiloClawApp extends DurableObject<KiloClawEnv> {
const s = parsed.data;
this.userId = s.userId || null;
this.flyAppName = s.flyAppName;
this.ipv4Allocated = s.ipv4Allocated;
this.ipv6Allocated = s.ipv6Allocated;
this.envKeySet = s.envKeySet;
this.envKey = s.envKey;
Expand All @@ -76,11 +75,11 @@ export class KiloClawApp extends DurableObject<KiloClawEnv> {

/** Check if all setup steps are complete. */
private isSetupComplete(): boolean {
return this.ipv4Allocated && this.ipv6Allocated && this.envKeySet;
return this.ipv6Allocated && this.envKeySet;
}

/**
* Ensure a Fly App exists for this user with IPs allocated and env key set.
* Ensure a Fly App exists for this user with IPv6 allocated and env key set.
* Idempotent: creates the app only if it doesn't exist yet.
* Returns the app name for callers to cache.
*/
Expand Down Expand Up @@ -112,7 +111,7 @@ export class KiloClawApp extends DurableObject<KiloClawEnv> {

try {
// Step 1: Create app if it doesn't exist
if (!this.ipv4Allocated || !this.ipv6Allocated) {
if (!this.ipv6Allocated) {
const existing = await apps.getApp({ apiToken }, appName);
if (!existing) {
await apps.createApp({ apiToken }, appName, orgSlug, userId, METADATA_KEY_USER_ID);
Expand All @@ -122,21 +121,13 @@ export class KiloClawApp extends DurableObject<KiloClawEnv> {

// Step 2: Allocate IPv6 if not done
if (!this.ipv6Allocated) {
await apps.allocateIP(apiToken, appName, 'v6');
await apps.allocateIPv6(apiToken, appName);
this.ipv6Allocated = true;
await this.ctx.storage.put({ ipv6Allocated: true } satisfies Partial<AppState>);
console.log('[AppDO] Allocated IPv6 for:', appName);
}

// Step 3: Allocate shared IPv4 if not done
if (!this.ipv4Allocated) {
await apps.allocateIP(apiToken, appName, 'shared_v4');
this.ipv4Allocated = true;
await this.ctx.storage.put({ ipv4Allocated: true } satisfies Partial<AppState>);
console.log('[AppDO] Allocated shared IPv4 for:', appName);
}

// Step 4: Generate and store env encryption key if not done.
// Step 3: Generate and store env encryption key if not done.
// Uses the same locked path as ensureEnvKey() to prevent interleaving.
if (!this.envKeySet) {
await this.ensureEnvKey(userId);
Expand Down Expand Up @@ -251,7 +242,6 @@ export class KiloClawApp extends DurableObject<KiloClawEnv> {

this.userId = null;
this.flyAppName = null;
this.ipv4Allocated = false;
this.ipv6Allocated = false;
this.envKeySet = false;
this.envKey = null;
Expand Down
Loading