From 3b7420d631a7998717b2af82a902febd167eedcf Mon Sep 17 00:00:00 2001 From: Xetera Date: Fri, 27 Mar 2026 19:00:01 +0300 Subject: [PATCH 01/10] fix: use query-doctor postgres patch in docker --- .github/workflows/custom.yaml | 5 ++++- action.yaml | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/.github/workflows/custom.yaml b/.github/workflows/custom.yaml index bdd02fe..813a1a6 100644 --- a/.github/workflows/custom.yaml +++ b/.github/workflows/custom.yaml @@ -43,8 +43,11 @@ jobs: - name: Run local GitHub Action uses: ./ + with: + # TODO: support pg 16 explicitly + postgres-version: 17 env: GITHUB_TOKEN: ${{ github.token }} - POSTGRES_URL: postgres://query_doctor@localhost:5432/testing + SOURCE_POSTGRES_URL: postgres://query_doctor@localhost:5432/testing LOG_PATH: /var/log/postgresql/postgres.log SITE_API_ENDPOINT: ${{ vars.SITE_API_ENDPOINT }} diff --git a/action.yaml b/action.yaml index 84893a2..b3680ce 100644 --- a/action.yaml +++ b/action.yaml @@ -8,6 +8,12 @@ branding: icon: "database" color: "blue" +inputs: + postgres-version: + description: "PostgreSQL version to use (14, 17, or 18)" + required: true + default: "18" + runs: using: "composite" steps: @@ -51,6 +57,17 @@ runs: sudo make install # Use sudo to install globally cd ${{ github.action_path }} # Return to action directory + - name: Start PostgreSQL + shell: bash + run: | + docker run -d \ + --name query-doctor-postgres \ + -p 5432:5432 \ + ghcr.io/query-doctor/postgres:pg-${{ inputs.postgres-version }} + until docker exec query-doctor-postgres pg_isready -U postgres; do + sleep 1 + done + # Run the application - name: Run Analyzer shell: bash @@ -61,3 +78,4 @@ runs: CI: "true" GITHUB_TOKEN: ${{ env.GITHUB_TOKEN }} SITE_API_ENDPOINT: ${{ env.SITE_API_ENDPOINT }} + POSTGRES_URL: postgresql://postgres@localhost:5432/postgres From cce979e826c5a3dfc7cae4a7cf37d2a544d9ac99 Mon Sep 17 00:00:00 2001 From: Xetera Date: Fri, 27 Mar 2026 19:42:38 +0300 Subject: [PATCH 02/10] feat: rework ci run to use `Remote` and `QueryOptimizer` --- src/main.ts | 18 +- src/remote/query-optimizer.ts | 4 + src/reporters/site-api.ts | 154 +++-------- src/runner.ts | 472 +++++++--------------------------- src/sql/recent-query.ts | 16 ++ 5 files changed, 154 insertions(+), 510 deletions(-) diff --git a/src/main.ts b/src/main.ts index 7e032c8..67ba066 100644 --- a/src/main.ts +++ b/src/main.ts @@ -15,9 +15,9 @@ import { formatCost, queryPreview } from "./reporters/github/github.ts"; import { DEFAULT_CONFIG, fetchAnalyzerConfig } from "./config.ts"; async function runInCI( - postgresUrl: Connectable, + targetPostgresUrl: Connectable, + sourcePostgresUrl: Connectable, logPath: string, - statisticsPath?: string, maxCost?: number, ) { const siteApiEndpoint = env.SITE_API_ENDPOINT; @@ -31,8 +31,8 @@ async function runInCI( : DEFAULT_CONFIG; const runner = await Runner.build({ - postgresUrl, - statisticsPath, + targetPostgresUrl, + sourcePostgresUrl, logPath, maxCost, ignoredQueryHashes: config.ignoredQueryHashes, @@ -85,8 +85,8 @@ async function runInCI( log.info( "main", `No baseline found on branch "${comparisonBranch}". Comparison will be skipped. ` + - `To establish a baseline, run the analyzer on pushes to "${comparisonBranch}" ` + - `(add "push: branches: [${comparisonBranch}]" to your workflow trigger).`, + `To establish a baseline, run the analyzer on pushes to "${comparisonBranch}" ` + + `(add "push: branches: [${comparisonBranch}]" to your workflow trigger).`, ); } } @@ -153,14 +153,18 @@ async function main() { core.setFailed("POSTGRES_URL environment variable is not set"); process.exit(1); } + if (!env.SOURCE_DATABASE_URL) { + core.setFailed("SOURCE_DATABASE_URL environment variable is not set"); + process.exit(1); + } if (!env.LOG_PATH) { core.setFailed("LOG_PATH environment variable is not set"); process.exit(1); } await runInCI( Connectable.fromString(env.POSTGRES_URL), + Connectable.fromString(env.SOURCE_DATABASE_URL), env.LOG_PATH, - env.STATISTICS_PATH, typeof env.MAX_COST === "number" ? env.MAX_COST : undefined, ); } else { diff --git a/src/remote/query-optimizer.ts b/src/remote/query-optimizer.ts index daabacf..061e5e9 100644 --- a/src/remote/query-optimizer.ts +++ b/src/remote/query-optimizer.ts @@ -100,6 +100,10 @@ export class QueryOptimizer extends EventEmitter { return this._finish.promise; } + get statisticsMode(): StatisticsMode { + return this.target?.statistics.mode ?? QueryOptimizer.defaultStatistics; + } + getDisabledIndexes(): PgIdentifier[] { return [...this.disabledIndexes]; } diff --git a/src/reporters/site-api.ts b/src/reporters/site-api.ts index 7ac4bb3..94f404f 100644 --- a/src/reporters/site-api.ts +++ b/src/reporters/site-api.ts @@ -1,7 +1,7 @@ import * as github from "@actions/github"; import type { IndexRecommendation, Nudge, SQLCommenterTag, TableReference } from "@query-doctor/core"; import { DEFAULT_CONFIG, type AnalyzerConfig } from "../config.ts"; -import type { QueryProcessResult } from "../runner.ts"; +import type { OptimizedQuery } from "../sql/recent-query.ts"; interface CiRunPayload { repo: string; @@ -25,25 +25,25 @@ export interface CiQueryPayload { export type CiOptimization = | { - state: "improvements_available"; - cost: number; - optimizedCost: number; - costReductionPercentage: number; - indexRecommendations: CiIndexRecommendation[]; - indexesUsed: string[]; - explainPlan?: object; - optimizedExplainPlan?: object; - } + state: "improvements_available"; + cost: number; + optimizedCost: number; + costReductionPercentage: number; + indexRecommendations: CiIndexRecommendation[]; + indexesUsed: string[]; + explainPlan?: object; + optimizedExplainPlan?: object; + } | { - state: "no_improvement_found"; - cost: number; - indexesUsed: string[]; - explainPlan?: object; - } + state: "no_improvement_found"; + cost: number; + indexesUsed: string[]; + explainPlan?: object; + } | { - state: "error"; - error: string; - }; + state: "error"; + error: string; + }; interface CiIndexRecommendation { schema: string; @@ -114,105 +114,25 @@ function mapIndexRecommendation(rec: IndexRecommendation): CiIndexRecommendation }; } -function mapResultToQuery(result: QueryProcessResult): CiQueryPayload | null { - switch (result.kind) { - case "recommendation": - return { - hash: result.recommendation.fingerprint, - query: result.rawQuery, - formattedQuery: result.recommendation.formattedQuery, - nudges: result.nudges, - tags: result.tags, - tableReferences: result.referencedTables ?? [], - optimization: { - state: "improvements_available", - cost: result.recommendation.baseCost, - optimizedCost: result.recommendation.optimizedCost, - costReductionPercentage: - result.recommendation.baseCost > 0 - ? ((result.recommendation.baseCost - result.recommendation.optimizedCost) / - result.recommendation.baseCost) * - 100 - : 0, - indexRecommendations: result.indexRecommendations.map(mapIndexRecommendation), - indexesUsed: result.recommendation.existingIndexes, - explainPlan: result.recommendation.baseExplainPlan, - optimizedExplainPlan: result.recommendation.explainPlan, - }, - }; - - case "no_improvement": - return { - hash: result.fingerprint, - query: result.rawQuery, - formattedQuery: result.formattedQuery, - nudges: result.nudges, - tags: result.tags, - tableReferences: result.referencedTables ?? [], - optimization: { - state: "no_improvement_found", - cost: result.cost, - indexesUsed: result.existingIndexes, - explainPlan: result.explainPlan, - }, - }; - - case "zero_cost_plan": - return { - hash: result.fingerprint, - query: result.rawQuery, - formattedQuery: result.formattedQuery, - nudges: result.nudges, - tags: result.tags, - tableReferences: result.referencedTables ?? [], - optimization: { - state: "no_improvement_found", - cost: 0, - indexesUsed: [], - explainPlan: result.explainPlan, - }, - }; - - case "error": - return { - hash: result.fingerprint, - query: result.rawQuery, - formattedQuery: result.formattedQuery, - nudges: result.nudges, - tags: result.tags, - tableReferences: result.referencedTables ?? [], - optimization: { - state: "error", - error: result.error.message, - }, - }; - - case "cost_past_threshold": - return { - hash: result.warning.fingerprint, - query: result.rawQuery, - formattedQuery: result.warning.formattedQuery, - nudges: result.nudges, - tags: result.tags, - tableReferences: result.referencedTables ?? [], - optimization: result.warning.optimization - ? { - state: "no_improvement_found", - cost: result.warning.baseCost, - indexesUsed: result.warning.optimization.existingIndexes, - explainPlan: result.warning.explainPlan, - } - : { - state: "no_improvement_found", - cost: result.warning.baseCost, - indexesUsed: [], - explainPlan: result.warning.explainPlan, - }, - }; - - case "invalid": - return null; +function mapResultToQuery(result: OptimizedQuery): CiQueryPayload | null { + const { optimization } = result; + if ( + optimization.state === "waiting" || + optimization.state === "optimizing" || + optimization.state === "not_supported" || + optimization.state === "timeout" + ) { + return null; } + return { + hash: result.hash, + query: result.query, + formattedQuery: result.formattedQuery, + nudges: result.nudges, + tags: result.tags, + tableReferences: result.tableReferences ?? [], + optimization, + }; } function getQueryCost(q: CiQueryPayload): number | null { @@ -228,7 +148,7 @@ function getQueryIndexes(q: CiQueryPayload): string[] { } export function buildQueries( - results: QueryProcessResult[], + results: OptimizedQuery[], config: AnalyzerConfig = DEFAULT_CONFIG, ): CiQueryPayload[] { const ignoredSet = new Set(config.ignoredQueryHashes); diff --git a/src/runner.ts b/src/runner.ts index 111c31c..e5098e5 100644 --- a/src/runner.ts +++ b/src/runner.ts @@ -1,27 +1,9 @@ import * as core from "@actions/core"; -import * as prettier from "prettier"; -import prettierPluginSql from "prettier-plugin-sql"; import csv from "fast-csv"; -import { Readable } from "node:stream"; -import { statSync, readFileSync } from "node:fs"; +import { statSync } from "node:fs"; import { spawn } from "node:child_process"; import { fingerprint } from "@libpg-query/parser"; import { preprocessEncodedJson } from "./sql/json.ts"; -import { - Analyzer, - ExportedStats, - IndexedTable, - IndexOptimizer, - type IndexRecommendation, - type Nudge, - type SQLCommenterTag, - type TableReference, - OptimizeResult, - type Postgres, - PostgresQueryBuilder, - Statistics, - StatisticsMode, -} from "@query-doctor/core"; import { ExplainedLog } from "./sql/pg_log.ts"; import { GithubReporter } from "./reporters/github/github.ts"; import { @@ -29,52 +11,39 @@ import { type ReportContext, type ReportIndexRecommendation, type ReportQueryCostWarning, - type ReportStatistics, } from "./reporters/reporter.ts"; import { DEFAULT_CONFIG, type AnalyzerConfig } from "./config.ts"; -const bgBrightMagenta = (s: string) => `\x1b[105m${s}\x1b[0m`; -const yellow = (s: string) => `\x1b[33m${s}\x1b[0m`; -const blue = (s: string) => `\x1b[34m${s}\x1b[0m`; import { env } from "./env.ts"; -import { connectToSource } from "./sql/postgresjs.ts"; -import { parse } from "@libpg-query/parser"; import { Connectable } from "./sync/connectable.ts"; +import { Remote } from "./remote/remote.ts"; +import { ConnectionManager } from "./sync/connection-manager.ts"; +import { RecentQuery } from "./sql/recent-query.ts"; +import { QueryHash } from "./sql/recent-query.ts"; +import type { OptimizedQuery } from "./sql/recent-query.ts"; export class Runner { - private readonly seenQueries = new Set(); - public readonly queryStats: ReportStatistics = { - total: 0, - errored: 0, - matched: 0, - optimized: 0, - }; constructor( - private readonly db: Postgres, - private readonly optimizer: IndexOptimizer, - private readonly existingIndexes: IndexedTable[], - private readonly stats: Statistics, + private readonly remote: Remote, private readonly logPath: string, private readonly maxCost?: number, private readonly ignoredQueryHashes: Set = new Set(), ) { } static async build(options: { - postgresUrl: Connectable; + targetPostgresUrl: Connectable; + sourcePostgresUrl: Connectable; statisticsPath?: string; maxCost?: number; logPath: string; ignoredQueryHashes?: string[]; }) { - const db = connectToSource(options.postgresUrl); - const statisticsMode = Runner.decideStatisticsMode(options.statisticsPath); - const stats = await Statistics.fromPostgres(db, statisticsMode); - const existingIndexes = await stats.getExistingIndexes(); - const optimizer = new IndexOptimizer(db, stats, existingIndexes); + const remote = new Remote( + options.targetPostgresUrl, + ConnectionManager.forLocalDatabase(), + ); + await remote.syncFrom(options.sourcePostgresUrl); return new Runner( - db, - optimizer, - existingIndexes, - stats, + remote, options.logPath, options.maxCost, new Set(options.ignoredQueryHashes ?? []), @@ -82,7 +51,7 @@ export class Runner { } async close() { - await (this.db as unknown as { close(): Promise }).close(); + await this.remote.cleanup(); } async run(config: AnalyzerConfig = DEFAULT_CONFIG) { @@ -110,11 +79,10 @@ export class Runner { error = err; }); - const recommendations: ReportIndexRecommendation[] = []; - const queriesPastThreshold: ReportQueryCostWarning[] = []; - const allResults: QueryProcessResult[] = []; + let total = 0; console.time("total"); + const recentQueries: RecentQuery[] = []; for await (const chunk of stream) { const [ _timestamp, @@ -135,6 +103,7 @@ export class Runner { if (loglevel !== "LOG" || !queryString.startsWith("plan:")) { continue; } + total++; const planString: string = queryString.split("plan:")[1].trim(); const json = preprocessEncodedJson(planString); if (!json) { @@ -151,30 +120,66 @@ export class Runner { ); continue; } - const result = await this.processQuery(parsed); - if (result.kind !== "invalid") { - allResults.push(result); + + const query = parsed.query; + const hash = QueryHash.parse(await fingerprint(query)); + if (this.ignoredQueryHashes.has(hash)) { + continue; } - switch (result.kind) { - case "error": - this.queryStats.errored++; - break; - case "cost_past_threshold": - queriesPastThreshold.push(result.warning); - break; - case "recommendation": - recommendations.push(result.recommendation); - break; - case "no_improvement": - case "zero_cost_plan": - case "invalid": - break; + if (parsed.isIntrospection) { + continue; } + + const recentQuery = await RecentQuery.fromLogEntry(query, hash); + recentQueries.push(recentQuery) } + await this.remote.optimizer.addQueries(recentQueries); + await new Promise((resolve) => child.on("close", () => resolve())); + await this.remote.optimizer.finish; + + const optimizedQueries = this.remote.optimizer.getQueries(); + console.log( - `Matched ${this.queryStats.matched} queries out of ${this.queryStats.total}`, + `Matched ${this.remote.optimizer.validQueriesProcessed} queries out of ${total}`, ); + + const recommendations: ReportIndexRecommendation[] = []; + const queriesPastThreshold: ReportQueryCostWarning[] = []; + const allResults: OptimizedQuery[] = []; + + for (const q of optimizedQueries) { + if (this.ignoredQueryHashes.has(q.hash)) { + continue; + } + allResults.push(q); + const { optimization } = q; + if (optimization.state === "improvements_available") { + recommendations.push({ + fingerprint: q.hash, + formattedQuery: q.formattedQuery, + baseCost: optimization.cost, + baseExplainPlan: optimization.explainPlan, + optimizedCost: optimization.optimizedCost, + existingIndexes: optimization.indexesUsed, + proposedIndexes: optimization.indexRecommendations.map((r) => r.definition), + explainPlan: optimization.optimizedExplainPlan, + }); + } else if ( + optimization.state === "no_improvement_found" && + typeof this.maxCost === "number" && + optimization.cost > this.maxCost + ) { + queriesPastThreshold.push({ + fingerprint: q.hash, + formattedQuery: q.formattedQuery, + baseCost: optimization.cost, + explainPlan: optimization.explainPlan, + maxCost: this.maxCost, + }); + } + } + const filteredRecommendations = config.minimumCost > 0 ? recommendations.filter((r) => r.baseCost > config.minimumCost) @@ -183,12 +188,10 @@ export class Runner { config.minimumCost > 0 ? queriesPastThreshold.filter((w) => w.baseCost > config.minimumCost) : queriesPastThreshold; - const statistics = deriveIndexStatistics(filteredRecommendations); - const timeElapsed = Date.now() - startDate.getTime(); + if (config.minimumCost > 0) { const filtered = - recommendations.length - - filteredRecommendations.length + + recommendations.length - filteredRecommendations.length + (queriesPastThreshold.length - filteredThresholdWarnings.length); if (filtered > 0) { console.log( @@ -196,11 +199,19 @@ export class Runner { ); } } + + const statistics = deriveIndexStatistics(filteredRecommendations); + const timeElapsed = Date.now() - startDate.getTime(); const reportContext: ReportContext = { - statisticsMode: this.stats.mode, + statisticsMode: this.remote.optimizer.statisticsMode, recommendations: filteredRecommendations, queriesPastThreshold: filteredThresholdWarnings, - queryStats: Object.freeze(this.queryStats), + queryStats: Object.freeze({ + total, + matched: this.remote.optimizer.validQueriesProcessed, + optimized: filteredRecommendations.length, + errored: optimizedQueries.filter((q) => q.optimization.state === "error").length, + }), statistics, error, metadata: { logSize, timeElapsed }, @@ -214,317 +225,6 @@ export class Runner { console.log(`Generating report (${reporter.provider()})`); await reporter.report(reportContext); } - - async processQuery(log: ExplainedLog): Promise { - this.queryStats.total++; - const { query } = log; - const queryFingerprint = await fingerprint(query); - if (this.ignoredQueryHashes.has(queryFingerprint)) { - if (env.DEBUG) { - console.log("Skipping ignored query", queryFingerprint); - } - return { kind: "invalid" }; - } - if (log.isIntrospection) { - if (env.DEBUG) { - console.log("Skipping introspection query", queryFingerprint); - } - return { kind: "invalid" }; - } - if (this.seenQueries.has(queryFingerprint)) { - if (env.DEBUG) { - console.log("Skipping duplicate query", queryFingerprint); - } - return { kind: "invalid" }; - } - this.seenQueries.add(queryFingerprint); - - const analyzer = new Analyzer(parse); - const formattedQuery = await this.formatQuery(query); - const { indexesToCheck, ansiHighlightedQuery, referencedTables, nudges, tags } = - await analyzer.analyze(formattedQuery); - - const selectsCatalog = referencedTables.find((ref) => - ref.table.startsWith("pg_"), - ); - if (selectsCatalog) { - if (env.DEBUG) { - console.log( - "Skipping query that selects from catalog tables", - selectsCatalog, - queryFingerprint, - ); - } - return { kind: "invalid" }; - } - const indexCandidates = analyzer.deriveIndexes( - this.stats.ownMetadata, - indexesToCheck, - referencedTables, - ); - if (indexCandidates.length === 0) { - if (env.DEBUG) { - console.log(ansiHighlightedQuery); - console.log("No index candidates found", queryFingerprint); - } - if (typeof this.maxCost === "number" && log.plan.cost > this.maxCost) { - return { - kind: "cost_past_threshold", - rawQuery: query, - nudges, - tags, - referencedTables, - warning: { - fingerprint: queryFingerprint, - formattedQuery, - baseCost: log.plan.cost, - explainPlan: log.plan.json, - maxCost: this.maxCost, - }, - }; - } - } - return core.group( - `query:${queryFingerprint}`, - async (): Promise => { - console.time(`timing`); - this.printLegend(); - console.log(ansiHighlightedQuery); - // TODO: give concrete type - let out: OptimizeResult; - this.queryStats.matched++; - try { - const builder = new PostgresQueryBuilder(query); - out = await this.optimizer.run(builder, indexCandidates); - } catch (err) { - console.error(err); - console.error( - `Something went wrong while running this query. Skipping`, - ); - // this.queryStats.errored++; - console.timeEnd(`timing`); - return { - kind: "error", - error: err as Error, - fingerprint: queryFingerprint, - rawQuery: query, - formattedQuery, - nudges, - tags, - referencedTables, - }; - } - if (out.kind === "ok") { - const existingIndexesForQuery = Array.from(out.existingIndexes) - .map((index) => { - const existing = this.existingIndexes.find( - (e) => e.index_name === index, - ); - if (existing) { - return `${existing.schema_name}.${existing.table_name}(${existing.index_columns - .map((c) => `"${c.name}" ${c.order}`) - .join(", ")})`; - } - }) - .filter((i) => i !== undefined); - if (out.newIndexes.size > 0) { - const costReductionPct = out.baseCost > 0 - ? ((out.baseCost - out.finalCost) / out.baseCost) * 100 - : 0; - if (Math.round(costReductionPct) <= 0) { - console.log( - `Skipping recommendation with ${costReductionPct.toFixed(1)}% cost reduction (rounds to 0%)`, - ); - console.timeEnd(`timing`); - return { - kind: "no_improvement", - fingerprint: queryFingerprint, - rawQuery: query, - formattedQuery, - cost: out.baseCost, - existingIndexes: existingIndexesForQuery, - nudges, - tags, - referencedTables, - explainPlan: out.baseExplainPlan, - }; - } - this.queryStats.optimized++; - const newIndexRecommendations = Array.from(out.newIndexes) - .map((n) => out.triedIndexes.get(n)) - .filter((n) => n !== undefined); - const newIndexes = newIndexRecommendations.map((n) => n.definition); - console.log(`New indexes: ${newIndexes.join(", ")}`); - return { - kind: "recommendation", - rawQuery: query, - nudges, - tags, - referencedTables, - indexRecommendations: newIndexRecommendations, - recommendation: { - fingerprint: queryFingerprint, - formattedQuery, - baseCost: out.baseCost, - baseExplainPlan: out.baseExplainPlan, - optimizedCost: out.finalCost, - existingIndexes: existingIndexesForQuery, - proposedIndexes: newIndexes, - explainPlan: out.explainPlan, - }, - }; - } else { - console.log("No new indexes found"); - if ( - typeof this.maxCost === "number" && - out.finalCost > this.maxCost - ) { - console.log( - "Query cost is too high", - out.finalCost, - this.maxCost, - ); - return { - kind: "cost_past_threshold", - rawQuery: query, - nudges, - tags, - referencedTables, - warning: { - fingerprint: queryFingerprint, - formattedQuery, - baseCost: out.baseCost, - optimization: { - newCost: out.finalCost, - existingIndexes: existingIndexesForQuery, - proposedIndexes: [], - }, - explainPlan: out.explainPlan, - maxCost: this.maxCost, - }, - }; - } - return { - kind: "no_improvement", - fingerprint: queryFingerprint, - rawQuery: query, - formattedQuery, - cost: out.baseCost, - existingIndexes: existingIndexesForQuery, - nudges, - tags, - referencedTables, - explainPlan: out.baseExplainPlan, - }; - } - } else if (out.kind === "zero_cost_plan") { - console.log("Zero cost plan found"); - console.log(out); - console.timeEnd(`timing`); - return { - kind: "zero_cost_plan", - explainPlan: out.explainPlan, - fingerprint: queryFingerprint, - rawQuery: query, - formattedQuery, - nudges, - tags, - referencedTables, - }; - } - console.timeEnd(`timing`); - console.error(out); - throw new Error(`Unexpected output: ${out}`); - }, - ); - } - - private async formatQuery(query: string): Promise { - try { - return await prettier.format(query, { - parser: "sql", - plugins: [prettierPluginSql], - language: "postgresql", - keywordCase: "upper", - }); - } catch { - return query; - } - } - - private printLegend() { - console.log(`--Legend--------------------------`); - console.log(`| ${bgBrightMagenta(" column ")} | Candidate |`); - console.log(`| ${yellow(" column ")} | Ignored |`); - console.log(`| ${blue(" column ")} | Temp table reference |`); - console.log(`-----------------------------------`); - console.log(); - } - - private static decideStatisticsMode(path?: string): StatisticsMode { - if (path) { - const data = Runner.readStatisticsFile(path); - return Statistics.statsModeFromExport(data); - } else { - return Statistics.defaultStatsMode; - } - } - private static readStatisticsFile(path: string): ExportedStats[] { - const data = readFileSync(path); - const json = JSON.parse(new TextDecoder().decode(data)); - return ExportedStats.array().parse(json); - } } -export type QueryProcessResult = - | { - kind: "invalid"; - } - | { - kind: "cost_past_threshold"; - rawQuery: string; - nudges: Nudge[]; - tags: SQLCommenterTag[]; - referencedTables: TableReference[]; - warning: ReportQueryCostWarning; - } - | { - kind: "recommendation"; - rawQuery: string; - nudges: Nudge[]; - tags: SQLCommenterTag[]; - referencedTables: TableReference[]; - indexRecommendations: IndexRecommendation[]; - recommendation: ReportIndexRecommendation; - } - | { - kind: "no_improvement"; - fingerprint: string; - rawQuery: string; - formattedQuery: string; - cost: number; - existingIndexes: string[]; - nudges: Nudge[]; - tags: SQLCommenterTag[]; - referencedTables: TableReference[]; - explainPlan?: object; - } | { - kind: "error"; - error: Error; - fingerprint: string; - rawQuery: string; - formattedQuery: string; - nudges: Nudge[]; - tags: SQLCommenterTag[]; - referencedTables: TableReference[]; - } - | { - kind: "zero_cost_plan"; - explainPlan: object; - fingerprint: string; - rawQuery: string; - formattedQuery: string; - nudges: Nudge[]; - tags: SQLCommenterTag[]; - referencedTables: TableReference[]; - }; +export type QueryProcessResult = OptimizedQuery; diff --git a/src/sql/recent-query.ts b/src/sql/recent-query.ts index 35d5d55..ede7d9f 100644 --- a/src/sql/recent-query.ts +++ b/src/sql/recent-query.ts @@ -75,6 +75,22 @@ export class RecentQuery { */ private static readonly MAX_ANALYZABLE_QUERY_SIZE = 50_000; + static fromLogEntry(query: string, hash: QueryHash, seenAt: number = Date.now()) { + return RecentQuery.analyze( + { + query, + formattedQuery: query, + username: "", + meanTime: 0, + calls: "1", + rows: "0", + topLevel: true, + }, + hash, + seenAt, + ); + } + static async analyze( data: RawRecentQuery, hash: QueryHash, From e13653ed838cf0ca95a5ef0f1d29a686b208d63a Mon Sep 17 00:00:00 2001 From: Xetera Date: Fri, 27 Mar 2026 19:48:40 +0300 Subject: [PATCH 03/10] fix: select a random port for optimizing db --- .github/workflows/custom.yaml | 2 +- action.yaml | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/custom.yaml b/.github/workflows/custom.yaml index 813a1a6..1780f0e 100644 --- a/.github/workflows/custom.yaml +++ b/.github/workflows/custom.yaml @@ -48,6 +48,6 @@ jobs: postgres-version: 17 env: GITHUB_TOKEN: ${{ github.token }} - SOURCE_POSTGRES_URL: postgres://query_doctor@localhost:5432/testing + SOURCE_DATABASE_URL: postgres://query_doctor@localhost:5432/testing LOG_PATH: /var/log/postgresql/postgres.log SITE_API_ENDPOINT: ${{ vars.SITE_API_ENDPOINT }} diff --git a/action.yaml b/action.yaml index b3680ce..d4c51a1 100644 --- a/action.yaml +++ b/action.yaml @@ -60,9 +60,11 @@ runs: - name: Start PostgreSQL shell: bash run: | + PORT=$(shuf -i 10000-65000 -n 1) + echo "PG_PORT=$PORT" >> $GITHUB_ENV docker run -d \ --name query-doctor-postgres \ - -p 5432:5432 \ + -p $PORT:5432 \ ghcr.io/query-doctor/postgres:pg-${{ inputs.postgres-version }} until docker exec query-doctor-postgres pg_isready -U postgres; do sleep 1 @@ -78,4 +80,4 @@ runs: CI: "true" GITHUB_TOKEN: ${{ env.GITHUB_TOKEN }} SITE_API_ENDPOINT: ${{ env.SITE_API_ENDPOINT }} - POSTGRES_URL: postgresql://postgres@localhost:5432/postgres + POSTGRES_URL: postgresql://postgres@localhost:${{ env.PG_PORT }}/postgres From e2372e04fa0d70e010fb2aa6a00b985f6b65612e Mon Sep 17 00:00:00 2001 From: Xetera Date: Fri, 27 Mar 2026 20:48:20 +0300 Subject: [PATCH 04/10] fix: wait for optimizer to finish --- src/runner.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/runner.ts b/src/runner.ts index e5098e5..72088c9 100644 --- a/src/runner.ts +++ b/src/runner.ts @@ -42,6 +42,7 @@ export class Runner { ConnectionManager.forLocalDatabase(), ); await remote.syncFrom(options.sourcePostgresUrl); + await remote.optimizer.finish; return new Runner( remote, options.logPath, From 3369e80b6fbae4794e7becf26b24c74d449b795f Mon Sep 17 00:00:00 2001 From: Xetera Date: Fri, 27 Mar 2026 20:55:37 +0300 Subject: [PATCH 05/10] chore: log when schema sync fails --- src/remote/remote.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/remote/remote.ts b/src/remote/remote.ts index 69892ba..00b436e 100644 --- a/src/remote/remote.ts +++ b/src/remote/remote.ts @@ -108,6 +108,10 @@ export class Remote extends EventEmitter { this.getDatabaseInfo(source), ]); + if (restoreResult.status === "rejected") { + throw new Error(`Schema sync failed: ${restoreResult.reason}`); + } + if (fullSchema.status === "fulfilled") { this.schemaLoader?.update(fullSchema.value); } From b38fb190ea49fe1c0c79bbe6d5c2fc6f6896e070 Mon Sep 17 00:00:00 2001 From: Xetera Date: Fri, 27 Mar 2026 21:03:45 +0300 Subject: [PATCH 06/10] fix: use the right pg_dump binaries in CI --- action.yaml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/action.yaml b/action.yaml index d4c51a1..5e5010f 100644 --- a/action.yaml +++ b/action.yaml @@ -62,6 +62,14 @@ runs: run: | PORT=$(shuf -i 10000-65000 -n 1) echo "PG_PORT=$PORT" >> $GITHUB_ENV + sudo apt-get install -y curl ca-certificates + curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo gpg --dearmor -o /usr/share/keyrings/postgresql.gpg + echo "deb [signed-by=/usr/share/keyrings/postgresql.gpg] https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list + sudo apt-get update + sudo apt-get install -y postgresql-client-18 || sudo apt-get install -y postgresql-client-17 + PG_CLIENT_VERSION=$(ls /usr/lib/postgresql | sort -rn | head -1) + echo "PG_DUMP_BINARY=/usr/lib/postgresql/${PG_CLIENT_VERSION}/bin/pg_dump" >> $GITHUB_ENV + echo "PG_RESTORE_BINARY=/usr/lib/postgresql/${PG_CLIENT_VERSION}/bin/pg_restore" >> $GITHUB_ENV docker run -d \ --name query-doctor-postgres \ -p $PORT:5432 \ @@ -76,7 +84,6 @@ runs: working-directory: ${{ github.action_path }} run: npm run start env: - PG_DUMP_BINARY: /usr/bin/pg_dump CI: "true" GITHUB_TOKEN: ${{ env.GITHUB_TOKEN }} SITE_API_ENDPOINT: ${{ env.SITE_API_ENDPOINT }} From e852c12c3ffbe7673b7665e6d2796cba96a61ebe Mon Sep 17 00:00:00 2001 From: Xetera Date: Fri, 27 Mar 2026 21:15:24 +0300 Subject: [PATCH 07/10] feat: allow disabling the query loader --- src/remote/remote.ts | 7 +++++-- src/runner.ts | 3 +++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/remote/remote.ts b/src/remote/remote.ts index 00b436e..425dd2f 100644 --- a/src/remote/remote.ts +++ b/src/remote/remote.ts @@ -71,6 +71,7 @@ export class Remote extends EventEmitter { /** The manager for ONLY the source db connections */ private readonly sourceManager: ConnectionManager = ConnectionManager .forRemoteDatabase(), + private readonly options: { disableQueryLoader: boolean } = { disableQueryLoader: false } ) { super(); this.baseDbURL = targetURL.withDatabaseName(Remote.baseDbName); @@ -204,7 +205,7 @@ export class Remote extends EventEmitter { * there isn't already an in-flight request */ private async pollQueriesOnce() { - if (this.queryLoader && !this.isPolling) { + if (this.queryLoader && !this.isPolling && !this.options.disableQueryLoader) { try { this.isPolling = true; await this.queryLoader.poll(); @@ -377,7 +378,9 @@ export class Remote extends EventEmitter { log.error("Query loader exited", "remote"); this.queryLoader = undefined; }); - this.queryLoader.start(); + if (!this.options.disableQueryLoader) { + this.queryLoader.start(); + } } async cleanup(): Promise { diff --git a/src/runner.ts b/src/runner.ts index 72088c9..c03ea42 100644 --- a/src/runner.ts +++ b/src/runner.ts @@ -40,6 +40,9 @@ export class Runner { const remote = new Remote( options.targetPostgresUrl, ConnectionManager.forLocalDatabase(), + ConnectionManager.forRemoteDatabase(), + // queries are already sourced from logs + { disableQueryLoader: true } ); await remote.syncFrom(options.sourcePostgresUrl); await remote.optimizer.finish; From df0ee34153ae38f54272e62b4080550f104af5d1 Mon Sep 17 00:00:00 2001 From: Xetera Date: Fri, 27 Mar 2026 21:20:53 +0300 Subject: [PATCH 08/10] fix: handle errors with main --- src/main.ts | 8 +++++++- src/runner.ts | 3 ++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/main.ts b/src/main.ts index 67ba066..176acc9 100644 --- a/src/main.ts +++ b/src/main.ts @@ -100,6 +100,7 @@ async function runInCI( ); } + console.log("Creating report...") // Generate PR comment with comparison data await runner.report(reportContext); @@ -172,4 +173,9 @@ async function main() { } } -await main(); +try { + await main(); +} catch (error) { + console.error(error); + process.exit(1); +} diff --git a/src/runner.ts b/src/runner.ts index c03ea42..959ee8f 100644 --- a/src/runner.ts +++ b/src/runner.ts @@ -80,6 +80,7 @@ export class Runner { headers: false, }) .on("error", (err) => { + console.error("Got a pgbadger error", err); error = err; }); @@ -137,9 +138,9 @@ export class Runner { const recentQuery = await RecentQuery.fromLogEntry(query, hash); recentQueries.push(recentQuery) } + console.log("Finished pgbadger stream"); await this.remote.optimizer.addQueries(recentQueries); - await new Promise((resolve) => child.on("close", () => resolve())); await this.remote.optimizer.finish; const optimizedQueries = this.remote.optimizer.getQueries(); From 4e9721abe5ec3e39ec7e98dbf7d1f34fd627e0f1 Mon Sep 17 00:00:00 2001 From: Xetera Date: Fri, 27 Mar 2026 21:54:09 +0300 Subject: [PATCH 09/10] fix: cache pg install step --- action.yaml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/action.yaml b/action.yaml index 5e5010f..472c48a 100644 --- a/action.yaml +++ b/action.yaml @@ -57,16 +57,28 @@ runs: sudo make install # Use sudo to install globally cd ${{ github.action_path }} # Return to action directory - - name: Start PostgreSQL + - name: Cache postgresql-client + uses: actions/cache@v4 + id: pg-client-cache + with: + path: /usr/lib/postgresql + key: pg-client-${{ runner.os }}-18 + + - name: Install postgresql-client shell: bash + if: steps.pg-client-cache.outputs.cache-hit != 'true' run: | - PORT=$(shuf -i 10000-65000 -n 1) - echo "PG_PORT=$PORT" >> $GITHUB_ENV sudo apt-get install -y curl ca-certificates curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo gpg --dearmor -o /usr/share/keyrings/postgresql.gpg echo "deb [signed-by=/usr/share/keyrings/postgresql.gpg] https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list sudo apt-get update sudo apt-get install -y postgresql-client-18 || sudo apt-get install -y postgresql-client-17 + + - name: Start PostgreSQL + shell: bash + run: | + PORT=$(shuf -i 10000-65000 -n 1) + echo "PG_PORT=$PORT" >> $GITHUB_ENV PG_CLIENT_VERSION=$(ls /usr/lib/postgresql | sort -rn | head -1) echo "PG_DUMP_BINARY=/usr/lib/postgresql/${PG_CLIENT_VERSION}/bin/pg_dump" >> $GITHUB_ENV echo "PG_RESTORE_BINARY=/usr/lib/postgresql/${PG_CLIENT_VERSION}/bin/pg_restore" >> $GITHUB_ENV From 28e1978a888e007565d532a4a9f9820519a06172 Mon Sep 17 00:00:00 2001 From: Xetera Date: Mon, 30 Mar 2026 17:25:29 +0300 Subject: [PATCH 10/10] fix: resolve index names --- src/remote/query-optimizer.ts | 9 +++++++-- src/runner.ts | 17 ++++++++++++++++- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/src/remote/query-optimizer.ts b/src/remote/query-optimizer.ts index 061e5e9..2152fbd 100644 --- a/src/remote/query-optimizer.ts +++ b/src/remote/query-optimizer.ts @@ -49,6 +49,7 @@ export class QueryOptimizer extends EventEmitter { private readonly queries = new Map(); private readonly disabledIndexes = new DisabledIndexes(); + private existingIndexes: IndexedTable[] = []; private target?: Target; private semaphore = new Sema(QueryOptimizer.MAX_CONCURRENCY); private _finish = Promise.withResolvers(); @@ -104,6 +105,10 @@ export class QueryOptimizer extends EventEmitter { return this.target?.statistics.mode ?? QueryOptimizer.defaultStatistics; } + getExistingIndexes(): IndexedTable[] { + return this.existingIndexes; + } + getDisabledIndexes(): PgIdentifier[] { return [...this.disabledIndexes]; } @@ -132,8 +137,8 @@ export class QueryOptimizer extends EventEmitter { const pg = this.manager.getOrCreateConnection(this.connectable); const ownStats = await Statistics.dumpStats(pg, version, "full"); const statistics = new Statistics(pg, version, ownStats, statsMode); - const existingIndexes = await statistics.getExistingIndexes(); - const filteredIndexes = this.filterDisabledIndexes(existingIndexes); + this.existingIndexes = await statistics.getExistingIndexes(); + const filteredIndexes = this.filterDisabledIndexes(this.existingIndexes); const optimizer = new IndexOptimizer(pg, statistics, filteredIndexes, { trace: false, }); diff --git a/src/runner.ts b/src/runner.ts index 959ee8f..44f5bd6 100644 --- a/src/runner.ts +++ b/src/runner.ts @@ -144,6 +144,15 @@ export class Runner { await this.remote.optimizer.finish; const optimizedQueries = this.remote.optimizer.getQueries(); + const existingIndexes = this.remote.optimizer.getExistingIndexes(); + + const resolveIndexNames = (names: string[]) => + names.map((name) => { + const idx = existingIndexes.find((e) => e.index_name === name); + return idx + ? `${idx.schema_name}.${idx.table_name}(${idx.index_columns.map((c) => `"${c.name}" ${c.order}`).join(", ")})` + : name; + }); console.log( `Matched ${this.remote.optimizer.validQueriesProcessed} queries out of ${total}`, @@ -157,8 +166,14 @@ export class Runner { if (this.ignoredQueryHashes.has(q.hash)) { continue; } - allResults.push(q); const { optimization } = q; + if ( + optimization.state === "improvements_available" || + optimization.state === "no_improvement_found" + ) { + optimization.indexesUsed = resolveIndexNames(optimization.indexesUsed); + } + allResults.push(q); if (optimization.state === "improvements_available") { recommendations.push({ fingerprint: q.hash,