diff --git a/firestore-bigquery-export/CHANGELOG.md b/firestore-bigquery-export/CHANGELOG.md index baae95d21..5fad3f733 100644 --- a/firestore-bigquery-export/CHANGELOG.md +++ b/firestore-bigquery-export/CHANGELOG.md @@ -1,3 +1,7 @@ +## Version 0.1.60 + +feat - configure a log level to control the verbosity of logs. + ## Version 0.1.59 docs - remove references to lifecycle backfill feature. diff --git a/firestore-bigquery-export/README.md b/firestore-bigquery-export/README.md index 088e59bdb..9112a421b 100644 --- a/firestore-bigquery-export/README.md +++ b/firestore-bigquery-export/README.md @@ -297,6 +297,8 @@ Available schema extensions table fields for clustering include: `document_id, d * Maximum number of enqueue attempts: This parameter will set the maximum number of attempts to enqueue a document to cloud tasks for export to BigQuery. +* Log level: The log level for the extension. The log level controls the verbosity of the extension's logs. The available log levels are: debug, info, warn, and error. To reduce the volume of logs, use a log level of warn or error. + **Cloud Functions:** diff --git a/firestore-bigquery-export/extension.yaml b/firestore-bigquery-export/extension.yaml index 0e664fc1b..c3bf5657c 100644 --- a/firestore-bigquery-export/extension.yaml +++ b/firestore-bigquery-export/extension.yaml @@ -13,7 +13,7 @@ # limitations under the License. name: firestore-bigquery-export -version: 0.1.59 +version: 0.1.60 specVersion: v1beta displayName: Stream Firestore to BigQuery @@ -461,6 +461,27 @@ params: validationErrorMessage: Please select an integer between 1 and 10 default: 3 + - param: LOG_LEVEL + label: Log level + description: >- + The log level for the extension. The log level controls the verbosity of + the extension's logs. The available log levels are: debug, info, warn, and + error. To reduce the volume of logs, use a log level of warn or error. + type: select + options: + - label: Debug + value: debug + - label: Info + value: info + - label: Warn + value: warn + - label: Error + value: error + - label: Silent + value: silent + default: info + required: true + events: # OLD event types for backward compatibility - type: firebase.extensions.firestore-counter.v1.onStart diff --git a/firestore-bigquery-export/functions/__tests__/__snapshots__/config.test.ts.snap b/firestore-bigquery-export/functions/__tests__/__snapshots__/config.test.ts.snap index 8a55674c0..66bd1f073 100644 --- a/firestore-bigquery-export/functions/__tests__/__snapshots__/config.test.ts.snap +++ b/firestore-bigquery-export/functions/__tests__/__snapshots__/config.test.ts.snap @@ -21,6 +21,7 @@ Object { "instanceId": undefined, "kmsKeyName": "test", "location": "us-central1", + "logLevel": "info", "maxDispatchesPerSecond": 10, "maxEnqueueAttempts": 3, "maxStaleness": undefined, diff --git a/firestore-bigquery-export/functions/__tests__/functions.test.ts b/firestore-bigquery-export/functions/__tests__/functions.test.ts index 8e2502345..64ae32c99 100644 --- a/firestore-bigquery-export/functions/__tests__/functions.test.ts +++ b/firestore-bigquery-export/functions/__tests__/functions.test.ts @@ -16,6 +16,21 @@ jest.mock("@firebaseextensions/firestore-bigquery-change-tracker", () => ({ UPDATE: 1, CREATE: 0, }, + LogLevel: { + DEBUG: "debug", + INFO: "info", + WARN: "warn", + ERROR: "error", + SILENT: "silent", + }, + Logger: jest.fn().mockImplementation(() => ({ + debug: jest.fn(), + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + log: jest.fn(), + setLogLevel: jest.fn(), + })), })); jest.mock("firebase-admin/functions", () => ({ diff --git a/firestore-bigquery-export/functions/package-lock.json b/firestore-bigquery-export/functions/package-lock.json index 74c1ce6c6..5251e359e 100644 --- a/firestore-bigquery-export/functions/package-lock.json +++ b/firestore-bigquery-export/functions/package-lock.json @@ -7,7 +7,7 @@ "name": "firestore-bigquery-export", "license": "Apache-2.0", "dependencies": { - "@firebaseextensions/firestore-bigquery-change-tracker": "^1.1.39", + "@firebaseextensions/firestore-bigquery-change-tracker": "^1.1.40", "@google-cloud/bigquery": "^7.6.0", "@types/chai": "^4.1.6", "@types/express-serve-static-core": "4.17.30", @@ -572,9 +572,9 @@ } }, "node_modules/@firebaseextensions/firestore-bigquery-change-tracker": { - "version": "1.1.39", - "resolved": "https://registry.npmjs.org/@firebaseextensions/firestore-bigquery-change-tracker/-/firestore-bigquery-change-tracker-1.1.39.tgz", - "integrity": "sha512-KLLdu6CN2azcisQtfxNnecXv44c4uypJ0II68jpKUbAUkfClSbutDCe5b25Wyt3Tkt7w4XGWFWhYx4IyuUPe8w==", + "version": "1.1.40", + "resolved": "https://registry.npmjs.org/@firebaseextensions/firestore-bigquery-change-tracker/-/firestore-bigquery-change-tracker-1.1.40.tgz", + "integrity": "sha512-e3qJpMgw2IlVWs6w/jV2E3cAd8byWYuYGbDJ3RUhvrxRihgIB06tQ5rupgiwzKASc06M/JcMHhAR1dxwHRyBUw==", "dependencies": { "@google-cloud/bigquery": "^7.6.0", "@google-cloud/resource-manager": "^5.1.0", diff --git a/firestore-bigquery-export/functions/package.json b/firestore-bigquery-export/functions/package.json index 83dffd3b6..8f45b31e2 100644 --- a/firestore-bigquery-export/functions/package.json +++ b/firestore-bigquery-export/functions/package.json @@ -13,7 +13,7 @@ "author": "Jan Wyszynski ", "license": "Apache-2.0", "dependencies": { - "@firebaseextensions/firestore-bigquery-change-tracker": "^1.1.39", + "@firebaseextensions/firestore-bigquery-change-tracker": "^1.1.40", "@google-cloud/bigquery": "^7.6.0", "@types/chai": "^4.1.6", "@types/express-serve-static-core": "4.17.30", diff --git a/firestore-bigquery-export/functions/src/config.ts b/firestore-bigquery-export/functions/src/config.ts index 8eb298fb7..2213b5f2d 100644 --- a/firestore-bigquery-export/functions/src/config.ts +++ b/firestore-bigquery-export/functions/src/config.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import { LogLevel } from "@firebaseextensions/firestore-bigquery-change-tracker"; function timePartitioning(type) { if ( @@ -73,4 +74,5 @@ export default { backupBucketName: process.env.BACKUP_GCS_BUCKET || `${process.env.PROJECT_ID}.appspot.com`, backupDir: `_${process.env.INSTANCE_ID || "firestore-bigquery-export"}`, + logLevel: process.env.LOG_LEVEL || LogLevel.INFO, }; diff --git a/firestore-bigquery-export/functions/src/index.ts b/firestore-bigquery-export/functions/src/index.ts index 32020919f..1f0807117 100644 --- a/firestore-bigquery-export/functions/src/index.ts +++ b/firestore-bigquery-export/functions/src/index.ts @@ -57,6 +57,7 @@ const eventTrackerConfig = { config.viewType === "materialized_incremental", maxStaleness: config.maxStaleness, refreshIntervalMinutes: config.refreshIntervalMinutes, + logLevel: config.logLevel, }; // Initialize the Firestore Event History Tracker with the given configuration. @@ -64,6 +65,7 @@ const eventTracker: FirestoreBigQueryEventHistoryTracker = new FirestoreBigQueryEventHistoryTracker(eventTrackerConfig); // Initialize logging. +logs.logger.setLogLevel(config.logLevel); logs.init(); /** Initialize Firebase Admin SDK if not already initialized */ @@ -212,10 +214,7 @@ export const fsexportbigquery = functions.firestore context ); } catch (err) { - functions.logger.warn( - "Failed to write event to BigQuery Immediately. Will attempt to Enqueue to Cloud Tasks.", - err - ); + logs.failedToWriteToBigQueryImmediately(err as Error); // Handle enqueue errors with retries and backup to GCS. await attemptToEnqueue( err, diff --git a/firestore-bigquery-export/functions/src/logs.ts b/firestore-bigquery-export/functions/src/logs.ts index d3a895fbf..1e6e0ca0f 100644 --- a/firestore-bigquery-export/functions/src/logs.ts +++ b/firestore-bigquery-export/functions/src/logs.ts @@ -13,24 +13,28 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { logger } from "firebase-functions"; import config from "./config"; -import { ChangeType } from "@firebaseextensions/firestore-bigquery-change-tracker"; +import { + ChangeType, + Logger, +} from "@firebaseextensions/firestore-bigquery-change-tracker"; + +export const logger = new Logger(); export const arrayFieldInvalid = (fieldName: string) => { logger.warn(`Array field '${fieldName}' does not contain an array, skipping`); }; export const bigQueryDatasetCreated = (datasetId: string) => { - logger.log(`Created BigQuery dataset: ${datasetId}`); + logger.info(`Created BigQuery dataset: ${datasetId}`); }; export const bigQueryDatasetCreating = (datasetId: string) => { - logger.log(`Creating BigQuery dataset: ${datasetId}`); + logger.debug(`Creating BigQuery dataset: ${datasetId}`); }; export const bigQueryDatasetExists = (datasetId: string) => { - logger.log(`BigQuery dataset already exists: ${datasetId}`); + logger.info(`BigQuery dataset already exists: ${datasetId}`); }; export const bigQueryErrorRecordingDocumentChange = (e: Error) => { @@ -38,106 +42,108 @@ export const bigQueryErrorRecordingDocumentChange = (e: Error) => { }; export const bigQueryLatestSnapshotViewQueryCreated = (query: string) => { - logger.log(`BigQuery latest snapshot view query:\n${query}`); + logger.info(`BigQuery latest snapshot view query:\n${query}`); }; export const bigQueryTableAlreadyExists = ( tableName: string, datasetName: string ) => { - logger.log( + logger.info( `BigQuery table with name ${tableName} already ` + `exists in dataset ${datasetName}!` ); }; export const bigQueryTableCreated = (tableName: string) => { - logger.log(`Created BigQuery table: ${tableName}`); + logger.info(`Created BigQuery table: ${tableName}`); }; export const bigQueryTableCreating = (tableName: string) => { - logger.log(`Creating BigQuery table: ${tableName}`); + logger.debug(`Creating BigQuery table: ${tableName}`); }; export const bigQueryTableUpdated = (tableName: string) => { - logger.log(`Updated existing BigQuery table: ${tableName}`); + logger.info(`Updated existing BigQuery table: ${tableName}`); }; export const bigQueryTableUpdating = (tableName: string) => { - logger.log(`Updating existing BigQuery table: ${tableName}`); + logger.debug(`Updating existing BigQuery table: ${tableName}`); }; export const bigQueryTableUpToDate = (tableName: string) => { - logger.log(`BigQuery table: ${tableName} is up to date`); + logger.info(`BigQuery table: ${tableName} is up to date`); }; export const bigQueryTableValidated = (tableName: string) => { - logger.log(`Validated existing BigQuery table: ${tableName}`); + logger.info(`Validated existing BigQuery table: ${tableName}`); }; export const bigQueryTableValidating = (tableName: string) => { - logger.log(`Validating existing BigQuery table: ${tableName}`); + logger.debug(`Validating existing BigQuery table: ${tableName}`); }; export const bigQueryUserDefinedFunctionCreating = ( functionDefinition: string ) => { - logger.log(`Creating BigQuery User-defined Function:\n${functionDefinition}`); + logger.debug( + `Creating BigQuery User-defined Function:\n${functionDefinition}` + ); }; export const bigQueryUserDefinedFunctionCreated = ( functionDefinition: string ) => { - logger.log(`Created BigQuery User-defined Function:\n${functionDefinition}`); + logger.info(`Created BigQuery User-defined Function:\n${functionDefinition}`); }; export const bigQueryViewCreated = (viewName: string) => { - logger.log(`Created BigQuery view: ${viewName}`); + logger.info(`Created BigQuery view: ${viewName}`); }; export const bigQueryViewCreating = (viewName: string) => { - logger.log(`Creating BigQuery view: ${viewName}`); + logger.debug(`Creating BigQuery view: ${viewName}`); }; export const bigQueryViewAlreadyExists = ( viewName: string, datasetName: string ) => { - logger.log( + logger.info( `View with id ${viewName} already exists in dataset ${datasetName}.` ); }; export const bigQueryViewUpdated = (viewName: string) => { - logger.log(`Updated existing BigQuery view: ${viewName}`); + logger.info(`Updated existing BigQuery view: ${viewName}`); }; export const bigQueryViewUpdating = (viewName: string) => { - logger.log(`Updating existing BigQuery view: ${viewName}`); + logger.debug(`Updating existing BigQuery view: ${viewName}`); }; export const bigQueryViewUpToDate = (viewName: string) => { - logger.log(`BigQuery view: ${viewName} is up to date`); + logger.info(`BigQuery view: ${viewName} is up to date`); }; export const bigQueryViewValidated = (viewName: string) => { - logger.log(`Validated existing BigQuery view: ${viewName}`); + logger.info(`Validated existing BigQuery view: ${viewName}`); }; export const bigQueryViewValidating = (viewName: string) => { - logger.log(`Validating existing BigQuery view: ${viewName}`); + logger.debug(`Validating existing BigQuery view: ${viewName}`); }; export const complete = () => { - logger.log("Completed execution of extension"); + logger.info("Completed execution of extension"); }; export const dataInserted = (rowCount: number) => { - logger.log(`Inserted ${rowCount} row(s) of data into BigQuery`); + logger.debug(`Inserted ${rowCount} row(s) of data into BigQuery`); }; export const dataInserting = (rowCount: number) => { - logger.log(`Inserting ${rowCount} row(s) of data into BigQuery`); + logger.debug(`Inserting ${rowCount} row(s) of data into BigQuery`); }; export const dataTypeInvalid = ( @@ -171,11 +177,11 @@ export const error = ( }; export const init = () => { - logger.log("Initializing extension with configuration", config); + logger.info("Initializing extension with configuration", config); }; export const start = () => { - logger.log("Started execution of extension with configuration", config); + logger.info("Started execution of extension with configuration", config); }; export const timestampMissingValue = (fieldName: string) => { @@ -218,3 +224,10 @@ export const logFailedEventAction = ( error, }); }; + +export const failedToWriteToBigQueryImmediately = (error: Error) => { + logger.warn( + "Failed to write event to BigQuery Immediately. Will attempt to Enqueue to Cloud Tasks.", + error + ); +};