From e41b15aa96019f24fbe04fe735fa841d3e6753d0 Mon Sep 17 00:00:00 2001 From: Taly Date: Wed, 11 Feb 2026 09:00:14 +0300 Subject: [PATCH 1/3] Add regex & memoize caches; optimize queries Prevent memory leaks and reduce work by introducing a RegExp compile cache and clearing memoization caches on clearCache. Key changes: - Override clearCache to clear regexpCache and any decorator-based memoize caches stored on the instance. - Add a private regexpCache Map and reuse compiled RegExp objects in getMatchedPattern to avoid repeated compilation. - Annotate getProjectPatterns with @memoize({ max: 100, ttl: MEMOIZATION_TTL, strategy: 'hash' }). - Tighten DB queries used with cache.get to only project _id and pass a 300s TTL to the cache.get calls to reduce payload and caching duration. - Improve log messages to include groupHash/totalCount or indicate none, for clearer diagnostics. - Change getEventCacheKey format to `event::` for clearer keys. These changes reduce memory usage, avoid repeated RegExp compilation, make logs more actionable, and minimize DB transfer sizes. --- workers/grouper/src/index.ts | 80 ++++++++++++++++++++++++++++-------- 1 file changed, 63 insertions(+), 17 deletions(-) diff --git a/workers/grouper/src/index.ts b/workers/grouper/src/index.ts index 542c701f..3b6cd27d 100644 --- a/workers/grouper/src/index.ts +++ b/workers/grouper/src/index.ts @@ -131,6 +131,35 @@ export default class GrouperWorker extends Worker { await this.redis.close(); } + /** + * Override clearCache to also clear memoization caches and RegExp cache + * This prevents memory leaks from decorator-based LRU caches + */ + public clearCache(): void { + super.clearCache(); + + /** + * Clear RegExp cache + */ + this.regexpCache.clear(); + + /** + * Clear memoization caches from decorators + * These are stored as properties on the instance + */ + const memoizeCachePrefix = 'memoizeCache:'; + + for (const key in this) { + if (key.startsWith(memoizeCachePrefix)) { + const cache = this[key] as any; + + if (cache && typeof cache.reset === 'function') { + cache.reset(); + } + } + } + } + /** * Task handling function * @@ -156,7 +185,7 @@ export default class GrouperWorker extends Worker { const similarEvent = await this.findSimilarEvent(task.projectId, task.payload.title); if (similarEvent) { - this.logger.info(`similar event: ${JSON.stringify(similarEvent)}`); + this.logger.info(`similar event found: groupHash=${similarEvent.groupHash}, totalCount=${similarEvent.totalCount}`); /** * Override group hash with found event's group hash @@ -386,7 +415,7 @@ export default class GrouperWorker extends Worker { try { const originalEvent = await this.findFirstEventByPattern(matchingPattern.pattern, projectId); - this.logger.info(`original event for pattern: ${JSON.stringify(originalEvent)}`); + this.logger.info(`original event for pattern found: groupHash=${originalEvent?.groupHash || 'none'}`); if (originalEvent) { return originalEvent; @@ -400,6 +429,11 @@ export default class GrouperWorker extends Worker { return undefined; } + /** + * Cache for compiled RegExp patterns to avoid repeated compilation + */ + private regexpCache = new Map(); + /** * Method that returns matched pattern for event, if event do not match any of patterns return null * @@ -416,7 +450,12 @@ export default class GrouperWorker extends Worker { } return patterns.filter(pattern => { - const patternRegExp = new RegExp(pattern.pattern); + let patternRegExp = this.regexpCache.get(pattern.pattern); + + if (!patternRegExp) { + patternRegExp = new RegExp(pattern.pattern); + this.regexpCache.set(pattern.pattern, patternRegExp); + } return title.match(patternRegExp); }).pop() || null; @@ -428,6 +467,7 @@ export default class GrouperWorker extends Worker { * @param projectId - id of the project to find related event patterns * @returns {ProjectEventGroupingPatternsDBScheme[]} EventPatterns object with projectId and list of patterns */ + @memoize({ max: 100, ttl: MEMOIZATION_TTL, strategy: 'hash' }) private async getProjectPatterns(projectId: string): Promise { const project = await this.accountsDb.getConnection() .collection('projects') @@ -478,11 +518,14 @@ export default class GrouperWorker extends Worker { const repetitionCacheKey = `repetitions:${task.projectId}:${existedEvent.groupHash}:${eventUser.id}`; const repetition = await this.cache.get(repetitionCacheKey, async () => { return this.eventsDb.getConnection().collection(`repetitions:${task.projectId}`) - .findOne({ - groupHash: existedEvent.groupHash, - 'payload.user.id': eventUser.id, - }); - }); + .findOne( + { + groupHash: existedEvent.groupHash, + 'payload.user.id': eventUser.id, + }, + { projection: { _id: 1 } } + ); + }, 300); if (repetition) { shouldIncrementRepetitionAffectedUsers = false; @@ -512,15 +555,18 @@ export default class GrouperWorker extends Worker { const repetitionDailyCacheKey = `repetitions:${task.projectId}:${existedEvent.groupHash}:${eventUser.id}:${eventMidnight}`; const repetitionDaily = await this.cache.get(repetitionDailyCacheKey, async () => { return this.eventsDb.getConnection().collection(`repetitions:${task.projectId}`) - .findOne({ - groupHash: existedEvent.groupHash, - 'payload.user.id': eventUser.id, - timestamp: { - $gte: eventMidnight, - $lt: eventNextMidnight, + .findOne( + { + groupHash: existedEvent.groupHash, + 'payload.user.id': eventUser.id, + timestamp: { + $gte: eventMidnight, + $lt: eventNextMidnight, + }, }, - }); - }); + { projection: { _id: 1 } } + ); + }, 300); /** * If daily repetition exists, don't increment daily affected users @@ -575,7 +621,7 @@ export default class GrouperWorker extends Worker { * @returns {string} cache key for event */ private async getEventCacheKey(projectId: string, groupHash: string): Promise { - return `${projectId}:${JSON.stringify({ groupHash })}`; + return `event:${projectId}:${groupHash}`; } /** From f5307d639fe3d600da8592d1461c1d3138badc34 Mon Sep 17 00:00:00 2001 From: Taly Date: Wed, 11 Feb 2026 09:03:11 +0300 Subject: [PATCH 2/3] Add REPETITION_CACHE_TTL and dedupe regexpCache Introduce REPETITION_CACHE_TTL (300s) and replace magic timeout literals with this constant for repetition cache lookups. Move the regexpCache Map declaration to the class top-level to avoid a duplicate declaration and add JSDoc comments for both the TTL and regexp cache to clarify intent and improve maintainability. --- workers/grouper/src/index.ts | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/workers/grouper/src/index.ts b/workers/grouper/src/index.ts index 3b6cd27d..9435a5bf 100644 --- a/workers/grouper/src/index.ts +++ b/workers/grouper/src/index.ts @@ -53,6 +53,11 @@ const DB_DUPLICATE_KEY_ERROR = '11000'; */ const MAX_CODE_LINE_LENGTH = 140; +/** + * TTL for repetition cache lookups in seconds + */ +const REPETITION_CACHE_TTL = 300; + /** * Worker for handling Javascript events */ @@ -87,6 +92,11 @@ export default class GrouperWorker extends Worker { */ private cacheCleanupInterval: NodeJS.Timeout | null = null; + /** + * Cache for compiled RegExp patterns to avoid repeated compilation + */ + private regexpCache = new Map(); + /** * Start consuming messages */ @@ -429,11 +439,6 @@ export default class GrouperWorker extends Worker { return undefined; } - /** - * Cache for compiled RegExp patterns to avoid repeated compilation - */ - private regexpCache = new Map(); - /** * Method that returns matched pattern for event, if event do not match any of patterns return null * @@ -525,7 +530,7 @@ export default class GrouperWorker extends Worker { }, { projection: { _id: 1 } } ); - }, 300); + }, REPETITION_CACHE_TTL); if (repetition) { shouldIncrementRepetitionAffectedUsers = false; @@ -566,7 +571,7 @@ export default class GrouperWorker extends Worker { }, { projection: { _id: 1 } } ); - }, 300); + }, REPETITION_CACHE_TTL); /** * If daily repetition exists, don't increment daily affected users From 36dd36f979920aee1a269122e30d099d440c20e1 Mon Sep 17 00:00:00 2001 From: Taly Date: Wed, 11 Feb 2026 09:21:08 +0300 Subject: [PATCH 3/3] Iterate only own keys when clearing memoize cache Replace a for..in loop over `this` with `Object.keys(this).forEach` to ensure only the object's own enumerable properties are inspected when clearing memoize caches. This avoids iterating inherited/prototype properties that could lead to unexpected behavior when resetting cached entries in workers/grouper/src/index.ts. --- workers/grouper/src/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workers/grouper/src/index.ts b/workers/grouper/src/index.ts index 9435a5bf..ab97f4ce 100644 --- a/workers/grouper/src/index.ts +++ b/workers/grouper/src/index.ts @@ -159,7 +159,7 @@ export default class GrouperWorker extends Worker { */ const memoizeCachePrefix = 'memoizeCache:'; - for (const key in this) { + Object.keys(this).forEach(key => { if (key.startsWith(memoizeCachePrefix)) { const cache = this[key] as any; @@ -167,7 +167,7 @@ export default class GrouperWorker extends Worker { cache.reset(); } } - } + }); } /**