diff --git a/workers/grouper/src/index.ts b/workers/grouper/src/index.ts index 542c701f..ab97f4ce 100644 --- a/workers/grouper/src/index.ts +++ b/workers/grouper/src/index.ts @@ -53,6 +53,11 @@ const DB_DUPLICATE_KEY_ERROR = '11000'; */ const MAX_CODE_LINE_LENGTH = 140; +/** + * TTL for repetition cache lookups in seconds + */ +const REPETITION_CACHE_TTL = 300; + /** * Worker for handling Javascript events */ @@ -87,6 +92,11 @@ export default class GrouperWorker extends Worker { */ private cacheCleanupInterval: NodeJS.Timeout | null = null; + /** + * Cache for compiled RegExp patterns to avoid repeated compilation + */ + private regexpCache = new Map(); + /** * Start consuming messages */ @@ -131,6 +141,35 @@ export default class GrouperWorker extends Worker { await this.redis.close(); } + /** + * Override clearCache to also clear memoization caches and RegExp cache + * This prevents memory leaks from decorator-based LRU caches + */ + public clearCache(): void { + super.clearCache(); + + /** + * Clear RegExp cache + */ + this.regexpCache.clear(); + + /** + * Clear memoization caches from decorators + * These are stored as properties on the instance + */ + const memoizeCachePrefix = 'memoizeCache:'; + + Object.keys(this).forEach(key => { + if (key.startsWith(memoizeCachePrefix)) { + const cache = this[key] as any; + + if (cache && typeof cache.reset === 'function') { + cache.reset(); + } + } + }); + } + /** * Task handling function * @@ -156,7 +195,7 @@ export default class GrouperWorker extends Worker { const similarEvent = await this.findSimilarEvent(task.projectId, task.payload.title); if (similarEvent) { - this.logger.info(`similar event: ${JSON.stringify(similarEvent)}`); + this.logger.info(`similar event found: groupHash=${similarEvent.groupHash}, totalCount=${similarEvent.totalCount}`); /** * Override group hash with found event's group hash @@ -386,7 +425,7 @@ export default class GrouperWorker extends Worker { try { const originalEvent = await this.findFirstEventByPattern(matchingPattern.pattern, projectId); - this.logger.info(`original event for pattern: ${JSON.stringify(originalEvent)}`); + this.logger.info(`original event for pattern found: groupHash=${originalEvent?.groupHash || 'none'}`); if (originalEvent) { return originalEvent; @@ -416,7 +455,12 @@ export default class GrouperWorker extends Worker { } return patterns.filter(pattern => { - const patternRegExp = new RegExp(pattern.pattern); + let patternRegExp = this.regexpCache.get(pattern.pattern); + + if (!patternRegExp) { + patternRegExp = new RegExp(pattern.pattern); + this.regexpCache.set(pattern.pattern, patternRegExp); + } return title.match(patternRegExp); }).pop() || null; @@ -428,6 +472,7 @@ export default class GrouperWorker extends Worker { * @param projectId - id of the project to find related event patterns * @returns {ProjectEventGroupingPatternsDBScheme[]} EventPatterns object with projectId and list of patterns */ + @memoize({ max: 100, ttl: MEMOIZATION_TTL, strategy: 'hash' }) private async getProjectPatterns(projectId: string): Promise { const project = await this.accountsDb.getConnection() .collection('projects') @@ -478,11 +523,14 @@ export default class GrouperWorker extends Worker { const repetitionCacheKey = `repetitions:${task.projectId}:${existedEvent.groupHash}:${eventUser.id}`; const repetition = await this.cache.get(repetitionCacheKey, async () => { return this.eventsDb.getConnection().collection(`repetitions:${task.projectId}`) - .findOne({ - groupHash: existedEvent.groupHash, - 'payload.user.id': eventUser.id, - }); - }); + .findOne( + { + groupHash: existedEvent.groupHash, + 'payload.user.id': eventUser.id, + }, + { projection: { _id: 1 } } + ); + }, REPETITION_CACHE_TTL); if (repetition) { shouldIncrementRepetitionAffectedUsers = false; @@ -512,15 +560,18 @@ export default class GrouperWorker extends Worker { const repetitionDailyCacheKey = `repetitions:${task.projectId}:${existedEvent.groupHash}:${eventUser.id}:${eventMidnight}`; const repetitionDaily = await this.cache.get(repetitionDailyCacheKey, async () => { return this.eventsDb.getConnection().collection(`repetitions:${task.projectId}`) - .findOne({ - groupHash: existedEvent.groupHash, - 'payload.user.id': eventUser.id, - timestamp: { - $gte: eventMidnight, - $lt: eventNextMidnight, + .findOne( + { + groupHash: existedEvent.groupHash, + 'payload.user.id': eventUser.id, + timestamp: { + $gte: eventMidnight, + $lt: eventNextMidnight, + }, }, - }); - }); + { projection: { _id: 1 } } + ); + }, REPETITION_CACHE_TTL); /** * If daily repetition exists, don't increment daily affected users @@ -575,7 +626,7 @@ export default class GrouperWorker extends Worker { * @returns {string} cache key for event */ private async getEventCacheKey(projectId: string, groupHash: string): Promise { - return `${projectId}:${JSON.stringify({ groupHash })}`; + return `event:${projectId}:${groupHash}`; } /**