Update scanner.ts

odota · Sep 16, 2024 · 9642be6 · 9642be6
1 parent 1edfd86
commit 9642be6
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/svc/scanner.ts b/svc/scanner.ts
@@ -47,7 +47,9 @@ async function scanApi(seqNum: number) {
     const resp =
       data && data.result && data.result.matches ? data.result.matches : [];
     console.log('[API] match_seq_num:%s, matches:%s', nextSeqNum, resp.length);
+    console.time('insert');
     await Promise.all(resp.map((match: ApiMatch) => processMatch(match)));
+    console.timeEnd('insert');
     // Completed inserting matches on this page so update redis
     if (resp.length) {
       nextSeqNum = resp[resp.length - 1].match_seq_num + 1;
@@ -88,6 +90,7 @@ async function processMatch(match: ApiMatch) {
       origin: 'scanner',
     });
     await redis.zadd('scanner_insert', match.match_id, match.match_id);
+    // To avoid dups we should always keep more matches here than SCANNER_OFFSET
     await redis.zremrangebyrank('scanner_insert', '0', '-100000');
   }
 }