fix: implement DNT in alt-gen and pdf-extract

src/api/crawler.ts

@@ -904,6 +904,7 @@ export class CrawlerHost extends RPCHost {
         }
         this.threadLocal.set('retainImages', opts.retainImages);
         this.threadLocal.set('noGfm', opts.noGfm);
+        this.threadLocal.set('DNT', Boolean(opts.doNotTrack))
 
         const crawlOpts: ExtraScrappingOptions = {
             proxyUrl: opts.proxyUrl,

src/services/alt-text.ts

@@ -5,6 +5,7 @@ import { CanvasService } from '../shared/services/canvas';
 import { ImageInterrogationManager } from '../shared/services/common-iminterrogate';
 import { ImgBrief } from './puppeteer';
 import { ImgAlt } from '../db/img-alt';
+import { AsyncLocalContext } from './async-context';
 
 const md5Hasher = new HashManager('md5', 'hex');
 
@@ -17,7 +18,8 @@ export class AltTextService extends AsyncService {
     constructor(
         protected globalLogger: Logger,
         protected imageInterrogator: ImageInterrogationManager,
-        protected canvasService: CanvasService
+        protected canvasService: CanvasService,
+        protected asyncLocalContext: AsyncLocalContext
     ) {
         super(...arguments);
     }
@@ -69,6 +71,11 @@ export class AltTextService extends AsyncService {
             this.logger.warn(`Unable to generate alt text for ${imgBrief.src}`, { err });
         }
 
+        if (this.asyncLocalContext.ctx.DNT) {
+            // Don't cache alt text if DNT is set
+            return;
+        }
+
         // Don't try again until the next day
         const expireMixin = generatedCaption ? {} : { expireAt: new Date(Date.now() + 1000 * 3600 * 24) };
 

src/services/pdf-extract.ts

@@ -10,6 +10,7 @@ import { FirebaseStorageBucketControl } from '../shared';
 import { randomUUID } from 'crypto';
 import type { PDFDocumentLoadingTask } from 'pdfjs-dist';
 import path from 'path';
+import { AsyncLocalContext } from './async-context';
 const utc = require('dayjs/plugin/utc');  // Import the UTC plugin
 dayjs.extend(utc);  // Extend dayjs with the UTC plugin
 const timezone = require('dayjs/plugin/timezone');
@@ -56,6 +57,7 @@ export class PDFExtractor extends AsyncService {
     constructor(
         protected globalLogger: Logger,
         protected firebaseObjectStorage: FirebaseStorageBucketControl,
+        protected asyncLocalContext: AsyncLocalContext,
     ) {
         super(...arguments);
     }
@@ -324,21 +326,23 @@ export class PDFExtractor extends AsyncService {
         try {
             extracted = await this.extract(data);
 
-            const theID = randomUUID();
-            await this.firebaseObjectStorage.saveFile(`pdfs/${theID}`,
-                Buffer.from(JSON.stringify(extracted), 'utf-8'), { contentType: 'application/json' });
-            PDFContent.save(
-                PDFContent.from({
-                    _id: theID,
-                    src: nameUrl,
-                    meta: extracted?.meta || {},
-                    urlDigest: digest,
-                    createdAt: new Date(),
-                    expireAt: new Date(Date.now() + this.cacheRetentionMs)
-                }).degradeForFireStore()
-            ).catch((r) => {
-                this.logger.warn(`Unable to cache PDF content for ${nameUrl}`, { err: r });
-            });
+            if (!this.asyncLocalContext.ctx.DNT) {
+                const theID = randomUUID();
+                await this.firebaseObjectStorage.saveFile(`pdfs/${theID}`,
+                    Buffer.from(JSON.stringify(extracted), 'utf-8'), { contentType: 'application/json' });
+                PDFContent.save(
+                    PDFContent.from({
+                        _id: theID,
+                        src: nameUrl,
+                        meta: extracted?.meta || {},
+                        urlDigest: digest,
+                        createdAt: new Date(),
+                        expireAt: new Date(Date.now() + this.cacheRetentionMs)
+                    }).degradeForFireStore()
+                ).catch((r) => {
+                    this.logger.warn(`Unable to cache PDF content for ${nameUrl}`, { err: r });
+                });
+            }
         } catch (err) {
             this.logger.warn(`Unable to extract from pdf ${nameUrl}`, { err });
             throw err;