Repository: parcel-bundler/parcel
File: packages/core/cache/src/LMDBCache.js (JavaScript)

Code before:
    return this.fs.createReadStream(path.join(this.dir, key));
  }

  async setStream(key: string, stream: Readable): Promise<void> {
    await this.setBlob(key, await bufferStream(stream));
  }

  async getBlob(key: string): Promise<Buffer> {
    let buffer = await this.getBuffer(key);
    if (buffer == null) throw new Error(`Key ${key} not found in cache`);
    return buffer;
  }

  async setBlob(key: string, contents: Buffer | string): Promise<void> {
    if (isLargeBlob(contents)) {
      // Remove the old blob if it has been 'upgraded' to large blob storage.
      if (this.store.get(key) != null) await this.store.remove(key);
      await this.setLargeBlob(key, contents);
    } else {
      await this.store.put(key, contents);
    }
  }

  async getBuffer(key: string): Promise<?Buffer> {
    let buffer = this.store.get(key);
    if (buffer == null && (await this.hasLargeBlob(key))) {
      buffer = await this.getLargeBlob(key);
    }
    return buffer;
  }

  hasLargeBlob(key: string): Promise<boolean> {
    return this.fs.exists(path.join(this.dir, key));
  }

  getLargeBlob(key: string): Promise<Buffer> {
    return this.fs.readFile(path.join(this.dir, key));
  }

  async setLargeBlob(key: string, contents: Buffer | string): Promise<void> {
    await this.fs.writeFile(path.join(this.dir, key), contents);
  }
}

// lmbd-store decodes cached binary data into a Node Buffer
// via `Nan::NewBuffer`, which enforces a max size of ~1GB.
// We subtract 9 bytes to account for any compression heaader
// added by lmbd-store when encoding the data.
// See: https://github.com/nodejs/nan/issues/883
const MAX_BUFFER_SIZE = 0x3fffffff - 9;


Review comment (performance):
I'm worried that this will result in a lot of extra `stat` calls to the filesystem since it will be checked for every key that is missing from the cache (everything on initial builds). Can we keep checking for large blobs explicitly opt-in? I thought we only needed it for certain known large entries (e.g. graphs)?

Code after:
    return this.fs.createReadStream(path.join(this.dir, key));
  }

  setStream(key: string, stream: Readable): Promise<void> {
    return new Promise((resolve, reject) => {
      stream
        .pipe(this.fs.createWriteStream(path.join(this.dir, key)))
        .on('error', reject)
        .on('finish', resolve);
    });
  }

  getBlob(key: string): Promise<Buffer> {
    let buffer = this.store.get(key);
    return buffer != null
      ? Promise.resolve(buffer)
      : Promise.reject(new Error(`Key ${key} not found in cache`));
  }

  async setBlob(key: string, contents: Buffer | string): Promise<void> {
    await this.store.put(key, contents);
  }

  getBuffer(key: string): Promise<?Buffer> {
    return Promise.resolve(this.store.get(key));
  }

  hasLargeBlob(key: string): Promise<boolean> {
    return this.fs.exists(path.join(this.dir, key));
  }

  getLargeBlob(key: string): Promise<Buffer> {
    return this.fs.readFile(path.join(this.dir, key));
  }

  async setLargeBlob(key: string, contents: Buffer | string): Promise<void> {
    await this.fs.writeFile(path.join(this.dir, key), contents);
  }
}

registerSerializableClass(`${packageJson.version}:LMDBCache`, LMDBCache);