Repository: electron/electron
Pull request: ci: upload object change stats to Datadog

#### Description of Change

<!--
Thank you for your Pull Request. Please provide a description above and review
the requirements below.

Contributors guide: https://github.com/electron/electron/blob/main/CONTRIBUTING.md

NOTE: PRS submitted without this template will be automatically closed.
-->

The goal here is to track in Datadog how much build thrash we're getting from upstream Chromium changes. This uploads everything for completeness in case there's anything else we're interested in in the future (like build thrash from patch changes), but it's easy to identify only builds related to a Chromium version bump with the `chromium-version-changed:true` tag.

How the tracking works is pretty simple: every successful build will upload a JSON file artifact with the Chromium version and the checksums of all `.o` files in the build. They will also try to download that data from the previous commit and if it's available run a diff between the two to give us some data on thrash. The most interesting being what percentage of `.o` files had their content changed as a result of the current build, and how many bytes of on-disk content does that represent. Also gonna track how many new `.o` files were introduced by the build.

This should let us make a nice dashboard graph on Datadog tracking this data over time so we can see what the overall trend is and what impact any upstream attempts to fight this thrash are having.

> [!NOTE]
> This isn't meant to be a foolproof or exhaustive metric, just a straight forward for us to get some metrics on build thrash.

#### Checklist
<!-- Remove items that do not apply. For completed items, change [ ] to [x]. -->

- [x] PR description included

#### Release Notes

Notes: none <!-- Please add a one-line description for app developers to read in the release notes, or 'none' if no notes relevant to app developers. Examples and help on special cases: https://github.com/electron/clerk/blob/main/README.md#examples -->


Changed files:
filename: .github/actions/build-electron/action.yml
patch: @@ -47,6 +47,20 @@ runs:
     - name: Add Clang problem matcher
       shell: bash
       run: echo "::add-matcher::src/electron/.github/problem-matchers/clang.json"
+    - name: Download previous object checksums
+      uses: dawidd6/action-download-artifact@09b07ec687d10771279a426c79925ee415c12906 # v17
+      if: ${{ (github.event_name == 'push' || github.event_name == 'pull_request') && inputs.is-asan != 'true' }}
+      with:
+        name: object_checksums_${{ inputs.artifact-platform }}_${{ inputs.target-arch }}
+        commit: ${{ case(github.event_name == 'push', github.event.push.before, github.event.pull_request.base.sha) }}
+        path: src
+        if_no_artifact_found: ignore
+    - name: Move previous object checksums
+      shell: bash
+      run: |
+        if [ -f src/object-checksums_${{ inputs.artifact-platform }}_${{ inputs.target-arch }}.json ]; then
+          mv src/object-checksums_${{ inputs.artifact-platform }}_${{ inputs.target-arch }}.json src/previous-object-checksums.json
+        fi
     - name: Build Electron ${{ inputs.step-suffix }}
       if: ${{ inputs.target-platform != 'win' }}
       shell: bash
@@ -72,12 +86,17 @@ runs:
         cp out/Default/.ninja_log out/electron_ninja_log
         node electron/script/check-symlinks.js
 
-        # Upload build stats to Datadog
-        if ! [ -z $DD_API_KEY ]; then
-          npx node electron/script/build-stats.mjs out/Default/siso.INFO --upload-stats || true          
+        # Build stats and object checksums
+        BUILD_STATS_ARGS="out/Default/siso.INFO --out-dir out/Default --output-object-checksums object-checksums.${{ inputs.artifact-platform }}_${{ inputs.target-arch }}.json"
+        if [ -f previous-object-checksums.json ]; then
+          BUILD_STATS_ARGS="$BUILD_STATS_ARGS --input-object-checksums previous-object-checksums.json"
+        fi
+        if ! [ -z "$DD_API_KEY" ]; then
+          BUILD_STATS_ARGS="$BUILD_STATS_ARGS --upload-stats"
         else
           echo "Skipping build-stats.mjs upload because DD_API_KEY is not set"
         fi
+        node electron/script/build-stats.mjs $BUILD_STATS_ARGS || true
     - name: Build Electron (Windows) ${{ inputs.step-suffix }}
       if: ${{ inputs.target-platform == 'win' }}
       shell: powershell
@@ -95,16 +114,21 @@ runs:
         Copy-Item out\Default\.ninja_log out\electron_ninja_log
         node electron\script\check-symlinks.js
 
-        # Upload build stats to Datadog
+        # Build stats and object checksums
+        $statsArgs = @("out\Default\siso.exe.INFO", "--out-dir", "out\Default", "--output-object-checksums", "object-checksums.${{ inputs.artifact-platform }}_${{ inputs.target-arch }}.json")
+        if (Test-Path previous-object-checksums.json) {
+          $statsArgs += @("--input-object-checksums", "previous-object-checksums.json")
+        }
         if ($env:DD_API_KEY) {
-          try {
-            npx node electron\script\build-stats.mjs out\Default\siso.exe.INFO --upload-stats ; $LASTEXITCODE = 0
-          } catch {
-            Write-Host "Build stats upload failed, continuing..."
-          }
+          $statsArgs += "--upload-stats"
         } else {
           Write-Host "Skipping build-stats.mjs upload because DD_API_KEY is not set"
         }
+        try {
+          & node electron\script\build-stats.mjs @statsArgs ; $LASTEXITCODE = 0
+        } catch {
+          Write-Host "Build stats failed, continuing..."
+        }
     - name: Verify dist.zip ${{ inputs.step-suffix }}
       shell: bash
       run: |
@@ -292,3 +316,10 @@ runs:
       with:
         name: out_gen_artifacts_${{ env.ARTIFACT_KEY }}
         path: ./src/out/Default/gen
+    - name: Upload Object Checksums ${{ inputs.step-suffix }}
+      if: ${{ always() && !cancelled() && inputs.is-asan != 'true' }}
+      uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+      with:
+        name: object_checksums_${{ inputs.artifact-platform }}_${{ inputs.target-arch }}
+        path: ./src/object-checksums.${{ inputs.artifact-platform }}_${{ inputs.target-arch }}.json
+        archive: false

filename: script/build-stats.mjs
patch: @@ -1,22 +1,143 @@
+import { createHash } from 'node:crypto';
 import * as fs from 'node:fs/promises';
+import { dirname, resolve } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { parseArgs } from 'node:util';
 
+import { getChromiumVersionFromDEPS } from './lib/utils.js';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ELECTRON_DIR = resolve(__dirname, '..');
+
+function getCommonTags () {
+  const tags = [];
+
+  if (process.env.TARGET_ARCH) tags.push(`target-arch:${process.env.TARGET_ARCH}`);
+  if (process.env.TARGET_PLATFORM) tags.push(`target-platform:${process.env.TARGET_PLATFORM}`);
+  if (process.env.GITHUB_HEAD_REF) {
+    // Will be set in pull requests
+    tags.push(`branch:${process.env.GITHUB_HEAD_REF}`);
+  } else if (process.env.GITHUB_REF_NAME) {
+    // Will be set for release branches
+    tags.push(`branch:${process.env.GITHUB_REF_NAME}`);
+  }
+
+  return tags;
+}
+
+async function uploadSeriesToDatadog (series) {
+  await fetch('https://api.datadoghq.com/api/v2/series', {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+      'DD-API-KEY': process.env.DD_API_KEY
+    },
+    body: JSON.stringify({ series })
+  });
+}
+
+async function uploadCacheHitRateStats (hitRate, stats) {
+  const timestamp = Math.round(new Date().getTime() / 1000);
+  const tags = getCommonTags();
+
+  const series = [
+    {
+      metric: 'electron.build.effective-cache-hit-rate',
+      points: [{ timestamp, value: (hitRate * 100).toFixed(2) }],
+      type: 3, // GAUGE
+      unit: 'percent',
+      tags
+    }
+  ];
+
+  // Add all raw stats as individual metrics
+  for (const [key, value] of Object.entries(stats)) {
+    series.push({
+      metric: `electron.build.stats.${key.toLowerCase()}`,
+      points: [{ timestamp, value }],
+      type: 1, // COUNT
+      tags
+    });
+  }
+
+  await uploadSeriesToDatadog(series);
+}
+
+async function uploadObjectChangeStats (stats) {
+  const timestamp = Math.round(new Date().getTime() / 1000);
+  const tags = getCommonTags();
+
+  if (stats['previous-chromium-version']) tags.push(`previous-chromium-version:${stats['previous-chromium-version']}`);
+  if (stats['chromium-version']) tags.push(`chromium-version:${stats['chromium-version']}`);
+
+  if (stats['previous-chromium-version'] && stats['chromium-version']) {
+    tags.push(`chromium-version-changed:${stats['previous-chromium-version'] !== stats['chromium-version']}`);
+  }
+
+  const series = [
+    {
+      metric: 'electron.build.object-change-rate',
+      points: [{ timestamp, value: (stats['change-rate'] * 100).toFixed(2) }],
+      type: 3, // GAUGE
+      unit: 'percent',
+      tags
+    },
+    {
+      metric: 'electron.build.object-change-size',
+      points: [{ timestamp, value: stats['change-size'] }],
+      type: 1, // COUNT
+      unit: 'byte',
+      tags
+    },
+    {
+      metric: 'electron.build.new-object-count',
+      points: [{ timestamp, value: stats['new-object-count'] }],
+      type: 1, // COUNT
+      unit: 'count',
+      tags
+    }
+  ];
+
+  await uploadSeriesToDatadog(series);
+}
+
 async function main () {
-  const { positionals: [filename], values: { 'upload-stats': uploadStats } } = parseArgs({
+  const { positionals: [filename], values } = parseArgs({
     allowPositionals: true,
     options: {
       'upload-stats': {
         type: 'boolean',
         default: false
+      },
+      'out-dir': {
+        type: 'string'
+      },
+      'input-object-checksums': {
+        type: 'string'
+      },
+      'output-object-checksums': {
+        type: 'string'
       }
     }
   });
 
+  const {
+    'upload-stats': uploadStats,
+    'out-dir': outDir,
+    'input-object-checksums': inputObjectChecksums,
+    'output-object-checksums': outputObjectChecksums
+  } = values;
+
   if (!filename) {
     throw new Error('filename is required (should be a siso.INFO file)');
   }
 
+  if ((inputObjectChecksums || outputObjectChecksums) && !outDir) {
+    throw new Error('--out-dir is required when using --input-object-checksums or --output-object-checksums');
+  } else if (outDir && (!inputObjectChecksums && !outputObjectChecksums)) {
+    throw new Error('--out-dir only makes sense with --input-object-checksums or --output-object-checksums');
+  }
+
   const log = await fs.readFile(filename, 'utf-8');
 
   // We expect to find a line which looks like stats=build.Stats{..., CacheHit:39008, Local:4778, Remote:0, LocalFallback:0, ...}
@@ -33,55 +154,83 @@ async function main () {
   const hitRate = stats.CacheHit / (stats.Remote + stats.CacheHit + stats.LocalFallback);
 
   const messagePrefix = process.env.GITHUB_ACTIONS ? '::notice title=Build Stats::' : '';
+
   console.log(`${messagePrefix}Effective cache hit rate: ${(hitRate * 100).toFixed(2)}%`);
 
-  if (uploadStats) {
-    if (!process.env.DD_API_KEY) {
-      throw new Error('DD_API_KEY is not set');
-    }
+  const objectChangeStats = {};
 
-    const timestamp = Math.round(new Date().getTime() / 1000);
+  if (inputObjectChecksums || outputObjectChecksums) {
+    const depsContent = await fs.readFile(resolve(ELECTRON_DIR, 'DEPS'), 'utf8');
+    const currentVersion = getChromiumVersionFromDEPS(depsContent);
+
+    // Calculate the SHA256 for each object file under `outDir`
+    const objectFiles = await fs.readdir(outDir, { encoding: 'utf8', recursive: true });
+    const checksums = {};
+    for (const file of objectFiles.filter(f => f.endsWith('.o'))) {
+      const content = await fs.readFile(resolve(outDir, file));
+      checksums[file] = createHash('sha256').update(content).digest('hex');
+    }
 
-    const tags = [];
+    if (outputObjectChecksums) {
+      const outputData = {
+        chromiumVersion: currentVersion,
+        checksums
+      };
 
-    if (process.env.TARGET_ARCH) tags.push(`target-arch:${process.env.TARGET_ARCH}`);
-    if (process.env.TARGET_PLATFORM) tags.push(`target-platform:${process.env.TARGET_PLATFORM}`);
-    if (process.env.GITHUB_HEAD_REF) {
-      // Will be set in pull requests
-      tags.push(`branch:${process.env.GITHUB_HEAD_REF}`);
-    } else if (process.env.GITHUB_REF_NAME) {
-      // Will be set for release branches
-      tags.push(`branch:${process.env.GITHUB_REF_NAME}`);
+      await fs.writeFile(outputObjectChecksums, JSON.stringify(outputData, null, 2));
     }
 
-    const series = [
-      {
-        metric: 'electron.build.effective-cache-hit-rate',
-        points: [{ timestamp, value: (hitRate * 100).toFixed(2) }],
-        type: 3, // GAUGE
-        unit: 'percent',
-        tags
+    if (inputObjectChecksums) {
+      const inputData = JSON.parse(await fs.readFile(inputObjectChecksums, 'utf8'));
+      const inputFiles = Object.keys(inputData.checksums);
+      let changedCount = 0;
+      let newObjectCount = 0;
+      let changedSize = 0;
+
+      // Count changed files (only those present in both input and current)
+      for (const file of inputFiles) {
+        if (!(file in checksums)) continue; // Skip deleted files
+        if (inputData.checksums[file] !== checksums[file]) {
+          changedCount++;
+          const stat = await fs.stat(resolve(outDir, file));
+          changedSize += stat.size;
+        }
+      }
+
+      // Count new files (in current but not in input)
+      for (const file of Object.keys(checksums)) {
+        if (!(file in inputData.checksums)) {
+          newObjectCount++;
+          const stat = await fs.stat(resolve(outDir, file));
+          changedSize += stat.size;
+        }
       }
-    ];
-
-    // Add all raw stats as individual metrics
-    for (const [key, value] of Object.entries(stats)) {
-      series.push({
-        metric: `electron.build.stats.${key.toLowerCase()}`,
-        points: [{ timestamp, value }],
-        type: 1, // COUNT
-        tags
-      });
+
+      const changeRate = inputFiles.length > 0 ? changedCount / inputFiles.length : 0;
+      console.log(`${messagePrefix}Object change rate: ${(changeRate * 100).toFixed(2)}%`);
+      if (newObjectCount > 0) {
+        console.log(`${messagePrefix}New object count: ${newObjectCount}`);
+      }
+      console.log(`${messagePrefix}Cumulative changed object sizes: ${changedSize.toLocaleString()} bytes`);
+
+      objectChangeStats['change-rate'] = changeRate;
+      objectChangeStats['change-size'] = changedSize;
+      objectChangeStats['new-object-count'] = newObjectCount;
+      objectChangeStats['previous-chromium-version'] = inputData.chromiumVersion;
+      objectChangeStats['chromium-version'] = currentVersion;
     }
+  }
 
-    await fetch('https://api.datadoghq.com/api/v2/series', {
-      method: 'POST',
-      headers: {
-        'Content-Type': 'application/json',
-        'DD-API-KEY': process.env.DD_API_KEY
-      },
-      body: JSON.stringify({ series })
-    });
+  if (uploadStats) {
+    if (!process.env.DD_API_KEY) {
+      throw new Error('DD_API_KEY is not set');
+    }
+
+    await uploadCacheHitRateStats(hitRate, stats);
+
+    if (Object.keys(objectChangeStats).length > 0) {
+      await uploadObjectChangeStats(objectChangeStats);
+    }
   }
 }
 

filename: script/lib/utils.js
patch: @@ -8,6 +8,8 @@ const path = require('node:path');
 const ELECTRON_DIR = path.resolve(__dirname, '..', '..');
 const SRC_DIR = path.resolve(ELECTRON_DIR, '..');
 
+const CHROMIUM_VERSION_DEPS_REGEX = /chromium_version':\n +'(.+?)',/m;
+
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
 const pass = chalk.green('✓');
 const fail = chalk.red('✗');
@@ -162,10 +164,15 @@ function compareVersions (v1, v2) {
   return 0;
 }
 
+function getChromiumVersionFromDEPS (depsContent) {
+  return CHROMIUM_VERSION_DEPS_REGEX.exec(depsContent)?.[1] ?? null;
+}
+
 module.exports = {
   chunkFilenames,
   compareVersions,
   findMatchingFiles,
+  getChromiumVersionFromDEPS,
   getCurrentBranch,
   getElectronExec,
   getOutDir,

filename: script/lint-roller-chromium-changes.mjs
patch: @@ -5,12 +5,11 @@ import * as fs from 'node:fs/promises';
 import { dirname, resolve } from 'node:path';
 import { fileURLToPath } from 'node:url';
 
-import { compareVersions } from './lib/utils.js';
+import { compareVersions, getChromiumVersionFromDEPS } from './lib/utils.js';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const ELECTRON_DIR = resolve(__dirname, '..');
 
-const DEPS_REGEX = /chromium_version':\n +'(.+?)',/m;
 const CL_REGEX = /https:\/\/chromium-review\.googlesource\.com\/c\/(?:chromium\/src|v8\/v8)\/\+\/(\d+)(#\S+)?/g;
 const ROLLER_BRANCH_PATTERN = /^roller\/chromium\/(.+)$/;
 
@@ -140,12 +139,12 @@ async function main () {
       cwd: ELECTRON_DIR,
       encoding: 'utf8'
     });
-    baseVersion = DEPS_REGEX.exec(baseDepsContent)?.[1] ?? null;
+    baseVersion = getChromiumVersionFromDEPS(baseDepsContent);
   } catch {
     // baseVersion remains null
   }
   const depsContent = await fs.readFile(resolve(ELECTRON_DIR, 'DEPS'), 'utf8');
-  const newVersion = DEPS_REGEX.exec(depsContent)?.[1] ?? null;
+  const newVersion = getChromiumVersionFromDEPS(depsContent);
 
   if (!baseVersion || !newVersion) {
     console.error('Could not determine Chromium version range');