Repository: elastic/golang-crossbuild
Pull request #612: bk: use monorepo-diff
Author: login: v1v
Labels: name: backport-skip
Created: 2025-05-11T08:42:09Z | Merged: 2025-05-20T07:25:10Z

### What

* Use monorepo-diff.
* Use bk pipeline upload instead of triggering to help with a centralised GH status comment.
* Fix apple pipeline to use the Google auth.
* Use [log groups](https://buildkite.com/docs/pipelines/configure/managing-log-output#collapsing-output)

### Why

I don't know the reason for not using the `monorepo-diff`, but as far as I see:

1) BK pipelines for LLM or apple are not running in `main` <--- I'm not sure whether we want this or not.
2) BK pipelines for LLM or apple are not running on PRs when changing some of the changeset files:
  - https://github.com/elastic/golang-crossbuild/pull/610 <-- changed the bk pipeline definitions.


This PR should help apply what we applied in Elastic Agent a few months ago to use the bk-agent upload and the monorepo diff.


### Questions

- does monorepo-diff work for diff in main instead of PRs?



Diff:
diff --git a/.buildkite/fpm-pipeline.yml b/.buildkite/fpm-pipeline.yml
index 9e622f5f..b1439d38 100644
--- a/.buildkite/fpm-pipeline.yml
+++ b/.buildkite/fpm-pipeline.yml
@@ -6,16 +6,14 @@ env:
   DOCKER_REGISTRY: "docker.elastic.co"
   STAGING_IMAGE: "${DOCKER_REGISTRY}/observability-ci"
   MAKEFILE: "fpm"
-  CHANGESET_FILE: ".buildkite/scripts/changeset/fpm"
-  DOCKER_FILTER_REF: "docker.elastic.co/beats-dev"
   BUILDX: "0"
 
 steps:
   - label: ":linux: multiarch Linux x86_64/arm64 FPM docker image"
-    key: "build-and-publish-ubuntu-x86"
+    key: "build-and-publish-ubuntu-x86-fpm"
     if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.source == "ui" || build.branch == "main" || build.branch =~ /^[0-9]+\.[0-9]+$$/
     command:
-      - ".buildkite/scripts/llvm-fpm/build_and_publish.sh ${MAKEFILE} ${CHANGESET_FILE} ${DOCKER_FILTER_REF}"
+      - ".buildkite/scripts/llvm-fpm/build_and_publish.sh ${MAKEFILE}"
     notify:
       - github_commit_status:
           context: "Build FPM / Ubuntu X86_64"
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index c5a98fc6..4b2b44fb 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -10,7 +10,7 @@ PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/observability-ingest/cloud/gcp"
 # Secrets must be redacted
 # https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "golang-crossbuild" && ( "$BUILDKITE_STEP_KEY" == build* || "$BUILDKITE_STEP_KEY" == release* ) ]]; then
+if [[ ("$BUILDKITE_PIPELINE_SLUG" == "golang-crossbuild" || "$BUILDKITE_PIPELINE_SLUG" == "llvm-apple") && ( "$BUILDKITE_STEP_KEY" == build* || "$BUILDKITE_STEP_KEY" == release* ) ]]; then
     export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field=data -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
 fi
 
@@ -21,8 +21,7 @@ if [[ ("$BUILDKITE_PIPELINE_SLUG" == "golang-crossbuild" || "$BUILDKITE_PIPELINE
 fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "golang-crossbuild" && "$BUILDKITE_STEP_KEY" == "release-post" ]]; then
-    GITHUB_USERNAME_SECRET="elasticmachine"
-    export GITHUB_USERNAME_SECRET=$GITHUB_USERNAME_SECRET
-    export GITHUB_EMAIL_SECRET="elasticmachine@elastic.co"
+    export GITHUB_USERNAME="elasticmachine"
+    export GITHUB_EMAIL="elasticmachine@elastic.co"
     export GITHUB_TOKEN_SECRET=$VAULT_GITHUB_TOKEN
 fi
diff --git a/.buildkite/llvm-apple-pipeline.yml b/.buildkite/llvm-apple-pipeline.yml
index ef1f5451..f25c52e6 100644
--- a/.buildkite/llvm-apple-pipeline.yml
+++ b/.buildkite/llvm-apple-pipeline.yml
@@ -7,16 +7,14 @@ env:
   DOCKER_REGISTRY: "docker.elastic.co"
   STAGING_IMAGE: "${DOCKER_REGISTRY}/observability-ci"
   MAKEFILE: "go/llvm-apple"
-  CHANGESET_FILE: ".buildkite/scripts/changeset/llvm-apple"
-  DOCKER_FILTER_REF: "*/*/golang-crossbuild:llvm-apple*"
   BUILDX: "0"
 steps:
   - label: ":linux: Build LLVM Apple / Ubuntu X86_64 - {{matrix.debianVersion}}"
-    key: "build-ubuntu-x86"
+    key: "build-ubuntu-x86-llvm-apple"
     if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.source == "ui" || build.branch == "main" || build.branch =~ /^[0-9]+\.[0-9]+$$/
     command:
-      - ".buildkite/scripts/llvm-apple/build.sh ${MAKEFILE} ${CHANGESET_FILE} ${DOCKER_FILTER_REF}"
-      - ".buildkite/scripts/llvm-apple/publish.sh ${MAKEFILE} ${CHANGESET_FILE}"
+      - ".buildkite/scripts/llvm-apple/build.sh ${MAKEFILE}"
+      - ".buildkite/scripts/llvm-apple/publish.sh ${MAKEFILE}"
     notify:
       - github_commit_status:
           context: "Build LLVM Apple / Ubuntu X86_64"
@@ -35,11 +33,11 @@ steps:
           - "12"
 
   - label: ":linux: Build LLVM Apple / Ubuntu ARM - {{matrix.debianVersion}}"
-    key: "build-ubuntu-arm"
+    key: "build-ubuntu-arm-llvm-apple"
     if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.source == "ui" || build.branch == "main" || build.branch =~ /^[0-9]+\.[0-9]+$$/
     command:
-      - ".buildkite/scripts/llvm-apple/build.sh ${MAKEFILE} ${CHANGESET_FILE} ${DOCKER_FILTER_REF}"
-      - ".buildkite/scripts/llvm-apple/publish.sh ${MAKEFILE} ${CHANGESET_FILE}"
+      - ".buildkite/scripts/llvm-apple/build.sh ${MAKEFILE}"
+      - ".buildkite/scripts/llvm-apple/publish.sh ${MAKEFILE}"
     notify:
       - github_commit_status:
           context: "Build LLVM Apple / Ubuntu ARM"
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 3b17dce9..e264e6f4 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -10,6 +10,73 @@ env:
   BUILDX: 1
 
 steps:
+
+  - group: "FPM"
+    key: "fpm"
+
+    steps:
+      - label: "Trigger fpm-pipeline"
+        if: build.pull_request.id != null
+        plugins:
+          - monorepo-diff#v1.0.1:
+              diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD"
+              interpolation: false
+              watch:
+                - path:
+                    - .buildkite/pipeline.yml
+                    - .buildkite/fpm-pipeline.yml
+                    - .buildkite/scripts/llvm-fpm
+                    - fpm/**
+                  config:
+                    label: ":pipeline: Upload FPM Pipeline"
+                    command: "buildkite-agent pipeline upload .buildkite/fpm-pipeline.yml"
+                    env:
+                      - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST}
+                      - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
+                      - GITHUB_PR_LABELS=${GITHUB_PR_LABELS}
+
+      # Trigger for the main branch only
+      - label: ":pipeline: Upload FPM Pipeline"
+        if: build.branch == 'main'
+        command: "buildkite-agent pipeline upload .buildkite/fpm-pipeline.yml"
+        env:
+          - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST}
+          - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
+          - GITHUB_PR_LABELS=${GITHUB_PR_LABELS}
+
+  - group: "llvm-apple"
+    key: "llvm-apple"
+
+    steps:
+      - label: "Trigger llvm-apple-pipeline"
+        if: build.pull_request.id != null
+        plugins:
+          - monorepo-diff#v1.0.1:
+              diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD"
+              interpolation: false
+              watch:
+                - path:
+                    - .buildkite/pipeline.yml
+                    - .buildkite/llvm-apple-pipeline.yml
+                    - .buildkite/scripts/llvm-apple
+                    - go/llvm-apple/**
+                  config:
+                    label: ":pipeline: Upload FPM Pipeline"
+                    command: "buildkite-agent pipeline upload .buildkite/llvm-apple-pipeline.yml"
+                    env:
+                      - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST}
+                      - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
+                      - GITHUB_PR_LABELS=${GITHUB_PR_LABELS}
+
+      # Trigger for the main branch only
+      - label: ":pipeline: Upload llvm-apple Pipeline"
+        if: build.branch == 'main'
+        command: "buildkite-agent pipeline upload .buildkite/llvm-apple-pipeline.yml"
+        env:
+          - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST}
+          - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
+          - GITHUB_PR_LABELS=${GITHUB_PR_LABELS}
+
   - group: "Staging"
     key: "staging"
 
diff --git a/.buildkite/scripts/build.sh b/.buildkite/scripts/build.sh
index 37592f7f..a43e2c1d 100755
--- a/.buildkite/scripts/build.sh
+++ b/.buildkite/scripts/build.sh
@@ -14,10 +14,11 @@ with_mage
 google_cloud_auth
 
 make -C go -f "${MAKEFILE}" build"${is_arm}" GS_BUCKET_PATH=ingest-buildkite-ci
-echo ":: List Docker images staging ::"
+
+echo "--- List Docker images staging"
 docker images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}" --filter=reference="${STAGING_IMAGE}/golang-crossbuild"
 
-echo ":: List Docker images production ::"
+echo "--- List Docker images production"
 docker images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}" --filter=reference="${DOCKER_REGISTRY}/beats-dev/golang-crossbuild"
 
 
diff --git a/.buildkite/scripts/changeset/fpm b/.buildkite/scripts/changeset/fpm
deleted file mode 100644
index 0e1c23e9..00000000
--- a/.buildkite/scripts/changeset/fpm
+++ /dev/null
@@ -1,4 +0,0 @@
-^\.buildkite/fpm-pipeline.yml$
-^\.buildkite/scripts/llvm-fpm
-^\.buildkite/scripts/changeset/fpm
-^fpm/
diff --git a/.buildkite/scripts/changeset/llvm-apple b/.buildkite/scripts/changeset/llvm-apple
deleted file mode 100644
index a07b09d9..00000000
--- a/.buildkite/scripts/changeset/llvm-apple
+++ /dev/null
@@ -1,4 +0,0 @@
-^\.buildkite/llvm-apple-pipeline.yml$
-^\.buildkite/scripts/llvm-apple
-^\.buildkite/scripts/changeset/llvm-fpm
-^go/llvm-apple/
diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index c35a0997..07af241a 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -8,7 +8,6 @@ BIN="${WORKSPACE}/bin"
 HW_TYPE="$(uname -m)"
 PLATFORM_TYPE="$(uname)"
 TMP_FOLDER="tmp.${REPO}"
-GOOGLE_CREDENTIALS_FILENAME="google-cloud-credentials.json"
 
 if [[ -z "${GOLANG_VERSION-""}" ]]; then
     export GOLANG_VERSION=$(cat "${WORKSPACE}/.go-version")
@@ -92,7 +91,9 @@ retry() {
 }
 
 google_cloud_auth() {
+    echo "running google_cloud_auth"
     local gsUtilLocation=$(mktemp -d -p ${BIN} -t "${TMP_FOLDER}.XXXXXXXXX")
+    GOOGLE_CREDENTIALS_FILENAME="google-cloud-credentials.json"
     local secretFileLocation=${gsUtilLocation}/${GOOGLE_CREDENTIALS_FILENAME}
     echo "${PRIVATE_CI_GCS_CREDENTIALS_SECRET}" > ${secretFileLocation}
     gcloud auth activate-service-account --key-file ${secretFileLocation} 2> /dev/null
@@ -146,15 +147,3 @@ check_is_arm() {
     is_arm=""
   fi
 }
-
-are_files_changed() {
-  changeset=$1
-
-  if git diff --name-only HEAD@{1} HEAD | grep -qE "$changeset"; then
-    return 0;
-  else
-    echo "WARN! No files changed in $changeset"
-    return 1;
-  fi
-}
-
diff --git a/.buildkite/scripts/llvm-apple/build.sh b/.buildkite/scripts/llvm-apple/build.sh
index 69b101fe..5892eb79 100755
--- a/.buildkite/scripts/llvm-apple/build.sh
+++ b/.buildkite/scripts/llvm-apple/build.sh
@@ -5,16 +5,13 @@ set -euo pipefail
 source .buildkite/scripts/common.sh
 
 makefile=${1}
-patterns=${2}
-docker_filter_ref=${3}
-
-if ! are_files_changed "$patterns" ; then
-    exit 0
-fi
 
 add_bin_path
 with_go "${GOLANG_VERSION}"
 with_mage
+google_cloud_auth
 
 retry 3 make -C "${makefile}" build GS_BUCKET_PATH=ingest-buildkite-ci
-docker images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}" --filter=reference="${docker_filter_ref}"
+
+echo "--- List Docker images"
+docker images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}"
diff --git a/.buildkite/scripts/llvm-apple/publish.sh b/.buildkite/scripts/llvm-apple/publish.sh
index 371d324b..a9025b6c 100755
--- a/.buildkite/scripts/llvm-apple/publish.sh
+++ b/.buildkite/scripts/llvm-apple/publish.sh
@@ -5,11 +5,6 @@ set -euo pipefail
 source .buildkite/scripts/common.sh
 
 makefile=${1}
-patterns=${2}
-
-if ! are_files_changed "$patterns" ; then
-    exit 0
-fi
 
 add_bin_path
 retry 3 make -C ${makefile} push
diff --git a/.buildkite/scripts/llvm-fpm/build_and_publish.sh b/.buildkite/scripts/llvm-fpm/build_and_publish.sh
index 69b101fe..423ca867 100755
--- a/.buildkite/scripts/llvm-fpm/build_and_publish.sh
+++ b/.buildkite/scripts/llvm-fpm/build_and_publish.sh
@@ -5,16 +5,12 @@ set -euo pipefail
 source .buildkite/scripts/common.sh
 
 makefile=${1}
-patterns=${2}
-docker_filter_ref=${3}
-
-if ! are_files_changed "$patterns" ; then
-    exit 0
-fi
 
 add_bin_path
 with_go "${GOLANG_VERSION}"
 with_mage
 
 retry 3 make -C "${makefile}" build GS_BUCKET_PATH=ingest-buildkite-ci
-docker images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}" --filter=reference="${docker_filter_ref}"
+
+echo "--- List Docker images"
+docker images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}"
diff --git a/.buildkite/scripts/post-release.sh b/.buildkite/scripts/post-release.sh
index 42ea2b84..129d38a4 100755
--- a/.buildkite/scripts/post-release.sh
+++ b/.buildkite/scripts/post-release.sh
@@ -8,8 +8,8 @@ TAG="v$1"
 TAG_EXISTS=$(tag_Exists ${TAG})
 
 set_git_config() {
-    git config user.name "${GITHUB_USERNAME_SECRET}"
-    git config user.email "${GITHUB_EMAIL_SECRET}"
+    git config user.name "${GITHUB_USERNAME}"
+    git config user.email "${GITHUB_EMAIL}"
 }
 
 tag_commit() {
@@ -19,7 +19,7 @@ tag_commit() {
 
 git_push_with_auth() {
   echo "Pushing tag ${TAG}"
-  retry 3 git push https://${GITHUB_USERNAME_SECRET}:${GITHUB_TOKEN_SECRET}@github.com/elastic/golang-crossbuild.git ${TAG}
+  retry 3 git push https://${GITHUB_USERNAME}:${GITHUB_TOKEN_SECRET}@github.com/elastic/golang-crossbuild.git ${TAG}
 }
 
 if [[ "${TAG_EXISTS}" == true ]]; then
diff --git a/catalog-info.yaml b/catalog-info.yaml
index 4a5657e4..5f3ddeb2 100644
--- a/catalog-info.yaml
+++ b/catalog-info.yaml
@@ -137,7 +137,7 @@ spec:
     spec:
       branch_configuration: "main 1.*" # temporarily disable to build PRs from forks
       pipeline_file: ".buildkite/fpm-pipeline.yml"
-      maximum_timeout_in_minutes: 120
+      maximum_timeout_in_minutes: 360 # cmake is taking at least 4h to run
       provider_settings:
         build_tags: true
         publish_commit_status: true


Reviews:
user: login: v1v
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-13T00:33:19Z
user: login: v1v
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-13T00:34:03Z
user: login: v1v
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-13T00:34:26Z
user: login: v1v
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-13T00:34:31Z
user: login: v1v
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-13T00:35:10Z
user: login: v1v
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-13T00:35:32Z
user: login: v1v
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-13T00:36:59Z
user: login: v1v
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-13T00:42:10Z
user: login: v1v
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-13T00:42:15Z
user: login: dliappis
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-13T09:41:36Z
user: login: v1v
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-19T14:23:20Z
user: login: v1v
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-19T14:23:46Z
user: login: oakrizan
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-05-19T14:40:26Z
user: login: dliappis
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-19T14:53:24Z
user: login: dliappis
body: LGTM
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-05-20T06:32:28Z

Review comments:
diff_hunk: @@ -6,16 +6,14 @@ env:
   DOCKER_REGISTRY: "docker.elastic.co"
   STAGING_IMAGE: "${DOCKER_REGISTRY}/observability-ci"
   MAKEFILE: "fpm"
-  CHANGESET_FILE: ".buildkite/scripts/changeset/fpm"
-  DOCKER_FILTER_REF: "docker.elastic.co/beats-dev"
   BUILDX: "0"
 
 steps:
   - label: ":linux: multiarch Linux x86_64/arm64 FPM docker image"
-    key: "build-and-publish-ubuntu-x86"
+    key: "build-and-publish-ubuntu-x86-fpm"
path: .buildkite/fpm-pipeline.yml
user: login: v1v
body: avoid clashing with the existing keys, use the format `build...` so the `pre-command` hook keeps working.
created_at: 2025-05-13T00:33:19Z
author_association: MEMBER
diff_hunk: @@ -10,7 +10,7 @@ PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/observability-ingest/cloud/gcp"
 # Secrets must be redacted
 # https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "golang-crossbuild" && ( "$BUILDKITE_STEP_KEY" == build* || "$BUILDKITE_STEP_KEY" == release* ) ]]; then
+if [[ ("$BUILDKITE_PIPELINE_SLUG" == "golang-crossbuild" || "$BUILDKITE_PIPELINE_SLUG" == "llvm-apple") && ( "$BUILDKITE_STEP_KEY" == build* || "$BUILDKITE_STEP_KEY" == release* ) ]]; then
path: .buildkite/hooks/pre-command
user: login: v1v
body: fix the current issue when not able to run the llvm and didn't configure the access to the Google Bucket. Although this won't be needed as part of:
- https://github.com/elastic/golang-crossbuild/pull/610
created_at: 2025-05-13T00:34:03Z
author_association: MEMBER
diff_hunk: @@ -21,8 +21,7 @@ if [[ ("$BUILDKITE_PIPELINE_SLUG" == "golang-crossbuild" || "$BUILDKITE_PIPELINE
 fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "golang-crossbuild" && "$BUILDKITE_STEP_KEY" == "release-post" ]]; then
-    GITHUB_USERNAME_SECRET="elasticmachine"
-    export GITHUB_USERNAME_SECRET=$GITHUB_USERNAME_SECRET
-    export GITHUB_EMAIL_SECRET="elasticmachine@elastic.co"
+    export GITHUB_USERNAME="elasticmachine"
+    export GITHUB_EMAIL="elasticmachine@elastic.co"
path: .buildkite/hooks/pre-command
user: login: v1v
body: These are not secrets since they are in plain hence the `masked` is unrequired.
created_at: 2025-05-13T00:34:26Z
author_association: MEMBER
diff_hunk: @@ -7,16 +7,14 @@ env:
   DOCKER_REGISTRY: "docker.elastic.co"
   STAGING_IMAGE: "${DOCKER_REGISTRY}/observability-ci"
   MAKEFILE: "go/llvm-apple"
-  CHANGESET_FILE: ".buildkite/scripts/changeset/llvm-apple"
-  DOCKER_FILTER_REF: "*/*/golang-crossbuild:llvm-apple*"
   BUILDX: "0"
 steps:
   - label: ":linux: Build LLVM Apple / Ubuntu X86_64 - {{matrix.debianVersion}}"
-    key: "build-ubuntu-x86"
+    key: "build-ubuntu-x86-llvm-apple"
path: .buildkite/llvm-apple-pipeline.yml
user: login: v1v
body: avoid clashing with the existing keys, use the format `build...` so the `pre-command` hook keeps working.
created_at: 2025-05-13T00:34:31Z
author_association: MEMBER
diff_hunk: @@ -14,10 +14,11 @@ with_mage
 google_cloud_auth
 
 make -C go -f "${MAKEFILE}" build"${is_arm}" GS_BUCKET_PATH=ingest-buildkite-ci
-echo ":: List Docker images staging ::"
+
+echo "--- List Docker images staging"
path: .buildkite/scripts/build.sh
user: login: v1v
body: let's use the BK approach for collapsing logs
created_at: 2025-05-13T00:35:10Z
author_association: MEMBER
diff_hunk: @@ -92,7 +91,9 @@ retry() {
 }
 
 google_cloud_auth() {
+    echo "running google_cloud_auth"
     local gsUtilLocation=$(mktemp -d -p ${BIN} -t "${TMP_FOLDER}.XXXXXXXXX")
+    GOOGLE_CREDENTIALS_FILENAME="google-cloud-credentials.json"
path: .buildkite/scripts/common.sh
user: login: v1v
body: Use the env variable where it's actually used...
created_at: 2025-05-13T00:35:32Z
author_association: MEMBER
diff_hunk: @@ -5,16 +5,13 @@ set -euo pipefail
 source .buildkite/scripts/common.sh
 
 makefile=${1}
-patterns=${2}
-docker_filter_ref=${3}
-
-if ! are_files_changed "$patterns" ; then
-    exit 0
-fi
 
 add_bin_path
 with_go "${GOLANG_VERSION}"
 with_mage
+google_cloud_auth
path: .buildkite/scripts/llvm-apple/build.sh
user: login: v1v
body: Resolve the issue with the llvm-apple pipeline, as it needs access to the Google bucket. I'm curious if this pipeline was tested previously.
created_at: 2025-05-13T00:36:58Z
author_association: MEMBER
diff_hunk: @@ -10,6 +10,73 @@ env:
   BUILDX: 1
 
 steps:
+
+  - group: "FPM"
+    key: "fpm"
+
+    steps:
+      - label: "Trigger fpm-pipeline"
+        if: build.pull_request.id != null
+        plugins:
+          - monorepo-diff#v1.0.1:
+              diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD"
path: .buildkite/pipeline.yml
user: login: v1v
body: We should not run for PRs targetting other branches but main, any idea how to do it?
created_at: 2025-05-13T00:42:10Z
author_association: MEMBER
diff_hunk: @@ -10,6 +10,73 @@ env:
   BUILDX: 1
 
 steps:
+
+  - group: "FPM"
+    key: "fpm"
+
+    steps:
+      - label: "Trigger fpm-pipeline"
+        if: build.pull_request.id != null
+        plugins:
+          - monorepo-diff#v1.0.1:
+              diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD"
+              interpolation: false
+              watch:
+                - path:
+                    - .buildkite/pipeline.yml
+                    - .buildkite/fpm-pipeline.yml
+                    - .buildkite/scripts/llvm-fpm
+                    - fpm/**
+                  config:
+                    label: ":pipeline: Upload FPM Pipeline"
+                    command: "buildkite-agent pipeline upload .buildkite/fpm-pipeline.yml"
+                    env:
+                      - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST}
+                      - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
+                      - GITHUB_PR_LABELS=${GITHUB_PR_LABELS}
+
+      # Trigger for the main branch only
+      - label: ":pipeline: Upload FPM Pipeline"
+        if: build.branch == 'main'
+        command: "buildkite-agent pipeline upload .buildkite/fpm-pipeline.yml"
+        env:
+          - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST}
+          - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
+          - GITHUB_PR_LABELS=${GITHUB_PR_LABELS}
+
+  - group: "llvm-apple"
+    key: "llvm-apple"
+
+    steps:
+      - label: "Trigger llvm-apple-pipeline"
+        if: build.pull_request.id != null
path: .buildkite/pipeline.yml
user: login: v1v
body: We should not run for PRs targetting other branches but main, any idea how to do it?
created_at: 2025-05-13T00:42:15Z
author_association: MEMBER
diff_hunk: @@ -10,6 +10,73 @@ env:
   BUILDX: 1
 
 steps:
+
+  - group: "FPM"
+    key: "fpm"
+
+    steps:
+      - label: "Trigger fpm-pipeline"
+        if: build.pull_request.id != null
+        plugins:
+          - monorepo-diff#v1.0.1:
+              diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD"
path: .buildkite/pipeline.yml
user: login: dliappis
body: If I understand correctly the question, couldn't we could simply add `&& build.branch == "main"` in the conditional in line 19?
created_at: 2025-05-13T09:40:47Z
author_association: CONTRIBUTOR
diff_hunk: @@ -10,6 +10,73 @@ env:
   BUILDX: 1
 
 steps:
+
+  - group: "FPM"
+    key: "fpm"
+
+    steps:
+      - label: "Trigger fpm-pipeline"
+        if: build.pull_request.id != null
+        plugins:
+          - monorepo-diff#v1.0.1:
+              diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD"
+              interpolation: false
+              watch:
+                - path:
+                    - .buildkite/pipeline.yml
+                    - .buildkite/fpm-pipeline.yml
+                    - .buildkite/scripts/llvm-fpm
+                    - fpm/**
+                  config:
+                    label: ":pipeline: Upload FPM Pipeline"
+                    command: "buildkite-agent pipeline upload .buildkite/fpm-pipeline.yml"
+                    env:
+                      - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST}
+                      - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
+                      - GITHUB_PR_LABELS=${GITHUB_PR_LABELS}
+
+      # Trigger for the main branch only
+      - label: ":pipeline: Upload FPM Pipeline"
+        if: build.branch == 'main'
+        command: "buildkite-agent pipeline upload .buildkite/fpm-pipeline.yml"
+        env:
+          - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST}
+          - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
+          - GITHUB_PR_LABELS=${GITHUB_PR_LABELS}
+
+  - group: "llvm-apple"
+    key: "llvm-apple"
+
+    steps:
+      - label: "Trigger llvm-apple-pipeline"
+        if: build.pull_request.id != null
path: .buildkite/pipeline.yml
user: login: dliappis
body: see above reply
created_at: 2025-05-13T09:41:05Z
author_association: CONTRIBUTOR
diff_hunk: @@ -10,6 +10,73 @@ env:
   BUILDX: 1
 
 steps:
+
+  - group: "FPM"
+    key: "fpm"
+
+    steps:
+      - label: "Trigger fpm-pipeline"
+        if: build.pull_request.id != null
+        plugins:
+          - monorepo-diff#v1.0.1:
+              diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD"
path: .buildkite/pipeline.yml
user: login: v1v
body: I don't know, `GITHUB_PR_TARGET_BRANCH` might be empty when `build.branch == main`

My question was, use monorepo diff for changees on PRs and main
created_at: 2025-05-19T14:23:20Z
author_association: MEMBER
diff_hunk: @@ -10,6 +10,73 @@ env:
   BUILDX: 1
 
 steps:
+
+  - group: "FPM"
+    key: "fpm"
+
+    steps:
+      - label: "Trigger fpm-pipeline"
+        if: build.pull_request.id != null
+        plugins:
+          - monorepo-diff#v1.0.1:
+              diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD"
path: .buildkite/pipeline.yml
user: login: v1v
body: `build.branch != main` for PRs
created_at: 2025-05-19T14:23:46Z
author_association: MEMBER
diff_hunk: @@ -10,6 +10,73 @@ env:
   BUILDX: 1
 
 steps:
+
+  - group: "FPM"
+    key: "fpm"
+
+    steps:
+      - label: "Trigger fpm-pipeline"
+        if: build.pull_request.id != null
+        plugins:
+          - monorepo-diff#v1.0.1:
+              diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD"
path: .buildkite/pipeline.yml
user: login: dliappis
body: > My question was, use monorepo diff for changees on PRs and main

I see, sorry I didn't understand your question correctly.

I don't think the monorepo-diff plugin supports conditional diff targets dynamically inside its own configuration (maybe someone else knows how do it off the top of their heads).

I think it's possible to collapse the logic for a PR and a non-PR in the above monorepo statement and override the diff command, by adding a `command: ` (either a script or inline bash) but it essentially bypasses the monorepo diff command:

```
  - command: |
      TARGET_BRANCH="${GITHUB_PR_TARGET_BRANCH:-main}"
      if [[ -n "$BUILDKITE_PULL_REQUEST" && "$BUILDKITE_PULL_REQUEST" != "false" ]]; then
        git diff --name-only origin/${TARGET_BRANCH}...HEAD
      else
        git diff --name-only origin/main...HEAD
      fi
    plugins:
      - monorepo-diff#v1.0.1:
          diff: "git diff --name-only origin/main...HEAD" # dummy, overridden by command
```

So some duplication might be tolerated and advisable for the non PR case.

created_at: 2025-05-19T14:53:24Z
author_association: CONTRIBUTOR

PR comments:
user: login: mergify[bot]
created_at: 2025-05-11T08:42:47Z
body: This pull request does not have a backport label. Could you fix it @v1v? 🙏
To fixup this pull request, you need to add the backport labels for the needed
branches, such as:
* `backport-v./d./d` is the label to automatically backport to the `1./d` branch. `/d` is the digit
**NOTE**: `backport-skip` has been added to this pull request.
author_association: NONE
user: login: v1v
created_at: 2025-05-11T08:59:12Z
body: There are some interesting errors:


- [multiarchcontext](https://buildkite.com/elastic/golang-crossbuild/builds/1075#0196be85-8bf1-4521-a7d1-a676fc389073/73-1801):
```
2025-05-11 08:53:42 UTC | multiarchcontext*       docker-container
-- | --
  | 2025-05-11 08:53:42 UTC | \_ multiarchcontext0    \_ unix:///var/run/docker.sock   running   v0.21.1                linux/amd64 (+3), linux/arm64, linux/arm (+2), linux/ppc64le, (6 more)
  | 2025-05-11 08:53:42 UTC | default                 docker
  | 2025-05-11 08:53:42 UTC | \_ default              \_ default                       running   v0.11.7+d3e6c1360f6e   linux/amd64 (+3), linux/arm64, linux/arm (+2), linux/ppc64le, (5 more)
  | 2025-05-11 08:53:42 UTC | Create builder
  | 2025-05-11 08:53:42 UTC | ERROR: existing instance for "multiarchcontext" but no append mode, specify the node name to make changes for existing instances
  | 2025-05-11 08:53:42 UTC | make: *** [Makefile:12: build] Error 1
  | 2025-05-11 08:53:42 UTC | make: Leaving directory '/opt/buildkite-agent/builds/bk-agent-prod-gcp-1746953151034029483/elastic/golang-crossbuild/fpm'
  | 2025-05-11 08:53:42 UTC | Retry 2/3 exited 2, retrying in 2 seconds...
  | 2025-05-11 08:53:44 UTC | make: Entering directory '/opt/buildkite-agent/builds/bk-agent-prod-gcp-1746953151034029483/elastic/golang-crossbuild/fpm'
  | 2025-05-11 08:53:44 UTC | >> Building docker.elastic.co/observability-ci/fpm:1.13.1

```

- [storage.objects.get access](https://buildkite.com/elastic/golang-crossbuild/builds/1075#0196be85-8fbd-434f-8a8c-c1fd33631c56/73-110)
```
2025-05-11 08:46:54 UTC | Completed files 0 \| 0B
-- | --
  | 2025-05-11 08:46:54 UTC | ERROR: (gcloud.storage.cp) HTTPError 403: vault-gcp-authentication@elastic-ci-prod.iam.gserviceaccount.com does not have storage.objects.get access to the Google Cloud Storage object. Permission 'storage.objects.get' denied on resource (or it may not exist). This command is authenticated as vault-gcp-authentication@elastic-ci-prod.iam.gserviceaccount.com which is the active account specified by the [core/account] property.
  | 2025-05-11 08:46:54 UTC | make: *** [..//../Makefile.common:23: copy-sdks] Error 1
  | 2025-05-11 08:46:54 UTC | make: Leaving directory '/opt/buildkite-agent/builds/bk-agent-prod-gcp-1746953121187552945/elastic/llvm-apple/go/llvm-apple'
  | 2025-05-11 08:46:54 UTC | Retry 2/3 exited 2, retrying in 2 seconds...
  | 2025-05-11 08:46:56 UTC | make: Entering directory '/opt/buildkite-agent/builds/bk-agent-prod-gcp-1746953121187552945/elastic/llvm-apple/go/llvm-apple'
  | 2025-05-11 08:46:56 UTC | Makefile:16: warning: overriding recipe for target 'copy-npcap'
  | 2025-05-11 08:46:56 UTC | ..//../Makefile.common:15: warning: ignoring old recipe for target 'copy-npcap'
```

@oakrizan, I've been searching for builds related to the LLM and Apple BK pipelines:
- https://buildkite.com/elastic/fpm/builds?branch=main
- https://buildkite.com/elastic/llvm-apple/builds?branch=main

So far, I haven't come across any builds. Since you have experience with integrating them as part of https://github.com/elastic/golang-crossbuild/pull/330, could you help me understand whether those pipelines ran? I’d like to compare the current failures with the results from when they were executed.

Thanks
author_association: MEMBER
user: login: oakrizan
created_at: 2025-05-12T09:04:55Z
body: > So far, I haven't come across any builds. Since you have experience with integrating them as part of https://github.com/elastic/golang-crossbuild/pull/330, could you help me understand whether those pipelines ran? I’d like to compare the current failures with the results from when they were executed.

BK stores the builds for 1 year max, as far as I understood, so none the builds executed on Nov 2023 would be available for comparison.

Btw, BK `multiarch` suceeded in my dummy PR: https://buildkite.com/elastic/fpm/builds/985#0196c4a1-c281-43fa-9108-6d1dab679bdd
author_association: CONTRIBUTOR
user: login: v1v
created_at: 2025-05-13T00:05:00Z
body: > Btw, BK multiarch suceeded in my dummy PR: [buildkite.com/elastic/fpm/builds/985#0196c4a1-c281-43fa-9108-6d1dab679bdd](https://buildkite.com/elastic/fpm/builds/985#0196c4a1-c281-43fa-9108-6d1dab679bdd)

I've just foudn there was another error earlier:

```
ERROR: failed to solve: failed to push docker.elastic.co/observability-ci/fpm:1.13.1: push access denied, repository does not exist or may require authorization: server message: insufficient_scope: authorization failed
```


[`21d4e3d` (#612)](https://github.com/elastic/golang-crossbuild/pull/612/commits/21d4e3db3c8cbcfa8c2ebc6163ad82f894c3a749) is the one that caused the `pre-command` hook to don't configure the correct environment.

Thanks for double checking it
author_association: MEMBER
user: login: elasticmachine
created_at: 2025-05-13T07:02:09Z
body: ## :broken_heart: Build Failed
* [Buildkite Build](https://buildkite.com/elastic/golang-crossbuild/builds/1090)
* Commit: b6d4451d662c9f5f0453b790ba42fdddc6c4bffb

### Failed CI Steps
* [:linux: Build LLVM Apple / Ubuntu X86_64 - 10](https://buildkite.com/elastic/golang-crossbuild/builds/1090#0196c803-8a7e-4aec-9ab9-615db89d5e39)
* [:linux: Build LLVM Apple / Ubuntu X86_64 - 11](https://buildkite.com/elastic/golang-crossbuild/builds/1090#0196c803-8a7e-4622-9b9b-e06aa7af89d9)
* [:linux: Build LLVM Apple / Ubuntu X86_64 - 12](https://buildkite.com/elastic/golang-crossbuild/builds/1090#0196c803-8a7e-4a58-82bd-ba248144525c)
* [:linux: Build LLVM Apple / Ubuntu ARM - 10](https://buildkite.com/elastic/golang-crossbuild/builds/1090#0196c803-8a80-4bef-a094-6bd1ee68756f)
* [:linux: Build LLVM Apple / Ubuntu ARM - 11](https://buildkite.com/elastic/golang-crossbuild/builds/1090#0196c803-8a80-4f3f-b81e-f415f60da50f)
* [:linux: Build LLVM Apple / Ubuntu ARM - 12](https://buildkite.com/elastic/golang-crossbuild/builds/1090#0196c803-8a80-47c3-a109-65466fe10662)

### History
* :broken_heart: [Build #1089](https://buildkite.com/elastic/golang-crossbuild/builds/1089) failed 47a6fe27336ce336a9259d619f7b1f874b808fdf
* :broken_heart: [Build #1086](https://buildkite.com/elastic/golang-crossbuild/builds/1086) failed 166ecdee84af966d5a9fee974a21ad2782067f3a
* :broken_heart: [Build #1084](https://buildkite.com/elastic/golang-crossbuild/builds/1084) failed fe1cc8326b98c4b57b666fab4ed2de4cc4211ded
* :broken_heart: [Build #1075](https://buildkite.com/elastic/golang-crossbuild/builds/1075) failed 21d4e3db3c8cbcfa8c2ebc6163ad82f894c3a749

cc @v1v

<!--buildkite-pr-comment-golang-crossbuild
{"builds":[{"buildStatus":{"state":"failed","success":false,"hasRetries":false,"hasNonPreemptionRetries":false},"url":"https://buildkite.com/elastic/golang-crossbuild/builds/1090","number":1090,"commit":"b6d4451d662c9f5f0453b790ba42fdddc6c4bffb"},{"buildStatus":{"state":"failed","success":false,"hasRetries":false,"hasNonPreemptionRetries":false},"url":"https://buildkite.com/elastic/golang-crossbuild/builds/1089","number":1089,"commit":"47a6fe27336ce336a9259d619f7b1f874b808fdf"},{"buildStatus":{"state":"failing","success":false,"hasRetries":true,"hasNonPreemptionRetries":true},"url":"https://buildkite.com/elastic/golang-crossbuild/builds/1086","number":1086,"commit":"166ecdee84af966d5a9fee974a21ad2782067f3a"},{"buildStatus":{"state":"failing","success":false,"hasRetries":false,"hasNonPreemptionRetries":false},"url":"https://buildkite.com/elastic/golang-crossbuild/builds/1084","number":1084,"commit":"fe1cc8326b98c4b57b666fab4ed2de4cc4211ded"},{"buildStatus":{"state":"failed","success":false,"hasRetries":false,"hasNonPreemptionRetries":false},"url":"https://buildkite.com/elastic/golang-crossbuild/builds/1075","number":1075,"commit":"21d4e3db3c8cbcfa8c2ebc6163ad82f894c3a749"}],"number":1090}
buildkite-pr-comment-->
author_association: NONE
user: login: v1v
created_at: 2025-05-20T07:25:04Z
body: I'm gonna merge it, so far `buildkite/llvm-apple` has never run in Buildkite
author_association: MEMBER