Repository: bacalhau-project/bacalhau (language: python)

Issue:
Bring back lint
lint seem to have been disabled in this PR [https://github.com/bacalhau-project/bacalhau/pull/4332](https://github.com/bacalhau-project/bacalhau/pull/4332) and no longer runs as part of the pre-commit hook, which is the lint GitHub Actions runs [https://github.com/bacalhau-project/bacalhau/blob/44332a82dec19995b948a0af5f5d6310cd3ef8d0/.github/workflows/lint.yml](https://github.com/bacalhau-project/bacalhau/blob/44332a82dec19995b948a0af5f5d6310cd3ef8d0/.github/workflows/lint.yml)


Discussion / hints:
<p><a href="https://linear.app/expanso/issue/ENG-292/bring-back-lint">ENG-292 Bring back lint</a></p>

Fix patch:
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
new file mode 100644
index 0000000000..b4532b2d82
--- /dev/null
+++ b/.github/workflows/build.yml
@@ -0,0 +1,77 @@
+name: Build Bacalhau Binaries
+
+on:
+  workflow_call:
+
+jobs:
+  build:
+    name: Build Binary
+    runs-on: ubuntu-latest
+    outputs:
+      status: ${{ job.status }}
+    strategy:
+      matrix:
+        include:
+          - os: linux
+            goarch: amd64
+          - os: linux
+            goarch: arm64
+          - os: linux
+            goarch: armv7
+          - os: linux
+            goarch: armv6
+          - os: darwin
+            goarch: amd64
+          - os: darwin
+            goarch: arm64
+          - os: windows
+            goarch: amd64
+
+    steps:
+      - name: Install earthly
+        uses: earthly/actions-setup@v1
+
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: "0" # Need to fetch all due to how bacalhau constructs semver
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.work
+
+      - name: Build
+        env:
+          GOOS: ${{ matrix.os }}
+          GOARCH: ${{ matrix.goarch }}
+          PRIVATE_PEM: ${{ secrets.PRIVATE_PEM }}
+          PUBLIC_PEM: ${{ secrets.PUBLIC_PEM }}
+          PRIVATE_KEY_PASSPHRASE: ${{ secrets.PRIVATE_KEY_PASSPHRASE }}
+        run: |
+          # Add Keys to expected files
+          echo "${PRIVATE_PEM}" > /tmp/private.pem && chmod 600 /tmp/private.pem
+          echo "${PUBLIC_PEM}" > /tmp/public.pem && chmod 600 /tmp/public.pem
+          
+          # Start build
+          echo "==> Building bacalhau binary for: ${GOOS} ${GOARCH}..."
+          make build-bacalhau-tgz
+          echo "===> Done building bacalhau binary."
+          
+          # Listing Builds
+          echo "===> Built Artifacts:"
+          ls -lh dist/
+          
+          # Remove keys, good security practice
+          rm /tmp/private.pem /tmp/public.pem
+
+      - name: Upload binary artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.os }}-${{ matrix.goarch }}
+          path: "dist/bacalhau_*"
+          retention-days: 1  # Short retention since these are intermediate artifacts, also save money
+
+      - name: Report build status
+        if: always()
+        run: |
+          echo "Build completed for ${{ matrix.os }}-${{ matrix.goarch }}"
+          echo "Status: ${{ job.status }}"
\ No newline at end of file
diff --git a/.github/workflows/cspell.yml b/.github/workflows/cspell.yml
index 0a88b458fc..3358731650 100644
--- a/.github/workflows/cspell.yml
+++ b/.github/workflows/cspell.yml
@@ -2,12 +2,7 @@
 name: Codespell
 
 on:
-  push:
-    branches: [main]
-  pull_request:
-    branches: [main]
-  repository_dispatch:
-    types: [ok-to-test]
+  workflow_call:
 
 jobs:
   codespell:
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
index 4b6d22816a..b6ba969abf 100644
--- a/.github/workflows/lint.yml
+++ b/.github/workflows/lint.yml
@@ -1,13 +1,32 @@
 name: Lints
 
 on:
-  pull_request:
-  push:
-    branches: [main]
-  repository_dispatch:
-    types: [ok-to-test]
+  workflow_call:
 
 jobs:
+  # A separate Go-Lint job since the current lint job is heavily
+  # coupled with a lot of tooling, which will be removed and
+  # simplified, one step at a time.
+  # For now, we are just extracting the go-lint.
+  go-lint:
+    strategy:
+      matrix:
+        os: [ ubuntu-latest ]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.work
+      - name: Setup Dummy WebUi Build
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          mkdir -p webui/build && touch webui/build/index.html
+      - uses: golangci/golangci-lint-action@v6
+        with:
+          skip-cache: true
+
+  # TODO: Decouple Each Component Linting
   lint:
     runs-on: ubuntu-latest
     steps:
diff --git a/.github/workflows/pipeline-main.yml b/.github/workflows/pipeline-main.yml
new file mode 100644
index 0000000000..87307258d2
--- /dev/null
+++ b/.github/workflows/pipeline-main.yml
@@ -0,0 +1,31 @@
+name: Main Pipeline
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  repository_dispatch:
+    types: [ok-to-test]
+
+jobs:
+  build:
+    uses: ./.github/workflows/build.yml
+    secrets: inherit
+
+  lint:
+    uses: ./.github/workflows/lint.yml
+
+  swagger-validation:
+    uses: ./.github/workflows/swagger-validation.yml
+
+  cspell:
+    uses: ./.github/workflows/cspell.yml
+
+  testcontainers-suite:
+    needs: [build, lint, swagger-validation, cspell]
+    if: success()
+    uses: ./.github/workflows/testcontainers-suite.yml
+    with:
+      os: linux
+      arch: amd64
diff --git a/.github/workflows/swagger-validation.yml b/.github/workflows/swagger-validation.yml
index 8d70e7ea3b..92e12f31ce 100644
--- a/.github/workflows/swagger-validation.yml
+++ b/.github/workflows/swagger-validation.yml
@@ -1,9 +1,7 @@
 name: Swagger Validation
 
 on:
-  pull_request:
-    branches:
-      - main
+  workflow_call:
 
 jobs:
   ensure-up-to-date:
@@ -17,35 +15,20 @@ jobs:
         with:
           fetch-depth: 1
 
-      # Detect Go version from go.mod
-      - name: Detect Go version from go.mod
-        id: detect-go-version
-        shell: bash
-        run: |
-          set -euo pipefail
-          go_version=$(grep '^go ' go.mod | awk '{print $2}')
-          echo "Go version detected: $go_version"
-          echo "golang-version=$go_version" >> $GITHUB_ENV
-
-      # Setup Go using the detected version
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
-          go-version: ${{ env.golang-version }}
+          go-version-file: go.work
           cache: true
 
       # Install swag (Swagger generator)
       - name: Install Swag
-        shell: bash
         run: |
-          set -euo pipefail
           go install github.com/swaggo/swag/cmd/swag@latest
 
       # Verify swag installation
       - name: Verify Swag Installation
-        shell: bash
         run: |
-          set -euo pipefail
           if ! command -v swag &> /dev/null; then
             echo "Swag is not installed. Please ensure Go is properly configured and Swag is installed."
             exit 1
@@ -53,14 +36,10 @@ jobs:
 
       # Generate the swagger.json
       - name: Generate Swagger file
-        shell: bash
         run: |
-          set -euo pipefail
           make generate-swagger
 
       # Compare the newly generated swagger.json with the committed swagger.json
       - name: Check for Swagger differences
-        shell: bash
         run: |
-          set -euo pipefail
           git diff --exit-code pkg/swagger/swagger.json || (echo "Swagger is outdated. Please regenerate it with 'make generate-swagger' and commit the changes." && exit 1)
diff --git a/.github/workflows/trigger-development-deployment.yaml b/.github/workflows/trigger-development-deployment.yaml
deleted file mode 100644
index 848e249efe..0000000000
--- a/.github/workflows/trigger-development-deployment.yaml
+++ /dev/null
@@ -1,23 +0,0 @@
-name: Trigger Deployment to Development on Merge to Main
-
-on:
-  push:
-    branches:
-      - main
-
-jobs:
-  trigger-deployment:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Trigger Deployment
-        shell: bash
-        run: |
-          set -euo pipefail
-          curl -X POST \
-          -H "Authorization: token ${{ secrets.BACALHAU_VMI_DEPLOYMENT_SECRET }}" \
-          -H "Accept: application/vnd.github+json" \
-          "https://api.github.com/repos/bacalhau-project/bacalhau-vmi/actions/workflows/deploy-commit.yaml/dispatches" \
-          --data '{"ref": "main", "inputs": {"commit_reference": "${{ github.sha }}", "redeploy": "false"}}'
diff --git a/.gitprecommit/golangci-lint.sh b/.gitprecommit/golangci-lint.sh
deleted file mode 100755
index 419f0e3dfa..0000000000
--- a/.gitprecommit/golangci-lint.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/usr/bin/env bash
-
-golangci-lint run --config=.golangci.yml --timeout 10m
diff --git a/.golangci.yml b/.golangci.yml
index 641b6e1d6e..530754c783 100644
--- a/.golangci.yml
+++ b/.golangci.yml
@@ -69,7 +69,7 @@ linters:
   - depguard
   - dogsled
   - errcheck
-  - exportloopref
+  - copyloopvar
   - funlen
   - gochecknoinits
   - goconst
@@ -100,7 +100,7 @@ run:
   # concurrency: 16
   # Timeout for analysis, e.g. 30s, 5m.
   # Default: 1m
-  timeout: 5m
+  timeout: 10m
   # Exit code when at least one issue was found.
   # Default: 1
   issues-exit-code: 2
diff --git a/buildkite/scripts/lint.sh b/buildkite/scripts/lint.sh
index 67231543fd..11c40c420c 100644
--- a/buildkite/scripts/lint.sh
+++ b/buildkite/scripts/lint.sh
@@ -6,9 +6,6 @@ set -x
 # NB(forrest/udit): this step needs to be done before linting as without it the code doesn't compile since webuid/build DNE.
 make build-webui
 
-# NB(forrest/udit): linting cannot be done by pre-commit because it doesn't work...
-make lint
-
 # TODO(forrest/udit): deprecate pre-commit and replace it with the individual steps required to validate the code.
 # e.g. modtidy check, credentials check, go fmt, test file header validation.
 pre-commit run --show-diff-on-failure --color=always --all-files
diff --git a/cmd/util/flags/types.go b/cmd/util/flags/types.go
index 8984b86b66..7f64ac5b4f 100644
--- a/cmd/util/flags/types.go
+++ b/cmd/util/flags/types.go
@@ -96,7 +96,6 @@ func (s *ArrayValueFlag[T]) Set(input string) error {
 func (s *ArrayValueFlag[T]) String() string {
 	strs := make([]string, 0, len(*s.value))
 	for _, spec := range *s.value {
-		spec := spec
 		strs = append(strs, s.stringer(&spec))
 	}
 	return strings.Join(strs, ", ")
@@ -152,7 +151,6 @@ func (s *MapValueFlag[K, V]) Set(input string) error {
 func (s *MapValueFlag[K, V]) String() string {
 	strs := make([]string, len(*s.value))
 	for key, value := range *s.value {
-		key, value := key, value
 		strs = append(strs, s.stringer(&key, &value))
 	}
 	return strings.Join(strs, ", ")
diff --git a/pkg/executor/results.go b/pkg/executor/results.go
index 981407c503..53c2433c3d 100644
--- a/pkg/executor/results.go
+++ b/pkg/executor/results.go
@@ -139,7 +139,6 @@ func WriteJobResults(
 
 	wg := multierrgroup.Group{}
 	for _, output := range outputs {
-		output := output
 		wg.Go(func() error {
 			return writeOutputResult(resultsDir, output)
 		})


Test patch:
diff --git a/.github/workflows/testcontainers-integration-tests.yml b/.github/workflows/testcontainers-integration-tests.yml
deleted file mode 100644
index 5a28cdff1a..0000000000
--- a/.github/workflows/testcontainers-integration-tests.yml
+++ /dev/null
@@ -1,31 +0,0 @@
-name: Test Containers Integration Tests
-
-on:
-  push:
-    branches: [main]
-  pull_request:
-    branches: [main]
-  repository_dispatch:
-    types: [ok-to-test]
-
-jobs:
-  test-build-and-run-testcontainers:
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Install Golang
-        uses: actions/setup-go@v5
-        with:
-          go-version-file: go.work
-
-      - name: Run Go tests in integration test directory
-        shell: bash
-        run: |
-          set -euo pipefail
-          # TODO: make WebUI happy when compiling
-          mkdir -p webui/build
-          echo "hello test" > webui/build/index.html
-          cd test_integration
-          go test -v -count=1 ./...
diff --git a/.github/workflows/testcontainers-suite.yml b/.github/workflows/testcontainers-suite.yml
new file mode 100644
index 0000000000..7e1a621d5a
--- /dev/null
+++ b/.github/workflows/testcontainers-suite.yml
@@ -0,0 +1,41 @@
+name: TestContainers Suite
+
+on:
+  workflow_call:
+    inputs:
+      os:
+        required: true
+        type: string
+      arch:
+        required: true
+        type: string
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Download compiled binary
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ inputs.os }}-${{ inputs.arch }}
+          path: compiled-artifacts/
+
+      - name: Install Golang
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.work
+
+      - name: Run Go tests in integration test directory
+        shell: bash
+        run: |
+          echo "===> Copy compiled artifact to use in docker images..."
+          cd compiled-artifacts
+          tar -xzf bacalhau_*.tar.gz
+          cp ./bacalhau ../test_integration/common_assets/bacalhau_bin
+          
+          echo "===> Running tests..."       
+          cd ../test_integration
+          go test -v -count=1 ./...
diff --git a/pkg/test/scenario/results.go b/pkg/test/scenario/results.go
index 8f466cde39..ea69d9c12d 100644
--- a/pkg/test/scenario/results.go
+++ b/pkg/test/scenario/results.go
@@ -132,7 +132,6 @@ func ManyChecks(checks ...CheckResults) CheckResults {
 	return func(resultsDir string) error {
 		var wg multierrgroup.Group
 		for _, check := range checks {
-			check := check
 			wg.Go(func() error { return check(resultsDir) })
 		}
 		return wg.Wait()
diff --git a/test_integration/README.md b/test_integration/README.md
index 35bcc5c076..19779430e1 100644
--- a/test_integration/README.md
+++ b/test_integration/README.md
@@ -38,5 +38,5 @@ Then:
 
 ```shell
 cd test_integration
-go test -v -count=1 ./...
+./run_locally.sh
 ```
diff --git a/test_integration/init_test.go b/test_integration/init_test.go
index 22f6d92840..2364b27df9 100644
--- a/test_integration/init_test.go
+++ b/test_integration/init_test.go
@@ -2,7 +2,6 @@ package test_integration
 
 import (
 	"bacalhau/integration_tests/utils"
-	"context"
 	"github.com/google/uuid"
 	"log"
 	"os"
@@ -22,15 +21,6 @@ func TestMain(m *testing.M) {
 		os.Exit(1)
 	}
 
-	ctx := context.Background()
-	err = utils.CompileBacalhau(ctx, "../main.go")
-	if err != nil {
-		log.Println("Error compiling the bacalhau binary: ", err.Error())
-		os.Exit(1)
-	}
-
-	// TODO: Maybe we do not need to created images, but just inject
-	// TODO: them with artifacts before container starts the starts (certs and binary and configs)
 	err = utils.BuildBaseImages(globalTestExecutionId)
 	if err != nil {
 		log.Println("Error building base images: ", err.Error())
@@ -45,8 +35,6 @@ func TestMain(m *testing.M) {
 		os.Exit(1)
 	}
 
-	// TODO: Better cleaning
-	os.Remove("./common_assets/bacalhau_bin")
 	//Exit with the same code as the test run
 	os.Exit(exitCode)
 }
diff --git a/test_integration/run_locally.sh b/test_integration/run_locally.sh
new file mode 100755
index 0000000000..636fe3234f
--- /dev/null
+++ b/test_integration/run_locally.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# Setup cleanup trap
+cleanup() {
+    echo "Cleaning up..."
+    rm -f common_assets/bacalhau_bin
+}
+trap cleanup EXIT
+
+# cd to main repo
+cd ..
+
+# Get GOARCH value and print it for verification
+GOARCH=$(go env GOARCH)
+echo "Building for architecture: ${GOARCH}"
+
+# Remove old binary if exists
+rm -f "bin/linux/${GOARCH}/bacalhau"
+
+# Remove old binary from test_integration directory
+rm -f test_integration/common_assets/bacalhau_bin
+
+# Run make build with GOOS=linux
+GOOS=linux make build
+
+# Verify the binary exists
+if [[ ! -f "bin/linux/${GOARCH}/bacalhau" ]]; then
+    echo "Error: Binary was not created at bin/linux/${GOARCH}/bacalhau"
+    exit 1
+fi
+
+# Copy binary to test assets directory
+mkdir -p test_integration/common_assets
+cp "bin/linux/${GOARCH}/bacalhau" test_integration/common_assets/bacalhau_bin
+
+cd test_integration || exit
+
+# Run tests
+go test -v -count=1 ./...
diff --git a/test_integration/utils/test_utils.go b/test_integration/utils/test_utils.go
index dde9b7255e..71ba5fb952 100644
--- a/test_integration/utils/test_utils.go
+++ b/test_integration/utils/test_utils.go
@@ -11,11 +11,8 @@ import (
 	dc "github.com/docker/docker/client"
 	"github.com/docker/docker/pkg/archive"
 	"github.com/stretchr/testify/suite"
-	"github.com/testcontainers/testcontainers-go"
-	"github.com/testcontainers/testcontainers-go/wait"
 	"io"
 	"os"
-	"path/filepath"
 	"regexp"
 	"strings"
 )
@@ -99,82 +96,6 @@ func DeleteDockerTestImagesAndPrune(suffix string) error {
 	return nil
 }
 
-func CompileBacalhau(ctx context.Context, programPath string) error {
-	programDir, err := filepath.Abs(filepath.Dir(programPath))
-	if err != nil {
-		return fmt.Errorf("compiling bacalhau: failed to get absolute path: %w", err)
-	}
-
-	// Create a container request
-	// TODO: Improve how we build our binary
-	req := testcontainers.ContainerRequest{
-		Image: "golang:1.23",
-		Cmd: []string{
-			"go",
-			"build",
-			"-buildvcs=false",
-			"-o", "/usr/src/bacalhau-binary",
-			"-ldflags", "-linkmode external -extldflags '-static'",
-			"-a",
-			"./"},
-		Files: []testcontainers.ContainerFile{
-			{
-				HostFilePath:      programDir,
-				ContainerFilePath: "/usr/src/",
-				FileMode:          0755,
-			},
-		},
-		WorkingDir: "/usr/src/bacalhau",
-		WaitingFor: wait.ForExit(),
-	}
-
-	// Start the container
-	container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
-		ContainerRequest: req,
-		Started:          true,
-	})
-	if err != nil {
-		return fmt.Errorf("compiling bacalhau: failed to start container: %w", err)
-	}
-	defer container.Terminate(ctx)
-
-	// Get the logs
-	logs, err := container.Logs(ctx)
-	if err != nil {
-		return fmt.Errorf("compiling bacalhau: failed to get container logs: %w", err)
-	}
-	defer logs.Close()
-
-	// Print the logs
-	if _, err := io.Copy(os.Stdout, logs); err != nil {
-		return fmt.Errorf("compiling bacalhau: failed to print logs: %w", err)
-	}
-
-	// Copy the compiled binary from the container to the host
-	reader, err := container.CopyFileFromContainer(ctx, "/usr/src/bacalhau-binary")
-	if err != nil {
-		return fmt.Errorf("compiling bacalhau: failed to copy file from container: %w", err)
-	}
-	defer reader.Close()
-
-	// Create the output file on the host
-	outFile, err := os.Create(filepath.Join(programDir, "test_integration", "common_assets", "bacalhau_bin"))
-	if err != nil {
-		return fmt.Errorf("compiling bacalhau: failed to create output file: %w", err)
-	}
-	defer outFile.Close()
-
-	// Copy the content from the reader to the output file
-	_, err = io.Copy(outFile, reader)
-	if err != nil {
-		return fmt.Errorf("compiling bacalhau: failed to write compiled program to file: %w", err)
-	}
-
-	fmt.Println("Compiled program has been copied to the host machine.")
-
-	return nil
-}
-
 func BuildBaseImages(testIdentifier string) error {
 	err := buildDockerImage(
 		"common_assets/dockerfiles/Dockerfile-JumpboxNode",