Repository: celestiaorg/celestia-app

Issue:
Add multiplexer test
## Context

https://github.com/celestiaorg/celestia-app/pull/4842#issuecomment-2898041529

## Problem

There are no unit tests in this repo that verify correct behavior of the multiplexer

## Proposal

Add a test like https://github.com/01builders/nova/blob/360d472/appd/run_test.go#L12-L35 to this repo

Hints from issue comments:

Solution patch:
diff --git a/.github/workflows/docker-build-publish.yml b/.github/workflows/docker-build-publish.yml
index 68a54af7..7a64c189 100644
--- a/.github/workflows/docker-build-publish.yml
+++ b/.github/workflows/docker-build-publish.yml
@@ -37,7 +37,6 @@ jobs:
       dockerfile: docker/multiplexer.Dockerfile
       checkout_ref: ${{ github.event.inputs.ref }}
       packageName: celestia-app
-      buildArgs: "TARGETOS=linux TARGETARCH=amd64"
     secrets: inherit
 
   docker-txsim-build:
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 310a5d0f..d9c8bb4a 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -69,6 +69,18 @@ jobs:
       - name: Run fuzz tests
         run: make test-fuzz
 
+  test-multiplexer:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2
+
+      - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 #v5.5.0
+        with:
+          go-version-file: "go.mod"
+
+      - name: Run multiplexer tests
+        run: make test-multiplexer
+
   test-docker-e2e:
     runs-on: ubuntu-latest
     # if one test fails, continue running the rest.
diff --git a/.goreleaser.yaml b/.goreleaser.yaml
index b8d97bd3..ee564349 100644
--- a/.goreleaser.yaml
+++ b/.goreleaser.yaml
@@ -2,11 +2,110 @@
 # Since GoReleaser doesn't support CGO natively, our GoReleaser process builds
 # binaries in a Docker image maintained by goreleaser-cross that has CGO support
 # for multiple platforms. See https://github.com/goreleaser/goreleaser-cross
+
 version: 2
 before:
   hooks:
     - go mod tidy
+    # these hooks get run once. We want to explicitly download every embedded binary so we can build the multiplexer
+    # for every arch.
+    - ./scripts/download_v3_binary.sh celestia-app_Darwin_arm64.tar.gz celestia-app_darwin_v3_arm64.tar.gz v3.10.0-arabica
+    - ./scripts/download_v3_binary.sh celestia-app_Linux_arm64.tar.gz celestia-app_linux_v3_arm64.tar.gz v3.10.0-arabica
+    - ./scripts/download_v3_binary.sh celestia-app_Darwin_x86_64.tar.gz celestia-app_darwin_v3_amd64.tar.gz v3.10.0-arabica
+    - ./scripts/download_v3_binary.sh celestia-app_Linux_x86_64.tar.gz celestia-app_linux_v3_amd64.tar.gz v3.10.0-arabica
 builds:
+  - id: darwin-amd64-multiplexer
+    main: ./cmd/celestia-appd
+    binary: celestia-appd
+    env:
+      - SDKPath={{ "github.com/cosmos/cosmos-sdk/version" }}
+      - CGO_ENABLED=1
+      - CC=o64-clang
+      - CXX=o64-clang++
+    goarch:
+      - amd64
+    goos:
+      - darwin
+    tags:
+      - ledger
+      - multiplexer
+    ldflags:
+      # Ref: https://goreleaser.com/customization/templates/#common-fields
+      # .Version is the version being released
+      # .FullCommit is git commit hash goreleaser is using for the release
+      - -X "{{ .Env.SDKPath }}.Name=celestia-app"
+      - -X "{{ .Env.SDKPath }}.AppName=celestia-appd"
+      - -X "{{ .Env.SDKPath }}.Version={{ .Version }}"
+      - -X "{{ .Env.SDKPath }}.Commit={{ .FullCommit }}"
+  - id: darwin-arm64-multiplexer
+    main: ./cmd/celestia-appd
+    binary: celestia-appd
+    env:
+      - SDKPath={{ "github.com/cosmos/cosmos-sdk/version" }}
+      - CGO_ENABLED=1
+      - CC=oa64-clang
+      - CXX=oa64-clang++
+    goarch:
+      - arm64
+    goos:
+      - darwin
+    tags:
+      - ledger
+      - multiplexer
+    ldflags:
+      # Ref: https://goreleaser.com/customization/templates/#common-fields
+      # .Version is the version being released
+      # .FullCommit is git commit hash goreleaser is using for the release
+      - -X "{{ .Env.SDKPath }}.Name=celestia-app"
+      - -X "{{ .Env.SDKPath }}.AppName=celestia-appd"
+      - -X "{{ .Env.SDKPath }}.Version={{ .Version }}"
+      - -X "{{ .Env.SDKPath }}.Commit={{ .FullCommit }}"
+  - id: linux-amd64-multiplexer
+    main: ./cmd/celestia-appd
+    binary: celestia-appd
+    env:
+      - SDKPath={{ "github.com/cosmos/cosmos-sdk/version" }}
+      - CGO_ENABLED=1
+      - CC=x86_64-linux-gnu-gcc
+      - CXX=x86_64-linux-gnu-g++
+    goarch:
+      - amd64
+    goos:
+      - linux
+    tags:
+      - ledger
+      - multiplexer
+    ldflags:
+      # Ref: https://goreleaser.com/customization/templates/#common-fields
+      # .Version is the version being released
+      # .FullCommit is git commit hash goreleaser is using for the release
+      - -X "{{ .Env.SDKPath }}.Name=celestia-app"
+      - -X "{{ .Env.SDKPath }}.AppName=celestia-appd"
+      - -X "{{ .Env.SDKPath }}.Version={{ .Version }}"
+      - -X "{{ .Env.SDKPath }}.Commit={{ .FullCommit }}"
+  - id: linux-arm64-multiplexer
+    main: ./cmd/celestia-appd
+    binary: celestia-appd
+    env:
+      - SDKPath={{ "github.com/cosmos/cosmos-sdk/version" }}
+      - CGO_ENABLED=1
+      - CC=aarch64-linux-gnu-gcc
+      - CXX=aarch64-linux-gnu-g++
+    goarch:
+      - arm64
+    goos:
+      - linux
+    tags:
+      - ledger
+      - multiplexer
+    ldflags:
+      # Ref: https://goreleaser.com/customization/templates/#common-fields
+      # .Version is the version being released
+      # .FullCommit is git commit hash goreleaser is using for the release
+      - -X "{{ .Env.SDKPath }}.Name=celestia-app"
+      - -X "{{ .Env.SDKPath }}.AppName=celestia-appd"
+      - -X "{{ .Env.SDKPath }}.Version={{ .Version }}"
+      - -X "{{ .Env.SDKPath }}.Commit={{ .FullCommit }}"
   - id: darwin-amd64
     main: ./cmd/celestia-appd
     binary: celestia-appd
@@ -95,11 +194,31 @@ builds:
       - -X "{{ .Env.SDKPath }}.Commit={{ .FullCommit }}"
 dist: ./build/goreleaser
 archives:
-  - formats: ['tar.gz']
+  - id: multiplexer
+    builds:
+      - darwin-amd64-multiplexer
+      - darwin-arm64-multiplexer
+      - linux-amd64-multiplexer
+      - linux-arm64-multiplexer
+    formats: ['tar.gz']
+    name_template: >-
+      {{ .ProjectName }}_
+      {{- title .Os }}_
+      {{- if eq .Arch "amd64" }}x86_64
+      {{- else if eq .Arch "386" }}i386
+      {{- else }}{{ .Arch }}{{ end }}
+      {{- if .Arm }}v{{ .Arm }}{{ end }}
+  - id: standalone
+    builds:
+      - darwin-amd64
+      - darwin-arm64
+      - linux-amd64
+      - linux-arm64
+    formats: ['tar.gz']
     # this name template makes the OS and Arch compatible with the results of
     # uname.
     name_template: >-
-      {{ .ProjectName }}_
+      {{ .ProjectName }}-standalone_
       {{- title .Os }}_
       {{- if eq .Arch "amd64" }}x86_64
       {{- else if eq .Arch "386" }}i386
diff --git a/Makefile b/Makefile
index efcfd4d9..459ea1e4 100644
--- a/Makefile
+++ b/Makefile
@@ -9,8 +9,6 @@ PACKAGE_NAME          := github.com/celestiaorg/celestia-app/v4
 # Before upgrading the GOLANG_CROSS_VERSION, please verify that a Docker image exists with the new tag.
 # See https://github.com/goreleaser/goreleaser-cross/pkgs/container/goreleaser-cross
 GOLANG_CROSS_VERSION  ?= v1.23.6
-# Set this to override the max square size of the binary
-OVERRIDE_MAX_SQUARE_SIZE ?=
 # Set this to override v2 upgrade height for the v3 embedded binaries
 V2_UPGRADE_HEIGHT ?= 0
 
@@ -246,9 +244,9 @@ test-docker-e2e:
 .PHONY: test-docker-e2e
 
 ## test-multiplexer: Run unit tests for the multiplexer package.
-test-multiplexer:
+test-multiplexer: download-v3-binaries
 	@echo "--> Running multiplexer tests"
-	make test -C ./multiplexer
+	@go test -tags multiplexer ./multiplexer/...
 .PHONY: test-multiplexer
 
 ## test-race: Run tests in race mode.
@@ -335,9 +333,25 @@ prebuilt-binary:
 		-v `pwd`:/go/src/$(PACKAGE_NAME) \
 		-w /go/src/$(PACKAGE_NAME) \
 		ghcr.io/goreleaser/goreleaser-cross:${GOLANG_CROSS_VERSION} \
-		release --clean --parallelism 1
+		release --clean --parallelism 2
 .PHONY: prebuilt-binary
 
+## go-releaser-dry-run ensures that the go releaser tool and build all the artefacts correctly.
+## specifies parallelism as 4 so should be run locally. On the regular github runners 2 should be the max.
+go-releaser-dry-run:
+	@echo "Running GoReleaser in dry-run mode..."
+	docker run \
+		--rm \
+		--env CGO_ENABLED=1 \
+		--env GORELEASER_CURRENT_TAG=${GORELEASER_CURRENT_TAG} \
+		--env-file .release-env \
+		-v /var/run/docker.sock:/var/run/docker.sock \
+		-v `pwd`:/go/src/$(PACKAGE_NAME) \
+		-w /go/src/$(PACKAGE_NAME) \
+		ghcr.io/goreleaser/goreleaser-cross:${GOLANG_CROSS_VERSION} \
+		release --snapshot --clean --parallelism 4
+.PHONY: go-releaser-dry-run
+
 ## goreleaser: Create prebuilt binaries and attach them to GitHub release. Requires Docker.
 goreleaser: prebuilt-binary
 .PHONY: goreleaser
diff --git a/docker/Dockerfile b/docker/Dockerfile
index ce6fb8da..39528fb2 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -41,8 +41,6 @@ COPY . .
 
 RUN uname -a &&\
     CGO_ENABLED=${CGO_ENABLED} GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
-    OVERRIDE_MAX_SQUARE_SIZE=${MAX_SQUARE_SIZE} \
-    OVERRIDE_UPGRADE_HEIGHT_DELAY=${UPGRADE_HEIGHT_DELAY} \
     make build-standalone
 
 # Stage 2: Create a minimal image to run the celestia-appd binary
diff --git a/docker/multiplexer.Dockerfile b/docker/multiplexer.Dockerfile
index 795df893..863136e3 100644
--- a/docker/multiplexer.Dockerfile
+++ b/docker/multiplexer.Dockerfile
@@ -64,8 +64,6 @@ RUN tar -cvzf internal/embedding/celestia-app_${TARGETOS}_v3_${TARGETARCH}.tar.g
 
 RUN uname -a &&\
     CGO_ENABLED=${CGO_ENABLED} GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
-    OVERRIDE_MAX_SQUARE_SIZE=${MAX_SQUARE_SIZE} \
-    OVERRIDE_UPGRADE_HEIGHT_DELAY=${UPGRADE_HEIGHT_DELAY} \
     make build DOWNLOAD=false
 
 # Stage 3: Create a minimal image to run the celestia-appd binary
diff --git a/go.mod b/go.mod
index 0c26a232..3c49954d 100644
--- a/go.mod
+++ b/go.mod
@@ -306,5 +306,5 @@ replace (
 	// goleveldb: canonical version
 	github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
 	// celestia-core(v0.34.x): used for multiplexing abci v1 requests
-	github.com/tendermint/tendermint => github.com/celestiaorg/celestia-core v1.45.0-tm-v0.34.35
+	github.com/tendermint/tendermint => github.com/celestiaorg/celestia-core v1.52.0-tm-v0.34.35
 )
diff --git a/go.sum b/go.sum
index 44f40346..9aa1bb1a 100644
--- a/go.sum
+++ b/go.sum
@@ -352,8 +352,8 @@ github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFos
 github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
 github.com/celestiaorg/bittwister v0.0.0-20231213180407-65cdbaf5b8c7 h1:nxplQi8wrLMjhu260RuigXylC3pWoDu4OVumPHeojnk=
 github.com/celestiaorg/bittwister v0.0.0-20231213180407-65cdbaf5b8c7/go.mod h1:1EF5MfOxVf0WC51Gb7pJ6bcZxnXKNAf9pqWtjgPBAYc=
-github.com/celestiaorg/celestia-core v1.45.0-tm-v0.34.35 h1:T21AhezjcByAlWDHmiVbpg743Uqk/dqBzJkQsAnbQf8=
-github.com/celestiaorg/celestia-core v1.45.0-tm-v0.34.35/go.mod h1:fQ46s1hYFTGFBsHsuGsbxDZ720ZPQow5Iyqw+yErZSo=
+github.com/celestiaorg/celestia-core v1.52.0-tm-v0.34.35 h1:iU2igosCxYKzEUi85u3YqNslIO+rCXtx4GKAHUzxrDQ=
+github.com/celestiaorg/celestia-core v1.52.0-tm-v0.34.35/go.mod h1:SI38xqZZ4ccoAxszUJqsJ/a5rOkzQRijzHQQlLKkyUc=
 github.com/celestiaorg/celestia-core v1.53.0-tm-v0.38.17 h1:AHjnWwPoOFH5zr+aPqdWkF1CutP1ynVEOGOuW1/8iq8=
 github.com/celestiaorg/celestia-core v1.53.0-tm-v0.38.17/go.mod h1:PiU80T/t0z8FPlz7DRZgvrBux0jJiqxFi9lNBFdGUps=
 github.com/celestiaorg/cosmos-sdk v1.29.1-sdk-v0.50.12 h1:EWJIBTMB9H2KFMqmNB22KvgqvAObqcToJbAdPCM+LBw=
diff --git a/multiplexer/appd/run.go b/multiplexer/appd/run.go
index 399aa556..f4667021 100644
--- a/multiplexer/appd/run.go
+++ b/multiplexer/appd/run.go
@@ -42,17 +42,11 @@ func New(name string, bin []byte, cfg ...CfgOption) (*Appd, error) {
 		return nil, fmt.Errorf("failed to create temp directory: %w", err)
 	}
 
-	// cleanups functions
+	// cleanup function to remove the extracted binary.
 	cleanup := func() {
 		_ = os.RemoveAll(tmpDir)
 	}
 
-	defer func() {
-		if err != nil {
-			cleanup()
-		}
-	}()
-
 	// extract all files from the tar archive to the temp directory
 	tr := tar.NewReader(gzr)
 	for {
diff --git a/scripts/download_v3_binary.sh b/scripts/download_v3_binary.sh
index 4f069ce7..266f3792 100755
--- a/scripts/download_v3_binary.sh
+++ b/scripts/download_v3_binary.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Usage: ./scripts/embed_bin.sh <url> <out> <version>
+# Usage: ./scripts/download_v3_binary.sh <url> <out> <version>
 
 set -euo pipefail
 



Test patch:
diff --git a/multiplexer/appd/run_test.go b/multiplexer/appd/run_test.go
index 7d510e04..9c812f9b 100644
--- a/multiplexer/appd/run_test.go
+++ b/multiplexer/appd/run_test.go
@@ -1,12 +1,47 @@
+//go:build multiplexer
+
 package appd
 
 import (
+	"bytes"
+	"github.com/celestiaorg/celestia-app/v4/internal/embedding"
 	"os"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/require"
 )
 
+// TestCreateExecCommand execs a command to an embedded binary.
+func TestCreateExecCommand(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping test which expects an embedded binary")
+	}
+
+	bin, err := embedding.CelestiaAppV3()
+	require.NoError(t, err)
+
+	appdInstance, err := New("test-app", bin)
+	require.NoError(t, err)
+	require.NotNil(t, appdInstance)
+
+	cmd := appdInstance.CreateExecCommand("version")
+	require.NotNil(t, cmd)
+
+	var outputBuffer bytes.Buffer
+	cmd.Stdout = &outputBuffer
+
+	require.NoError(t, cmd.Run())
+
+	output := outputBuffer.String()
+	require.NotEmpty(t, output)
+	require.True(t, strings.HasPrefix(output, "3"), "major version should be 3 of embedded binary. Got: %s", output)
+
+	appdInstance.cleanup()
+	require.Error(t, cmd.Run(), "expected error when running command after cleanup")
+
+}
+
 // TestStart_Success ensures that the provided executable is launched and provided a pid.
 // and that the pid is reset once the process exits.
 func TestStart_Success(t *testing.T) {
diff --git a/test/docker-e2e/go.mod b/test/docker-e2e/go.mod
index 9fd96eda..195b66ac 100644
--- a/test/docker-e2e/go.mod
+++ b/test/docker-e2e/go.mod
@@ -272,12 +272,12 @@ replace (
 	github.com/celestiaorg/celestia-app/v4 => ../..
 	// celestia-core: v1.53.0-tm-v0.38.17
 	github.com/cometbft/cometbft => github.com/celestiaorg/celestia-core v1.53.0-tm-v0.38.17
-	// cosmos-sdk: v1.29.0-sdk-v0.50.12
+	// cosmos-sdk: v1.29.1-sdk-v0.50.12
 	github.com/cosmos/cosmos-sdk => github.com/celestiaorg/cosmos-sdk v1.29.1-sdk-v0.50.12
 
 	github.com/gogo/protobuf => github.com/gogo/protobuf v1.3.2
 	// goleveldb: canonical version
 	github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
 	// celestia-core(v0.34.x): used for multiplexing abci v1 requests
-	github.com/tendermint/tendermint => github.com/celestiaorg/celestia-core v1.45.0-tm-v0.34.35
+	github.com/tendermint/tendermint => github.com/celestiaorg/celestia-core v1.52.0-tm-v0.34.35
 )