Repository: celestiaorg/celestia-node
Pull request #526 (2022-03-21 19:54:16)

Update docker-build GH action to push sha tagged images
This PR updates the docker-build GH Action. Notable changes
- Runs on pushes to all branches and tags the image with the short sha of the git commit
- Builds multi platform (amd64 and arm64) images using cross compilation

You can see an example of a successful run [here](https://github.com/jbowen93/celestia-node/actions/runs/2018410000) with the resulting images pushed [here](https://github.com/jbowen93/celestia-node/pkgs/container/celestia-node)


Patch:
diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml
index 87b0d0c0..97b7cb91 100644
--- a/.github/ISSUE_TEMPLATE/bug-report.yml
+++ b/.github/ISSUE_TEMPLATE/bug-report.yml
@@ -41,7 +41,7 @@ body:
       label: Steps to reproduce it
       description: What steps have you made to reproduce it? 
       placeholder: Tell us what you see!
-      value: "Example: 1. Make build 2. Execute binary"
+      value:
     validations:
       required: true
   - type: textarea
@@ -50,7 +50,7 @@ body:
       label: Expected result
       description: What do you expect to happen as a final result? 
       placeholder: Let us know what is expected
-      value: "Example: Celestia node is running in default mode"
+      value:
     validations:
       required: true
   - type: textarea
@@ -59,7 +59,7 @@ body:
       label: Actual result
       description: What do you see happened instead as a final result? 
       placeholder: This is the crucial part in detecting the root cause of the issue
-      value: "Example: Celestia node is throwing errors during startup"
+      value:
     validations:
       required: true
   - type: textarea
diff --git a/.github/ISSUE_TEMPLATE/change-request.yml b/.github/ISSUE_TEMPLATE/change-request.yml
deleted file mode 100644
index f9e31cbc..00000000
--- a/.github/ISSUE_TEMPLATE/change-request.yml
+++ /dev/null
@@ -1,54 +0,0 @@
-name: Change Request
-description: Request a change to inform the community on what existing part of the project can be improved!
-title: "[Change Request]: "
-labels: ["change request",]
-body:
-  - type: markdown
-    attributes:
-      value: |
-        Thank you for finding new ways of improving existing features!
-  - type: input
-    id: summary
-    attributes: 
-      label: Summary
-      description: Keep it simple and short. This helps with further deep dive into the change request
-      value: "Example: New ABC plug-n-play replacement for lib XYZ that is used in DEF part"
-    validations:
-      required: true
-  - type: textarea
-    id: gain
-    attributes:
-      label: Gain 
-      description: What is the gain of this change request? Please describe the benefits
-      value: "Example: After official release of ABC lib, it has proven in many benchmarks to outperform existing XYZ lib by N%"
-    validations:
-      required: true
-  - type: textarea
-    id: evidence
-    attributes:
-      label: Evidence
-      description: What are the resources that you have read to present it to others?
-      value: "Example: https:/some links"
-    validations:
-      required: true
-  - type: textarea
-    id: ideas
-    attributes:
-      label: Implementation ideas
-      description: What ideas do you have on how to implement this?
-      value: | 
-        Example: We can start with wrapping existing N code snippet to make implementation easier for feature ABC. 
-        Still the other option is re-writing this code snippet to achieve neater code design.
-  - type: dropdown
-    id: urgency
-    attributes:
-      label: Urgency
-      description: How urgent this request is? 
-      options:
-        - Don't Know
-        - Low
-        - Medium
-        - High
-        - Critical
-    validations:
-      required: true
diff --git a/.github/ISSUE_TEMPLATE/feature-request.yml b/.github/ISSUE_TEMPLATE/feature-request.yml
index 962075e8..ba7439cf 100644
--- a/.github/ISSUE_TEMPLATE/feature-request.yml
+++ b/.github/ISSUE_TEMPLATE/feature-request.yml
@@ -7,52 +7,8 @@ body:
     attributes:
       value: |
         Thank you for taking your time and creating a new feature request!
-  - type: input
-    id: summary
-    attributes: 
-      label: Summary
-      description: Keep it simple and short. This helps with further deep dive into the feature
-      value: "Example: Users have pain-points without ABC feature"
-    validations:
-      required: true
-  - type: textarea
-    id: impact
-    attributes:
-      label: Impact on users
-      description: Suggestions on how this impact users
-      value: "Example: This wonderful ABC feature impacts X that matters to users that need Y"
-    validations:
-      required: true
-  - type: textarea
-    id: undone
-    attributes:
-      label: Cost of not doing
-      description: What can go wrong if this is not implemented
-      value: "Example: The risk of losing Z% of future users is high due to no ABC feature"
-  - type: textarea
-    id: evidence
-    attributes:
-      label: Evidence
-      description: What are the evidences that are validating this feature?
-      value: "Example: The W post/implementation in other projects has validated this ABC feature to be very useful for the majority of users"
   - type: textarea
     id: ideas
     attributes:
       label: Implementation ideas
       description: What ideas do you have on how to implement this?
-      value: | 
-        Example: We can start with wrapping existing N code snippet to make implementation easier for feature ABC. 
-        Still the other option is re-writing this code snippet to achieve neater code design.
-  - type: dropdown
-    id: urgency
-    attributes:
-      label: Urgency
-      description: How urgent this request is? 
-      options:
-        - Don't Know
-        - Low
-        - Medium
-        - High
-        - Critical
-    validations:
-      required: true
diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index f267523a..a80fedf8 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -3,9 +3,13 @@ name: "docker-build"
 on:
   push:
     branches:
-      - "main"
+      - "**"
   workflow_dispatch:
 
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
 jobs:
   docker-build:
     runs-on: "ubuntu-latest"
@@ -19,10 +23,13 @@ jobs:
         uses: actions/setup-go@v2
         with:
           go-version: 1.17
-      - name: "Build"
-        run: "make build"
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=sha
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
       - name: Login to GHCR
@@ -34,6 +41,7 @@ jobs:
       - name: Build and push
         uses: docker/build-push-action@v2
         with:
+          platforms: linux/amd64, linux/arm64
           push: true
-          tags: ghcr.io/celestiaorg/celestia-node:latest
+          tags: ${{ steps.meta.outputs.tags }}
           file: docker/Dockerfile
diff --git a/docker/Dockerfile b/docker/Dockerfile
index fa4fde10..21d6ff17 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -1,10 +1,12 @@
-FROM golang:1.17 as builder
-RUN apt update && \
-    apt install make bash
-ENV HOME /celestia-node
-COPY / ${HOME}
-WORKDIR ${HOME}
-RUN env GOOS=linux GOARCH=amd64 make build
+FROM --platform=$BUILDPLATFORM golang:1.17 as builder
+RUN apt-get update && \
+    apt-get install make bash
+WORKDIR /src
+COPY go.mod go.sum ./
+RUN go mod download
+COPY . .
+ARG TARGETOS TARGETARCH
+RUN env GOOS=$TARGETOS GOARCH=$TARGETARCH make build
 
 FROM ubuntu
 # Default node type can be overwritten in deployment manifest
@@ -13,7 +15,7 @@ ENV NODE_TYPE bridge
 COPY docker/entrypoint.sh /
 
 # Copy in the binary
-COPY --from=builder /celestia-node/celestia /
+COPY --from=builder /src/celestia /
 
 EXPOSE 2121
 
diff --git a/docs/adr/adr-006-fraud-service.md b/docs/adr/adr-006-fraud-service.md
new file mode 100644
index 00000000..30859f3a
--- /dev/null
+++ b/docs/adr/adr-006-fraud-service.md
@@ -0,0 +1,197 @@
+# ADR #006: Celestia-Node Fraud Service
+
+## Changelog
+
+- 2022.03.03 - init commit
+- 2022.03.08 - added pub-sub
+- 2022.03.15 - added BEFP verification
+
+## Authors
+
+@vgonkivs @Bidon15 @adlerjohn @Wondertan @renaynay
+ 
+## Bad Encoding Fraud Proof (BEFP)
+## Context
+
+In the case where a Full Node receives `ErrByzantineRow`/`ErrByzantineCol` from the [rsmt2d](https://github.com/celestiaorg/rsmt2d) library, it generates a fraud-proof and broadcasts it to DA network such that the light nodes are notified that the corresponding block could be malicious.
+
+## Decision
+
+BEFPs were first addressed in the two issues below:
+
+- https://github.com/celestiaorg/celestia-node/issues/4
+- https://github.com/celestiaorg/celestia-node/issues/263
+
+## Detailed Design
+A fraud proof is generated if recovered data does not match with its respective row/column roots during block reparation. 
+
+The result of `RepairExtendedDataSquare` will be an error [`ErrByzantineRow`](https://github.com/celestiaorg/rsmt2d/blob/f34ec414859fc834835ea97ed54300404eec1ac5/extendeddatacrossword.go#L18-L22)/[`ErrByzantineCol`](https://github.com/celestiaorg/rsmt2d/blob/f34ec414859fc834835ea97ed54300404eec1ac5/extendeddatacrossword.go#L28-L32):
+
+- Both errors consist of 
+  - row/column numbers that do not match with the Merkle root
+  - shares that were successfully repaired and verified (all correct shares).
+
+Based on `ErrByzantineRow`/`ErrByzantineCol` internal fields, we should generate [MerkleProof](https://github.com/celestiaorg/nmt/blob/e381b44f223e9ac570a8d59bbbdbb2d5a5f1ad5f/proof.go#L17) for respective verified shares from [nmt](https://github.com/celestiaorg/nmt) tree return as the `ErrBadEncoding` from `RetrieveData`. 
+
+```go
+type ErrBadEncoding struct {
+   // Shares contains all shares from row/col.
+   // For non-nil shares MerkleProof is computed
+   Shares []*Share
+   // Position represents the number of row/col where ErrByzantineRow/ErrByzantineColl occurred.
+   Position uint8
+   isRow bool
+}
+
+type Share struct {
+   Share []byte
+   Proof nmt.Proof
+}
+```
+
+In addition, `das.Daser`:
+
+1. Creates a BEFP:
+
+```go
+const (
+   BadEncoding ProofType = "BadEncoding"
+)
+
+type BadEncodingProof struct {
+   Height uint64
+   // Shares contains all shares from row/col
+   // Shares that did not pass verification in rmst2d will be nil
+   // For non-nil shares MerkleProofs are computed
+   Shares []*Share
+   // Position represents the number of row/col where ErrByzantineRow/ErrByzantineColl occurred
+   Position uint8
+   isRow bool
+}
+```
+
+2. Full node broadcasts BEFP to all light nodes via separate sub-service via proto message:
+
+```proto3
+
+message MerkleProof {
+  int64          total     = 1;
+  int64          index     = 2;
+  bytes          leaf_hash = 3;
+  repeated bytes aunts     = 4;
+}
+
+message Share {
+   bytes Share = 1;
+   MerkleProof Proof = 2;
+}
+
+message BadEnconding {
+   required uint64 Height = 1;
+   repeated Share Shares = 2;
+   uint8 Position = 3;
+   bool isRow = 4;
+}
+```
+
+`das.Daser` imports a data structure that implements `proof.Broadcaster` interface that uses libp2p.pubsub under the hood:
+
+```go
+// Broadcaster is a generic interface that sends a `Proof` to all nodes subscribed on the Broadcaster's topic.
+type Broadcaster interface {
+   // Broadcast takes a fraud `Proof` data structure that implements standard BinaryMarshal interface and broadcasts it to all subscribed peers.
+   Broadcast(ctx context.Context, p Proof) error
+}
+```
+
+```go
+// ProofType is a enum type that represents a particular type of fraud proof.
+type ProofType string
+
+// Proof is a generic interface that will be used for all types of fraud proofs in the network.
+type Proof interface {
+   Type() ProofType
+   Height() (uint64, error)
+   Validate(*header.ExtendedHeader) error
+
+   encoding.BinaryMarshaller
+}
+```
+
+2a. From the other side, light nodes will, by default, subscribe to the BEFP topic and verify messages received on the topic:
+
+```go
+type proofUnmarshaller func([]byte) Proof
+// Subscriber encompasses the behavior necessary to
+// subscribe/unsubscribe from new FraudProofs events from the
+// network.
+type Subscriber interface {
+   // Subscribe allows to subscribe on pub sub topic by it's type.
+   // Subscribe should register pub-sub validator on topic.
+   Subscribe(ctx context.Context, proofType ProofType) (Subscription, error)
+   // RegisterUnmarshaller registers unmarshaller for the given ProofType.
+   // If there is no umarshaller for `ProofType`, then `Subscribe` returns an error.
+   RegisterUnmarshaller(proofType ProofType, f proofUnmarshaller) error
+   // UnregisterUnmarshaller removes unmarshaller for the given ProofType.
+   // If there is no unmarshaller for `ProofType`, then it returns an error.
+   UnregisterUnmarshaller(proofType ProofType) error{}
+}
+```
+
+```go
+// Subscription returns a valid proof if one is received on the topic.
+type Subscription interface {
+   Proof() (Proof, error)
+   Cancel() error
+}
+```
+
+```go
+// FraudSub implements Subscriber and Broadcaster.
+type FraudSub struct {
+   pubsub *pubsub.PubSub
+   topics map[ProofType]*pubsub.Topic
+   unmarshallers map[ProofType]proofUnmarshaller
+}
+
+func(s *FraudSub) RegisterUnmarshaller(proofType ProofType, f proofUnmarshaller) error{}
+func(s *FraudSub) UnregisterUnmarshaller(proofType ProofType) error{}
+
+func(s *FraudSub) Subscribe(ctx context.Context, proofType ProofType) (Subscription, error){}
+func(s *FraudSub) Broadcast(ctx context.Context, p Proof) error{}
+```
+### BEFP verification
+Once a light node receives a `BadEncodingProof` fraud proof, it should:
+* verify that Merkle proofs correspond to particular shares. If the Merkle proof does not correspond to a share, then the BEFP is not valid.
+* using `BadEncodingProof.Shares`, light node should re-construct full row or column, compute its Merkle root as in [rsmt2d](https://github.com/celestiaorg/rsmt2d/blob/ac0f1e1a51bf7b5420965fb7c35fa32a56e02292/extendeddatacrossword.go#L410) and compare it with Merkle root that could be retrieved from the `DataAvailabilityHeader` inside the `ExtendedHeader`. If Merkle roots do not match, then the BEFP is not valid.
+
+3. All celestia-nodes should stop some dependent services upon receiving a legitimate BEFP:
+Both full and light nodes should stop `DAS`, `Syncer` and `SubmitTx` services.
+
+4. Valid BadEncodingFraudProofs should be stored on the disk using `FraudStore` interface:
+
+```go
+type FraudStore interface {
+   // Put stores given Proof by header's hash in respective for this fraud proof directory
+   Put(path FraudProofType, headerHash string, p Proof) error
+   // Get retrieves Proof by header's hash from respective for this fraud proof directory
+   Get(path FraudProofType, headerHash string) (Proof, error)
+   // GetMany retrieves all Proofs from the respective for this fraud proof directory
+   GetMany(path FraudProofType) ([]Proof, error)
+}
+
+type fraudstore struct{
+   path string
+}
+```
+### Bridge node behaviour
+Bridge nodes will behave as light nodes do by subscribing to BEFP fraud sub and listening for BEFPs. If a BEFP is received, it will similarly shut down all dependent services, including broadcasting new `ExtendedHeader`s to the network.
+
+## Status
+Proposed
+
+## References
+
+Data Availability(Bad Encoding) Fraud Proofs: [#4](https://github.com/celestiaorg/celestia-node/issues/4)
+   
+Implement stubs for BadEncodingFraudProofs: [#263](https://github.com/celestiaorg/celestia-node/issues/263) 


Test patch:
diff --git a/node/tests/swamp/swamp.go b/node/tests/swamp/swamp.go
index a106c25b..9e4c23ab 100644
--- a/node/tests/swamp/swamp.go
+++ b/node/tests/swamp/swamp.go
@@ -178,76 +178,54 @@ func (s *Swamp) getTrustedHash(ctx context.Context) (string, error) {
 }
 
 // NewBridgeNode creates a new instance of a BridgeNode providing a default config
-// and a mockstore to the NewBridgeNodeWithStore method
+// and a mockstore to the NewNodeWithStore method
 func (s *Swamp) NewBridgeNode(options ...node.Option) *node.Node {
 	cfg := node.DefaultConfig(node.Bridge)
 	store := node.MockStore(s.t, cfg)
 
-	return s.NewBridgeNodeWithStore(store, options...)
+	return s.NewNodeWithStore(node.Bridge, store, options...)
 }
 
 // NewFullNode creates a new instance of a FullNode providing a default config
-// and a mockstore to the NewFullNodeWithStore method
+// and a mockstore to the NewNodeWithStore method
 func (s *Swamp) NewFullNode(options ...node.Option) *node.Node {
 	cfg := node.DefaultConfig(node.Full)
 	store := node.MockStore(s.t, cfg)
 
-	return s.NewFullNodeWithStore(store, options...)
+	return s.NewNodeWithStore(node.Full, store, options...)
 }
 
 // NewLightNode creates a new instance of a LightNode providing a default config
-// and a mockstore to the NewLightNodeWithStore method
+// and a mockstore to the NewNodeWithStore method
 func (s *Swamp) NewLightNode(options ...node.Option) *node.Node {
 	cfg := node.DefaultConfig(node.Light)
 	store := node.MockStore(s.t, cfg)
 
-	return s.NewLightNodeWithStore(store, options...)
+	return s.NewNodeWithStore(node.Light, store, options...)
 }
 
-// NewBridgeNodeWithStore creates a new instance of BridgeNodes with predefined Store.
-// Afterwards, the instance is stored in the swamp's BridgeNodes slice.
-func (s *Swamp) NewBridgeNodeWithStore(store node.Store, options ...node.Option) *node.Node {
-	ks, err := store.Keystore()
-	require.NoError(s.t, err)
-
-	// TODO(@Bidon15): If for some reason, we receive one of existing options
-	// like <core, host, hash> from the test case, we need to check them and not use
-	// default that are set here
-	options = append(options,
-		node.WithCoreClient(s.CoreClient),
-		node.WithHost(s.createPeer(ks)),
-		node.WithTrustedHash(s.trustedHash),
-	)
-
-	node, err := node.New(node.Bridge, store, options...)
-	require.NoError(s.t, err)
-	s.BridgeNodes = append(s.BridgeNodes, node)
-	return node
-}
+func (s *Swamp) NewNodeWithStore(t node.Type, store node.Store, options ...node.Option) *node.Node {
+	var n *node.Node
 
-// NewFullNodeWithStore creates a new instance of FullNode with predefined Store.
-// Afterwards, the instance is stored in the swamp's FullNodes slice
-func (s *Swamp) NewFullNodeWithStore(store node.Store, options ...node.Option) *node.Node {
-	ks, err := store.Keystore()
-	require.NoError(s.t, err)
-
-	// TODO(@Bidon15): If for some reason, we receive one of existing options
-	// like <core, host, hash> from the test case, we need to check them and not use
-	// default that are set here
-	options = append(options,
-		node.WithHost(s.createPeer(ks)),
-		node.WithTrustedHash(s.trustedHash),
-	)
+	switch t {
+	case node.Bridge:
+		options = append(options,
+			node.WithCoreClient(s.CoreClient),
+		)
+		n = s.newNode(node.Bridge, store, options...)
+		s.BridgeNodes = append(s.BridgeNodes, n)
+	case node.Full:
+		n = s.newNode(node.Full, store, options...)
+		s.FullNodes = append(s.FullNodes, n)
+	case node.Light:
+		n = s.newNode(node.Light, store, options...)
+		s.LightNodes = append(s.LightNodes, n)
+	}
 
-	node, err := node.New(node.Full, store, options...)
-	require.NoError(s.t, err)
-	s.FullNodes = append(s.FullNodes, node)
-	return node
+	return n
 }
 
-// NewLightNodeWithStore creates a new instance of LightNode with predefined Store.
-// Afterwards, the instance is stored in the swamp's LightNodes slice
-func (s *Swamp) NewLightNodeWithStore(store node.Store, options ...node.Option) *node.Node {
+func (s *Swamp) newNode(t node.Type, store node.Store, options ...node.Option) *node.Node {
 	ks, err := store.Keystore()
 	require.NoError(s.t, err)
 
@@ -259,9 +237,8 @@ func (s *Swamp) NewLightNodeWithStore(store node.Store, options ...node.Option)
 		node.WithTrustedHash(s.trustedHash),
 	)
 
-	node, err := node.New(node.Light, store, options...)
+	node, err := node.New(t, store, options...)
 	require.NoError(s.t, err)
-	s.LightNodes = append(s.LightNodes, node)
 
 	return node
 }
diff --git a/node/tests/sync_test.go b/node/tests/sync_test.go
index 904364d8..4334c114 100644
--- a/node/tests/sync_test.go
+++ b/node/tests/sync_test.go
@@ -29,7 +29,7 @@ func TestSyncLightWithBridge(t *testing.T) {
 
 	bridge := sw.NewBridgeNode()
 
-	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+	ctx, cancel := context.WithTimeout(context.Background(), 40*time.Second)
 	t.Cleanup(cancel)
 
 	sw.WaitTillHeight(ctx, 20)
@@ -76,7 +76,7 @@ func TestSyncStartStopLightWithBridge(t *testing.T) {
 
 	bridge := sw.NewBridgeNode()
 
-	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+	ctx, cancel := context.WithTimeout(context.Background(), 40*time.Second)
 	t.Cleanup(cancel)
 
 	sw.WaitTillHeight(ctx, 50)
@@ -93,7 +93,7 @@ func TestSyncStartStopLightWithBridge(t *testing.T) {
 	require.NoError(t, err)
 
 	store := node.MockStore(t, node.DefaultConfig(node.Light))
-	light := sw.NewLightNodeWithStore(store, node.WithTrustedPeer(addrs[0].String()))
+	light := sw.NewNodeWithStore(node.Light, store, node.WithTrustedPeer(addrs[0].String()))
 	require.NoError(t, light.Start(ctx))
 
 	h, err = light.HeaderServ.GetByHeight(ctx, 30)
@@ -104,7 +104,7 @@ func TestSyncStartStopLightWithBridge(t *testing.T) {
 	require.NoError(t, light.Stop(ctx))
 	require.NoError(t, sw.RemoveNode(light, node.Light))
 
-	light = sw.NewLightNodeWithStore(store, node.WithTrustedPeer(addrs[0].String()))
+	light = sw.NewNodeWithStore(node.Light, store, node.WithTrustedPeer(addrs[0].String()))
 	require.NoError(t, light.Start(ctx))
 
 	h, err = light.HeaderServ.GetByHeight(ctx, 40)
@@ -128,7 +128,7 @@ func TestSyncFullWithBridge(t *testing.T) {
 
 	bridge := sw.NewBridgeNode()
 
-	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+	ctx, cancel := context.WithTimeout(context.Background(), 40*time.Second)
 	t.Cleanup(cancel)
 
 	sw.WaitTillHeight(ctx, 20)
@@ -175,7 +175,7 @@ func TestSyncLightWithFull(t *testing.T) {
 
 	bridge := sw.NewBridgeNode()
 
-	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
 	t.Cleanup(cancel)
 
 	sw.WaitTillHeight(ctx, 20)