Repository: neondatabase/autoscaling
Pull request #1359: neonvm: GZIP-compress base64 runner args
Author: login: sharnoff
Created: 2025-04-15T19:44:57Z | Merged: 2025-04-24T13:48:24Z

For `VirtualMachine` objects with a large amount of data in the spec, we see that the base64 encoded version, which is passed as an argument to `neonvm-runner`, massively contributes to the total size of the resulting pod object.

Because of the high rate of updates to pods, this can have additional large effects on the total amount of data stored in etcd.

Long-term I would like us to move away from packing the entire spec and status into the `neonvm-runner` args, but that's a much larger project.

See INC-492 for more: https://neondb.slack.com/archives/C08N5D2CF7A/p1744380652602419


Diff:
diff --git a/neonvm-runner/cmd/main.go b/neonvm-runner/cmd/main.go
index fb2ca3ce0..a49813268 100644
--- a/neonvm-runner/cmd/main.go
+++ b/neonvm-runner/cmd/main.go
@@ -33,6 +33,7 @@ import (
 
 	vmv1 "github.com/neondatabase/autoscaling/neonvm/apis/neonvm/v1"
 	"github.com/neondatabase/autoscaling/pkg/util"
+	"github.com/neondatabase/autoscaling/pkg/util/gzip64"
 	"github.com/neondatabase/autoscaling/pkg/util/taskgroup"
 )
 
@@ -127,9 +128,9 @@ func newConfig(logger *zap.Logger) *Config {
 		architecture:         runtime.GOARCH,
 	}
 	flag.StringVar(&cfg.vmSpecDump, "vmspec", cfg.vmSpecDump,
-		"Base64 encoded VirtualMachine json specification")
+		"Base64 gzip compressed VirtualMachine json specification")
 	flag.StringVar(&cfg.vmStatusDump, "vmstatus", cfg.vmStatusDump,
-		"Base64 encoded VirtualMachine json status")
+		"Base64 gzip compressed VirtualMachine json status")
 	flag.StringVar(&cfg.kernelPath, "kernelpath", cfg.kernelPath,
 		"Override path for kernel to use")
 	flag.StringVar(&cfg.appendKernelCmdline, "appendKernelCmdline",
@@ -165,11 +166,11 @@ func main() {
 func run(logger *zap.Logger) error {
 	cfg := newConfig(logger)
 
-	vmSpecJson, err := base64.StdEncoding.DecodeString(cfg.vmSpecDump)
+	vmSpecJson, err := gzip64.Decode(cfg.vmSpecDump)
 	if err != nil {
 		return fmt.Errorf("failed to decode VirtualMachine Spec dump: %w", err)
 	}
-	vmStatusJson, err := base64.StdEncoding.DecodeString(cfg.vmStatusDump)
+	vmStatusJson, err := gzip64.Decode(cfg.vmStatusDump)
 	if err != nil {
 		return fmt.Errorf("failed to decode VirtualMachine Status dump: %w", err)
 	}
diff --git a/pkg/neonvm/controllers/vm_controller.go b/pkg/neonvm/controllers/vm_controller.go
index db3afbe1c..f277614b7 100644
--- a/pkg/neonvm/controllers/vm_controller.go
+++ b/pkg/neonvm/controllers/vm_controller.go
@@ -21,7 +21,6 @@ import (
 	"crypto"
 	"crypto/ed25519"
 	"crypto/rand"
-	"encoding/base64"
 	"encoding/json"
 	"encoding/pem"
 	"errors"
@@ -55,6 +54,7 @@ import (
 	"github.com/neondatabase/autoscaling/pkg/api"
 	"github.com/neondatabase/autoscaling/pkg/neonvm/controllers/buildtag"
 	"github.com/neondatabase/autoscaling/pkg/neonvm/ipam"
+	"github.com/neondatabase/autoscaling/pkg/util/gzip64"
 	"github.com/neondatabase/autoscaling/pkg/util/patch"
 )
 
@@ -1399,8 +1399,8 @@ func podSpec(
 						// can get quite large)
 						cmd = append(
 							cmd,
-							"-vmspec", base64.StdEncoding.EncodeToString(vmSpecJson),
-							"-vmstatus", base64.StdEncoding.EncodeToString(vmStatusJson),
+							"-vmspec", gzip64.Encode(vmSpecJson),
+							"-vmstatus", gzip64.Encode(vmStatusJson),
 						)
 						// NB: We don't need to check if the value is nil because the default value
 						// was set in Reconcile
diff --git a/pkg/util/gzip64/gzip64.go b/pkg/util/gzip64/gzip64.go
new file mode 100644
index 000000000..a4d89b4be
--- /dev/null
+++ b/pkg/util/gzip64/gzip64.go
@@ -0,0 +1,51 @@
+package gzip64
+
+// Package for manipulating base64-encoded gzip'd data.
+
+import (
+	"bytes"
+	"compress/gzip"
+	"encoding/base64"
+	"fmt"
+	"io"
+	"strings"
+)
+
+// Encode returns the base64 string representing the gzip compressed data
+func Encode(data []byte) string {
+	buf := &bytes.Buffer{}
+
+	b64w := base64.NewEncoder(base64.StdEncoding, buf)
+	gzw := gzip.NewWriter(b64w)
+
+	if _, err := gzw.Write(data); err != nil {
+		panic(fmt.Sprintf("failed to write data: %s", err))
+	}
+	if err := gzw.Close(); err != nil {
+		panic(fmt.Sprintf("failed to close gzip writer: %s", err))
+	}
+	// gzip writer doesn't close the base64 encoder; we need to close that as well:
+	if err := b64w.Close(); err != nil {
+		panic(fmt.Sprintf("failed to close base64 encoder: %s", err))
+	}
+
+	return buf.String()
+}
+
+// Decode returns the inner data after base64 decoding and gzip decompression
+func Decode(data string) ([]byte, error) {
+	input := strings.NewReader(data)
+
+	b64r := base64.NewDecoder(base64.StdEncoding, input)
+	gzr, err := gzip.NewReader(b64r)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create gzip reader: %w", err)
+	}
+
+	output, err := io.ReadAll(gzr)
+	if err != nil {
+		return nil, err
+	}
+
+	return output, nil
+}
diff --git a/pkg/util/gzip64/gzip64_test.go b/pkg/util/gzip64/gzip64_test.go
new file mode 100644
index 000000000..529d8b38f
--- /dev/null
+++ b/pkg/util/gzip64/gzip64_test.go
@@ -0,0 +1,25 @@
+package gzip64_test
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/neondatabase/autoscaling/pkg/util/gzip64"
+)
+
+func TestBasicUsage(t *testing.T) {
+	exampleInput := []byte(strings.Repeat("test string! ", 100))
+	assert.Equal(t, 1300, len(exampleInput))
+
+	encoded := gzip64.Encode(exampleInput)
+	// Compressed string should be smaller:
+	assert.Less(t, len(encoded), len(exampleInput))
+
+	decoded, err := gzip64.Decode(encoded)
+	assert.NoError(t, err, "decode failed")
+
+	// result after decompressing should give the original string:
+	assert.Equal(t, exampleInput, decoded)
+}


Reviews:
user: login: olegbbtr
body: Some suggestions, but LGTM as it is.

Beyond that,

> Long-term I would like us to move away from packing the entire spec and status into the neonvm-runner args, but that's a much larger project.

Very naïvely, what's stopping us from creating a ConfigMap per compute, and mount it? Implementation should be fairly simple. However, the extra load on k8s can be a deal-breaker.

state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-15T23:01:48Z
user: login: sharnoff
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-16T18:41:35Z
user: login: sharnoff
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-16T18:41:40Z
user: login: AlexanderYastrebov
state: COMMENTED
author_association: NONE
submitted_at: 2025-04-19T18:44:26Z
user: login: olegbbtr
body: > WDYT about going the gzip route for now (as something small & safe), then trying the ConfigMap change as a follow-up?

Sounds good! My suggestion were for the follow-up.
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-24T13:12:54Z
user: login: sharnoff
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-24T13:25:22Z

Review comments:
diff_hunk: @@ -162,14 +163,36 @@ func main() {
 	}
 }
 
+func decodeGZIPBase64(input string) ([]byte, error) {
+	buf := bytes.Buffer{}
+
+	gz, err := gzip.NewReader(bytes.NewBuffer([]byte(input)))
+	if err != nil {
+		return nil, fmt.Errorf("failed to create gzip reader: %w", err)
+	}
+	if _, err := buf.ReadFrom(gz); err != nil {
path: neonvm-runner/cmd/main.go
user: login: olegbbtr
body: Can it be io.ReadAll?
created_at: 2025-04-15T22:44:52Z
author_association: CONTRIBUTOR
diff_hunk: @@ -162,14 +163,36 @@ func main() {
 	}
 }
 
+func decodeGZIPBase64(input string) ([]byte, error) {
path: neonvm-runner/cmd/main.go
user: login: olegbbtr
body: Consider putting this and `encodeGZIPBase64` together into something like `pkg/util/gzip64`. Will be used as `gzip64.Encode()`, looks like a nice interface.
created_at: 2025-04-15T22:51:59Z
author_association: CONTRIBUTOR
diff_hunk: @@ -162,14 +163,36 @@ func main() {
 	}
 }
 
+func decodeGZIPBase64(input string) ([]byte, error) {
+	buf := bytes.Buffer{}
+
+	gz, err := gzip.NewReader(bytes.NewBuffer([]byte(input)))
+	if err != nil {
+		return nil, fmt.Errorf("failed to create gzip reader: %w", err)
+	}
+	if _, err := buf.ReadFrom(gz); err != nil {
path: neonvm-runner/cmd/main.go
user: login: sharnoff
body: Good point, thanks -- made that change as part of 6f094c83d1d297a843f9e31230bdd3f4f4c4aeca
created_at: 2025-04-16T18:41:35Z
author_association: CONTRIBUTOR
diff_hunk: @@ -162,14 +163,36 @@ func main() {
 	}
 }
 
+func decodeGZIPBase64(input string) ([]byte, error) {
path: neonvm-runner/cmd/main.go
user: login: sharnoff
body: Makes sense — switched to that in 6f094c83d1d297a843f9e31230bdd3f4f4c4aeca
created_at: 2025-04-16T18:41:40Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,51 @@
+package gzip64
+
+// Package for manipulating base64-encoded gzip'd data.
+
+import (
+	"bytes"
+	"compress/gzip"
+	"encoding/base64"
+	"fmt"
+	"io"
+	"strings"
+)
+
+// Encode returns the base64 string representing the gzip compressed data
+func Encode(data []byte) string {
+	buf := &bytes.Buffer{}
+
+	b64w := base64.NewEncoder(base64.StdEncoding, buf)
+	gzw := gzip.NewWriter(b64w)
path: pkg/util/gzip64/gzip64.go
user: login: AlexanderYastrebov
body: Maybe use https://pkg.go.dev/compress/gzip#NewWriterLevel with https://pkg.go.dev/compress/gzip#BestCompression to buy a bit more time...
created_at: 2025-04-19T18:44:25Z
author_association: NONE
diff_hunk: @@ -0,0 +1,51 @@
+package gzip64
+
+// Package for manipulating base64-encoded gzip'd data.
+
+import (
+	"bytes"
+	"compress/gzip"
+	"encoding/base64"
+	"fmt"
+	"io"
+	"strings"
+)
+
+// Encode returns the base64 string representing the gzip compressed data
+func Encode(data []byte) string {
+	buf := &bytes.Buffer{}
+
+	b64w := base64.NewEncoder(base64.StdEncoding, buf)
+	gzw := gzip.NewWriter(b64w)
path: pkg/util/gzip64/gzip64.go
user: login: sharnoff
body: Unclear to me how much benefit this provides -- it's a good suggestion though, and we can look into it if we see a need. Thanks!
created_at: 2025-04-24T13:25:22Z
author_association: CONTRIBUTOR

PR comments:
user: login: sharnoff
created_at: 2025-04-22T10:32:44Z
body: backref: https://neondb.slack.com/archives/C08N5D2CF7A/p1745317936515369?thread_ts=1744380652.602419
author_association: CONTRIBUTOR
user: login: sharnoff
created_at: 2025-04-24T12:30:06Z
body: > Very naïvely, what's stopping us from creating a ConfigMap per compute, and mount it? Implementation should be fairly simple. However, the extra load on k8s can be a deal-breaker.

That's a good question ­— I hadn't considered it, mostly because I was focused on a smaller change within the existing strategy.

WDYT about going the gzip route for now (as something small & safe), then trying the ConfigMap change as a follow-up?
author_association: CONTRIBUTOR