Repository: rqlite/rqlite (2025-07-24T15:46:26Z)

Issue:
Support providing backups in DELETE mode
Here is the current Backup documentation for rqlite:

"You can also access the rqlite API directly, via a HTTP GET request to the endpoint /db/backup. For example, using curl, and assuming the node is listening on localhost:4001, you could retrieve a backup as follows:

curl -s -XGET localhost:4001/db/backup -o bak.sqlite3
The backup copy returned by rqlite is always in [WAL](https://www.sqlite.org/wal.html) mode. If you wish, you can always change the backup copy to [DELETE mode using the SQLite shell](https://www.sqlite.org/pragma.html#pragma_journal_mode)."

Let's enhance backup so the following is accepted:

curl -s -XGET localhost:4001/db/backup?fmt=delete -o bak.sqlite3

If the fmt is delete (as shown above) the SQL backup file be in DELETE mode. This will require a) making a copy of the entire SQLite database and its associated WAL files, this copy will be somewhere on temp storage, and then changing it to DELETE mode (use the functions in the DB module to do this). Then stream that file out to the caller. Then delete the temp file. The backup request proto will need to change to support this new format.

Update the unit tests to test for this mode. You'll also need to enhance the HTTP layer to check for this query parameter.



Discussion:

Patch:
diff --git a/command/proto/command.pb.go b/command/proto/command.pb.go
index 4ec6ddaae..5d5335b44 100644
--- a/command/proto/command.pb.go
+++ b/command/proto/command.pb.go
@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.36.6
-// 	protoc        v3.6.1
+// 	protoc        v3.21.12
 // source: command.proto
 
 package proto
@@ -82,6 +82,7 @@ const (
 	BackupRequest_BACKUP_REQUEST_FORMAT_NONE   BackupRequest_Format = 0
 	BackupRequest_BACKUP_REQUEST_FORMAT_SQL    BackupRequest_Format = 1
 	BackupRequest_BACKUP_REQUEST_FORMAT_BINARY BackupRequest_Format = 2
+	BackupRequest_BACKUP_REQUEST_FORMAT_DELETE BackupRequest_Format = 3
 )
 
 // Enum value maps for BackupRequest_Format.
@@ -90,11 +91,13 @@ var (
 		0: "BACKUP_REQUEST_FORMAT_NONE",
 		1: "BACKUP_REQUEST_FORMAT_SQL",
 		2: "BACKUP_REQUEST_FORMAT_BINARY",
+		3: "BACKUP_REQUEST_FORMAT_DELETE",
 	}
 	BackupRequest_Format_value = map[string]int32{
 		"BACKUP_REQUEST_FORMAT_NONE":   0,
 		"BACKUP_REQUEST_FORMAT_SQL":    1,
 		"BACKUP_REQUEST_FORMAT_BINARY": 2,
+		"BACKUP_REQUEST_FORMAT_DELETE": 3,
 	}
 )
 
@@ -2047,16 +2050,17 @@ const file_command_proto_rawDesc = "" +
 	"\x01q\x18\x01 \x01(\v2\x12.command.QueryRowsH\x00R\x01q\x12&\n" +
 	"\x01e\x18\x02 \x01(\v2\x16.command.ExecuteResultH\x00R\x01e\x12\x16\n" +
 	"\x05error\x18\x03 \x01(\tH\x00R\x05errorB\b\n" +
-	"\x06result\"\xfd\x01\n" +
+	"\x06result\"\xa0\x02\n" +
 	"\rBackupRequest\x125\n" +
 	"\x06format\x18\x01 \x01(\x0e2\x1d.command.BackupRequest.FormatR\x06format\x12\x16\n" +
 	"\x06Leader\x18\x02 \x01(\bR\x06Leader\x12\x16\n" +
 	"\x06Vacuum\x18\x03 \x01(\bR\x06Vacuum\x12\x1a\n" +
-	"\bCompress\x18\x04 \x01(\bR\bCompress\"i\n" +
+	"\bCompress\x18\x04 \x01(\bR\bCompress\"\x8b\x01\n" +
 	"\x06Format\x12\x1e\n" +
 	"\x1aBACKUP_REQUEST_FORMAT_NONE\x10\x00\x12\x1d\n" +
 	"\x19BACKUP_REQUEST_FORMAT_SQL\x10\x01\x12 \n" +
-	"\x1cBACKUP_REQUEST_FORMAT_BINARY\x10\x02\"!\n" +
+	"\x1cBACKUP_REQUEST_FORMAT_BINARY\x10\x02\x12 \n" +
+	"\x1cBACKUP_REQUEST_FORMAT_DELETE\x10\x03\"!\n" +
 	"\vLoadRequest\x12\x12\n" +
 	"\x04data\x18\x01 \x01(\fR\x04data\"\x95\x01\n" +
 	"\x10LoadChunkRequest\x12\x1b\n" +
diff --git a/command/proto/command.proto b/command/proto/command.proto
index 197383e1b..dd7242cf3 100644
--- a/command/proto/command.proto
+++ b/command/proto/command.proto
@@ -88,6 +88,7 @@ message BackupRequest {
 		BACKUP_REQUEST_FORMAT_NONE = 0;
 		BACKUP_REQUEST_FORMAT_SQL = 1;
 		BACKUP_REQUEST_FORMAT_BINARY = 2;
+		BACKUP_REQUEST_FORMAT_DELETE = 3;
 	}
 	Format format = 1;
 	bool Leader = 2;
diff --git a/http/query_params.go b/http/query_params.go
index eaac0f586..2534907c7 100644
--- a/http/query_params.go
+++ b/http/query_params.go
@@ -188,6 +188,8 @@ func (qp QueryParams) BackupFormat() command.BackupRequest_Format {
 	switch f {
 	case "sql":
 		return command.BackupRequest_BACKUP_REQUEST_FORMAT_SQL
+	case "delete":
+		return command.BackupRequest_BACKUP_REQUEST_FORMAT_DELETE
 	default:
 		return command.BackupRequest_BACKUP_REQUEST_FORMAT_BINARY
 	}
diff --git a/store/store.go b/store/store.go
index a590acd4d..160195666 100644
--- a/store/store.go
+++ b/store/store.go
@@ -1592,6 +1592,39 @@ func (s *Store) Backup(br *proto.BackupRequest, dst io.Writer) (retErr error) {
 			ww = dstGz
 		}
 		return s.db.Dump(ww)
+	} else if br.Format == proto.BackupRequest_BACKUP_REQUEST_FORMAT_DELETE {
+		// Create a temporary database file in DELETE mode
+		tmpFD, err := createTemp(s.dbDir, backupScratchPattern)
+		if err != nil {
+			return err
+		}
+		defer os.Remove(tmpFD.Name())
+		defer tmpFD.Close()
+
+		// Copy the current database to the temporary file and convert to DELETE mode
+		if err := s.db.Backup(tmpFD.Name(), br.Vacuum); err != nil {
+			return err
+		}
+
+		// Re-open the temporary file for reading (to ensure all data is written)
+		tmpReadFD, err := os.Open(tmpFD.Name())
+		if err != nil {
+			return err
+		}
+		defer tmpReadFD.Close()
+
+		// Stream the DELETE mode database to the destination
+		if br.Compress {
+			dstGz, err := gzip.NewWriterLevel(dst, gzip.BestSpeed)
+			if err != nil {
+				return err
+			}
+			defer dstGz.Close()
+			_, err = io.Copy(dstGz, tmpReadFD)
+		} else {
+			_, err = io.Copy(dst, tmpReadFD)
+		}
+		return err
 	}
 	return ErrInvalidBackupFormat
 }


Test patch:
diff --git a/http/service_test.go b/http/service_test.go
index fda15991a..6b5610cb7 100644
--- a/http/service_test.go
+++ b/http/service_test.go
@@ -754,6 +754,41 @@ func Test_BackupFlagsInvalid(t *testing.T) {
 	}
 }
 
+func Test_BackupDeleteOK(t *testing.T) {
+	m := &MockStore{}
+	c := &mockClusterService{}
+	s := New("127.0.0.1:0", m, c, nil)
+	if err := s.Start(); err != nil {
+		t.Fatalf("failed to start service")
+	}
+	defer s.Close()
+
+	// Track that the backup function is called with DELETE format
+	var capturedRequest *command.BackupRequest
+	m.backupFn = func(br *command.BackupRequest, dst io.Writer) error {
+		capturedRequest = br
+		return nil
+	}
+
+	client := &http.Client{}
+	host := fmt.Sprintf("http://%s", s.Addr().String())
+	resp, err := client.Get(host + "/db/backup?fmt=delete")
+	if err != nil {
+		t.Fatalf("failed to make backup request")
+	}
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("failed to get expected StatusOK for backup, got %d", resp.StatusCode)
+	}
+
+	// Verify that the backup request has the DELETE format
+	if capturedRequest == nil {
+		t.Fatalf("backup function was not called")
+	}
+	if capturedRequest.Format != command.BackupRequest_BACKUP_REQUEST_FORMAT_DELETE {
+		t.Fatalf("expected DELETE format, got %v", capturedRequest.Format)
+	}
+}
+
 func Test_LoadOK(t *testing.T) {
 	m := &MockStore{
 		leaderAddr: "foo:1234",
diff --git a/store/store_test.go b/store/store_test.go
index cad66aab9..ca585859c 100644
--- a/store/store_test.go
+++ b/store/store_test.go
@@ -15,6 +15,7 @@ import (
 
 	"github.com/rqlite/rqlite/v8/command/encoding"
 	"github.com/rqlite/rqlite/v8/command/proto"
+	"github.com/rqlite/rqlite/v8/db"
 	"github.com/rqlite/rqlite/v8/internal/random"
 	"github.com/rqlite/rqlite/v8/internal/rarchive"
 	"github.com/rqlite/rqlite/v8/testdata/chinook"
@@ -1582,6 +1583,95 @@ COMMIT;
 	}
 }
 
+// Test_SingleNodeBackupDelete tests that requesting a DELETE-formatted
+// backup works as expected.
+func Test_SingleNodeBackupDelete(t *testing.T) {
+	s, ln := mustNewStore(t)
+	defer ln.Close()
+
+	if err := s.Open(); err != nil {
+		t.Fatalf("failed to open single-node store: %s", err.Error())
+	}
+	if err := s.Bootstrap(NewServer(s.ID(), s.Addr(), true)); err != nil {
+		t.Fatalf("failed to bootstrap single-node store: %s", err.Error())
+	}
+	defer s.Close(true)
+	if _, err := s.WaitForLeader(10 * time.Second); err != nil {
+		t.Fatalf("Error waiting for leader: %s", err)
+	}
+
+	dump := `PRAGMA foreign_keys=OFF;
+BEGIN TRANSACTION;
+CREATE TABLE foo (id integer not null primary key, name text);
+INSERT INTO "foo" VALUES(1,'fiona');
+COMMIT;
+`
+	_, _, err := s.Execute(executeRequestFromString(dump, false, false))
+	if err != nil {
+		t.Fatalf("failed to load simple dump: %s", err.Error())
+	}
+
+	// Test uncompressed DELETE format backup
+	f, err := os.CreateTemp("", "rqlite-baktest-delete-")
+	if err != nil {
+		t.Fatalf("Backup Failed: unable to create temp file, %s", err.Error())
+	}
+	defer os.Remove(f.Name())
+	defer f.Close()
+
+	if err := s.Backup(backupRequestDelete(true, false, false), f); err != nil {
+		t.Fatalf("DELETE backup failed %s", err.Error())
+	}
+
+	// Check that the backup file is a valid SQLite database
+	f.Seek(0, 0)
+	header := make([]byte, 20) // Need at least 20 bytes for DELETE mode check
+	if _, err := f.Read(header); err != nil {
+		t.Fatalf("failed to read backup file header: %s", err.Error())
+	}
+	if !db.IsValidSQLiteData(header) {
+		t.Fatalf("backup file is not a valid SQLite database")
+	}
+
+	// Check that the backup file is in DELETE mode
+	if !db.IsDELETEModeEnabled(header) {
+		t.Fatalf("backup file is not in DELETE mode")
+	}
+
+	// Test compressed DELETE format backup
+	gzf, err := os.CreateTemp("", "rqlite-baktest-delete-gz-")
+	if err != nil {
+		t.Fatalf("Compressed backup Failed: unable to create temp file, %s", err.Error())
+	}
+	defer os.Remove(gzf.Name())
+	defer gzf.Close()
+
+	if err := s.Backup(backupRequestDelete(true, false, true), gzf); err != nil {
+		t.Fatalf("Compressed DELETE backup failed %s", err.Error())
+	}
+
+	// Verify the compressed backup can be decompressed and is valid
+	guzf, err := rarchive.Gunzip(gzf.Name())
+	if err != nil {
+		t.Fatalf("Failed to gunzip DELETE backup file %s", err.Error())
+	}
+	defer os.Remove(guzf)
+
+	guzData, err := os.ReadFile(guzf)
+	if err != nil {
+		t.Fatalf("failed to read gunzipped DELETE backup: %s", err.Error())
+	}
+	if len(guzData) < 20 {
+		t.Fatalf("gunzipped DELETE backup file is too short: %d bytes", len(guzData))
+	}
+	if !db.IsValidSQLiteData(guzData[:16]) {
+		t.Fatalf("gunzipped DELETE backup file is not a valid SQLite database")
+	}
+	if !db.IsDELETEModeEnabled(guzData[:20]) {
+		t.Fatalf("gunzipped DELETE backup file is not in DELETE mode")
+	}
+}
+
 // Test_SingleNodeSingleCommandTrigger tests that a SQLite trigger works.
 func Test_SingleNodeSingleCommandTrigger(t *testing.T) {
 	s, ln := mustNewStore(t)
@@ -3165,6 +3255,15 @@ func backupRequestBinary(leader, vacuum, compress bool) *proto.BackupRequest {
 	}
 }
 
+func backupRequestDelete(leader, vacuum, compress bool) *proto.BackupRequest {
+	return &proto.BackupRequest{
+		Format:   proto.BackupRequest_BACKUP_REQUEST_FORMAT_DELETE,
+		Leader:   leader,
+		Vacuum:   vacuum,
+		Compress: compress,
+	}
+}
+
 func loadRequestFromFile(path string) *proto.LoadRequest {
 	return &proto.LoadRequest{
 		Data: mustReadFile(path),