Repository: authzed/spicedb

Issue:
Using SPICEDB_DATASTORE_READ_REPLICA_CONN_URI and SPICEDB_DATASTORE_CONN_URI environment variables causes duplicate metrics error
### What platforms are affected?

linux

### What architectures are affected?

amd64

### What SpiceDB version are you using?

v1.43.0

### Steps to Reproduce

1. Run SpiceDB container with the following environment variables:
- SPICEDB_DATASTORE_READ_REPLICA_CONN_URI=read-replica endpoint  (user:password@tcp(read-only-host:3306)/database?tls=skip-verify&parseTime=true
- SPICEDB_DATASTORE_CONN_URI=write endpoint (user:password@tcp(write-host:3306)/database?tls=skip-verify&parseTime=true
2. Our spicedb config:
  containers:
    - args:
        - spicedb
        - serve
        - '--datastore-engine=mysql'
        - '--grpc-enabled=true'
        - '--http-enabled=true'
        - '--dispatch-cluster-enabled=true'
        - '--metrics-enabled=true'
        - '--otel-endpoint=datadog-statsd.monitoring.svc.cluster.local:4318'
        - '--otel-provider=otlphttp'
        - '--dispatch-upstream-addr=kubernetes://spicedb.spicedb:50053'
        - '--log-format=json'
        - '--otel-service-name=spicedb'
        - '--otel-insecure'
    - envs: (besides the datastore environment variables)
        - name: SPICEDB_GRPC_PRESHARED_KEY
          value: grpckey

We are running SpiceDB in EKS v1.31.1-eks-1b3e656


### Expected Result

``{"level":"info","time":"2025-08-05T08:16:34Z","message":"using mysql datastore engine"}``


### Actual Result

``{"level":"trace","time":"2025-08-03T21:48:27Z","message":"NewMySQLDatastore: unable to instrument connector: unable to register metric: duplicate metrics collector registration attempted"}``

``{"level":"error","error":"failed to create datastore: NewMySQLDatastore: unable to instrument connector. To view details of this error (that may contain sensitive information), please run with --log-level=trace``

Discussion:
@shaafekhan: Just an addendum to testing, it seems that the following setup also leads to the same error when using a docker compose setup. 

```
services:
  spicedb-engine-migrate:
    image: ghcr.io/authzed/spicedb:v1.43.0 #also tried 1.45.1
    command: migrate head
    environment:
      SPICEDB_GRPC_PRESHARED_KEY: somerandomkey
      SPICEDB_LOG_LEVEL: trace
      SPICEDB_DATASTORE_ENGINE: mysql 
      SPICEDB_DATASTORE_READ_REPLICA_CONN_URI: admin:password@tcp(database-ro.eu-west-1.rds.amazonaws.com:3306)/spicedb?tls=skip-verify&parseTime=true
      SPICEDB_DATASTORE_CONN_URI: admin:password@tcp(database.eu-west-1.rds.amazonaws.com:3306)/spicedb?tls=skip-verify&parseTime=true
  spicedb-engine:
    image: ghcr.io/authzed/spicedb:v1.43.0 #also tried 1.45.1
    restart: always
    ports:
    - "8080:8080"
    depends_on:
     - spicedb-engine-migrate
    command: serve
    environment:
      SPICEDB_GRPC_PRESHARED_KEY: somerandomkey
      SPICEDB_LOG_LEVEL: trace
      SPICEDB_DATASTORE_ENGINE: mysql 
      SPICEDB_DATASTORE_READ_REPLICA_CONN_URI: admin:password@tcp(database-ro.eu-west-1.rds.amazonaws.com:3306)/spicedb?tls=skip-verify&parseTime=true
      SPICEDB_DATASTORE_CONN_URI: admin:password@tcp(database.eu-west-1.rds.amazonaws.com:3306)/spicedb?tls=skip-verify&parseTime=true
```
@miparnisari: Funny, I was able to reproduce this error while working on something unrelated. See https://github.com/authzed/spicedb/pull/2609. When I run `TestExecuteRepair` I get `failed to create datastore: duplicate metrics collector registration attempted`.

The issue is this code https://github.com/authzed/spicedb/blob/a6224a7f0192b7245a0c4c5900fa2c02f33ff412/internal/datastore/postgres/postgres.go#L249-L254 which runs once per replica.

And I think MySQL has the same problem: https://github.com/authzed/spicedb/blob/a6224a7f0192b7245a0c4c5900fa2c02f33ff412/internal/datastore/mysql/datastore.go#L190

In fact we can see here that this code isn't covered by tests :) https://app.codecov.io/gh/authzed/spicedb/blob/main/pkg%2Fcmd%2Fdatastore%2Fdatastore.go#L725@miparnisari: Hello again @shaafekhan. I've opened a PR to fix this: https://github.com/authzed/spicedb/pull/2707.

Solution patch:
diff --git a/internal/datastore/common/gc.go b/internal/datastore/common/gc.go
index 7aba0c04..0ac844e2 100644
--- a/internal/datastore/common/gc.go
+++ b/internal/datastore/common/gc.go
@@ -60,21 +60,22 @@ var (
 )
 
 // RegisterGCMetrics registers garbage collection metrics to the default
-// registry.
-func RegisterGCMetrics() error {
-	for _, metric := range []prometheus.Collector{
+// registry and returns them (so that they be unregistered).
+func RegisterGCMetrics() ([]prometheus.Collector, error) {
+	collectors := []prometheus.Collector{
 		gcDurationHistogram,
 		gcRelationshipsCounter,
 		gcTransactionsCounter,
 		gcNamespacesCounter,
 		gcFailureCounter,
-	} {
+	}
+	for _, metric := range collectors {
 		if err := prometheus.Register(metric); err != nil {
-			return err
+			return nil, err
 		}
 	}
 
-	return nil
+	return collectors, nil
 }
 
 // GarbageCollectableDatastore represents a datastore supporting external
diff --git a/internal/datastore/crdb/crdb.go b/internal/datastore/crdb/crdb.go
index 97870910..4b69051b 100644
--- a/internal/datastore/crdb/crdb.go
+++ b/internal/datastore/crdb/crdb.go
@@ -213,22 +213,10 @@ func newCRDBDatastore(ctx context.Context, url string, options ...Option) (datas
 		return nil, common.RedactAndLogSensitiveConnString(ctx, errUnableToInstantiate, err, url)
 	}
 
-	if config.enablePrometheusStats {
-		if err := prometheus.Register(pgxpoolprometheus.NewCollector(ds.writePool, map[string]string{
-			"db_name":    "spicedb",
-			"pool_usage": "write",
-		})); err != nil {
-			ds.cancel()
-			return nil, err
-		}
-
-		if err := prometheus.Register(pgxpoolprometheus.NewCollector(ds.readPool, map[string]string{
-			"db_name":    "spicedb",
-			"pool_usage": "read",
-		})); err != nil {
-			ds.cancel()
-			return nil, err
-		}
+	err = ds.registerPrometheusCollectors(config.enablePrometheusStats)
+	if err != nil {
+		ds.cancel()
+		return nil, err
 	}
 
 	// TODO: this (and the GC startup that it's based on for mysql/pg) should
@@ -275,6 +263,7 @@ type crdbDatastore struct {
 
 	dburl                   string
 	readPool, writePool     *pool.RetryPool
+	collectors              []prometheus.Collector
 	watchBufferLength       uint16
 	watchBufferWriteTimeout time.Duration
 	watchConnectTimeout     time.Duration
@@ -475,6 +464,9 @@ func (cds *crdbDatastore) Close() error {
 	cds.cancel()
 	cds.readPool.Close()
 	cds.writePool.Close()
+	for _, collector := range cds.collectors {
+		_ = prometheus.Unregister(collector)
+	}
 	return nil
 }
 
@@ -640,3 +632,31 @@ func readClusterTTLNanos(ctx context.Context, conn pgxcommon.DBFuncQuerier) (int
 
 	return gcSeconds * 1_000_000_000, nil
 }
+
+func (cds *crdbDatastore) registerPrometheusCollectors(enablePrometheusStats bool) error {
+	if !enablePrometheusStats {
+		return nil
+	}
+
+	readCollector := pgxpoolprometheus.NewCollector(cds.writePool, map[string]string{
+		"db_name":    "spicedb",
+		"pool_usage": "read",
+	})
+
+	if err := prometheus.Register(readCollector); err != nil {
+		return err
+	}
+	cds.collectors = append(cds.collectors, readCollector)
+
+	writeCollector := pgxpoolprometheus.NewCollector(cds.readPool, map[string]string{
+		"db_name":    "spicedb",
+		"pool_usage": "write",
+	})
+
+	if err := prometheus.Register(writeCollector); err != nil {
+		return err
+	}
+	cds.collectors = append(cds.collectors, writeCollector)
+
+	return nil
+}
diff --git a/internal/datastore/mysql/connection.go b/internal/datastore/mysql/connection.go
index 10562ff6..7e24b068 100644
--- a/internal/datastore/mysql/connection.go
+++ b/internal/datastore/mysql/connection.go
@@ -12,27 +12,13 @@ import (
 	log "github.com/authzed/spicedb/internal/logging"
 )
 
-var (
-	connectHistogram = prometheus.NewHistogram(prometheus.HistogramOpts{
-		Namespace: "spicedb",
-		Subsystem: "datastore",
-		Name:      "mysql_connect_duration",
-		Help:      "distribution in seconds of time spent opening a new MySQL connection.",
-		Buckets:   []float64{0.01, 0.1, 0.5, 1, 5, 10, 25, 60, 120},
-	})
-	connectCount = prometheus.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "spicedb",
-		Subsystem: "datastore",
-		Name:      "mysql_connect_count_total",
-		Help:      "number of mysql connections opened.",
-	}, []string{"success"})
-)
-
 // instrumentedConnector wraps the default MySQL driver connector
 // to get metrics and tracing when creating a new connection
 type instrumentedConnector struct {
-	conn driver.Connector
-	drv  driver.Driver
+	conn             driver.Connector
+	drv              driver.Driver
+	connectHistogram prometheus.Histogram
+	connectCount     *prometheus.CounterVec
 }
 
 func (d *instrumentedConnector) Connect(ctx context.Context) (driver.Conn, error) {
@@ -41,11 +27,11 @@ func (d *instrumentedConnector) Connect(ctx context.Context) (driver.Conn, error
 
 	startTime := time.Now()
 	defer func() {
-		connectHistogram.Observe(time.Since(startTime).Seconds())
+		d.connectHistogram.Observe(time.Since(startTime).Seconds())
 	}()
 
 	conn, err := d.conn.Connect(ctx)
-	connectCount.WithLabelValues(strconv.FormatBool(err == nil)).Inc()
+	d.connectCount.WithLabelValues(strconv.FormatBool(err == nil)).Inc()
 	if err != nil {
 		span.RecordError(err)
 		log.Ctx(ctx).Error().Err(err).Msg("failed to open mysql connection")
@@ -59,21 +45,47 @@ func (d *instrumentedConnector) Driver() driver.Driver {
 	return d.drv
 }
 
-func instrumentConnector(c driver.Connector) (driver.Connector, error) {
+func instrumentConnector(c driver.Connector, replicaIndex string) (driver.Connector, []prometheus.Collector, error) {
+	var (
+		connectHistogram = prometheus.NewHistogram(prometheus.HistogramOpts{
+			Namespace: "spicedb",
+			Subsystem: "datastore",
+			Name:      "mysql_connect_duration",
+			ConstLabels: prometheus.Labels{
+				"replica": replicaIndex, // this is needed to avoid "duplicate metrics collector registration attempted"
+			},
+			Help:    "distribution in seconds of time spent opening a new MySQL connection.",
+			Buckets: []float64{0.01, 0.1, 0.5, 1, 5, 10, 25, 60, 120},
+		})
+		connectCount = prometheus.NewCounterVec(prometheus.CounterOpts{
+			Namespace: "spicedb",
+			Subsystem: "datastore",
+			Name:      "mysql_connect_count_total",
+			ConstLabels: prometheus.Labels{
+				"replica": replicaIndex, // this is needed to avoid "duplicate metrics collector registration attempted"
+			},
+			Help: "number of mysql connections opened.",
+		}, []string{"success"})
+	)
+
 	err := prometheus.Register(connectHistogram)
 	if err != nil {
-		return nil, fmt.Errorf("unable to register metric: %w", err)
+		return nil, nil, err
 	}
 
 	err = prometheus.Register(connectCount)
 	if err != nil {
-		return nil, fmt.Errorf("unable to register metric: %w", err)
+		return nil, nil, err
 	}
 
+	collectors := []prometheus.Collector{connectCount, connectHistogram}
+
 	return &instrumentedConnector{
-		conn: c,
-		drv:  c.Driver(),
-	}, nil
+		conn:             c,
+		drv:              c.Driver(),
+		connectHistogram: connectHistogram,
+		connectCount:     connectCount,
+	}, collectors, nil
 }
 
 type sessionVariableConnector struct {
diff --git a/internal/datastore/mysql/datastore.go b/internal/datastore/mysql/datastore.go
index a64cb99b..18d9056e 100644
--- a/internal/datastore/mysql/datastore.go
+++ b/internal/datastore/mysql/datastore.go
@@ -3,6 +3,7 @@ package mysql
 import (
 	"context"
 	"database/sql"
+	"database/sql/driver"
 	"errors"
 	"fmt"
 	"math"
@@ -174,31 +175,9 @@ func newMySQLDatastore(ctx context.Context, uri string, replicaIndex int, option
 		}
 	}
 
-	var db *sql.DB
-	if config.enablePrometheusStats {
-		connector, err = instrumentConnector(connector)
-		if err != nil {
-			return nil, common.RedactAndLogSensitiveConnString(ctx, "NewMySQLDatastore: unable to instrument connector", err, uri)
-		}
-
-		dbName := "spicedb"
-		if replicaIndex != primaryInstanceID {
-			dbName = fmt.Sprintf("spicedb_replica_%d", replicaIndex)
-		}
-
-		db = sql.OpenDB(connector)
-		collector := sqlstats.NewStatsCollector(dbName, db)
-		if err := prometheus.Register(collector); err != nil {
-			return nil, fmt.Errorf(errUnableToInstantiate, err)
-		}
-
-		if isPrimary {
-			if err := common.RegisterGCMetrics(); err != nil {
-				return nil, fmt.Errorf(errUnableToInstantiate, err)
-			}
-		}
-	} else {
-		db = sql.OpenDB(connector)
+	db, collectors, err := registerAndReturnPrometheusCollectors(replicaIndex, isPrimary, connector, config.enablePrometheusStats)
+	if err != nil {
+		return nil, err
 	}
 
 	db.SetConnMaxLifetime(config.connMaxLifetime)
@@ -275,6 +254,7 @@ func newMySQLDatastore(ctx context.Context, uri string, replicaIndex int, option
 		MigrationValidator:      common.NewMigrationValidator(headMigration, config.allowedMigrations),
 		db:                      db,
 		driver:                  driver,
+		collectors:              collectors,
 		url:                     uri,
 		revisionQuantization:    config.revisionQuantization,
 		gcWindow:                config.gcWindow,
@@ -496,6 +476,7 @@ type Datastore struct {
 	readTxOptions      *sql.TxOptions
 	url                string
 	analyzeBeforeStats bool
+	collectors         []prometheus.Collector
 
 	revisionQuantization    time.Duration
 	gcWindow                time.Duration
@@ -534,6 +515,9 @@ func (mds *Datastore) Close() error {
 			log.Error().Err(err).Msg("error waiting for garbage collector to shutdown")
 		}
 	}
+	for _, collector := range mds.collectors {
+		_ = prometheus.Unregister(collector)
+	}
 	return mds.db.Close()
 }
 
@@ -698,3 +682,35 @@ type debugLogger struct{}
 func (debugLogger) Print(v ...any) {
 	log.Logger.Debug().CallerSkipFrame(1).Str("datastore", "mysql").Msg(fmt.Sprint(v...))
 }
+
+func registerAndReturnPrometheusCollectors(replicaIndex int, isPrimary bool, connector driver.Connector, enablePrometheusStats bool) (*sql.DB, []prometheus.Collector, error) {
+	if !enablePrometheusStats {
+		return sql.OpenDB(connector), nil, nil
+	}
+
+	connector, collectors, err := instrumentConnector(connector, strconv.Itoa(replicaIndex))
+	if err != nil {
+		return nil, nil, err
+	}
+
+	dbName := "spicedb"
+	if replicaIndex != primaryInstanceID {
+		dbName = fmt.Sprintf("spicedb_replica_%d", replicaIndex)
+	}
+
+	db := sql.OpenDB(connector)
+	collector := sqlstats.NewStatsCollector(dbName, db)
+	if err := prometheus.Register(collector); err != nil {
+		return nil, nil, err
+	}
+	collectors = append(collectors, collector)
+
+	if isPrimary {
+		gcMetrics, err := common.RegisterGCMetrics()
+		if err != nil {
+			return nil, nil, err
+		}
+		collectors = append(collectors, gcMetrics...)
+	}
+	return db, collectors, nil
+}
diff --git a/internal/datastore/postgres/postgres.go b/internal/datastore/postgres/postgres.go
index 662808a4..f14a2515 100644
--- a/internal/datastore/postgres/postgres.go
+++ b/internal/datastore/postgres/postgres.go
@@ -240,31 +240,9 @@ func newPostgresDatastore(
 		}
 	}
 
-	if config.enablePrometheusStats {
-		replicaIndexStr := strconv.Itoa(replicaIndex)
-		dbname := "spicedb"
-		if replicaIndex != primaryInstanceID {
-			dbname = "spicedb_replica_" + replicaIndexStr
-		}
-
-		if err := prometheus.Register(pgxpoolprometheus.NewCollector(readPool, map[string]string{
-			"db_name":    dbname,
-			"pool_usage": "read",
-		})); err != nil {
-			return nil, err
-		}
-
-		if isPrimary {
-			if err := prometheus.Register(pgxpoolprometheus.NewCollector(writePool, map[string]string{
-				"db_name":    "spicedb",
-				"pool_usage": "write",
-			})); err != nil {
-				return nil, err
-			}
-			if err := common.RegisterGCMetrics(); err != nil {
-				return nil, err
-			}
-		}
+	collectors, err := registerAndReturnPrometheusCollectors(replicaIndex, isPrimary, readPool, writePool, config.enablePrometheusStats)
+	if err != nil {
+		return nil, err
 	}
 
 	headMigration, err := migrations.DatabaseMigrations.HeadRevision()
@@ -331,6 +309,7 @@ func newPostgresDatastore(
 		dburl:                   pgURL,
 		readPool:                pgxcommon.MustNewInterceptorPooler(readPool, config.queryInterceptor),
 		writePool:               nil, /* disabled by default */
+		collectors:              collectors,
 		watchBufferLength:       config.watchBufferLength,
 		watchBufferWriteTimeout: config.watchBufferWriteTimeout,
 		optimizedRevisionQuery:  revisionQuery,
@@ -397,6 +376,7 @@ type pgDatastore struct {
 
 	dburl                          string
 	readPool, writePool            pgxcommon.ConnPooler
+	collectors                     []prometheus.Collector
 	watchBufferLength              uint16
 	watchBufferWriteTimeout        time.Duration
 	optimizedRevisionQuery         string
@@ -662,7 +642,9 @@ func (pgd *pgDatastore) Close() error {
 	if pgd.writePool != nil {
 		pgd.writePool.Close()
 	}
-
+	for _, collector := range pgd.collectors {
+		prometheus.Unregister(collector)
+	}
 	return nil
 }
 
@@ -823,3 +805,45 @@ func currentlyLivingObjects(original sq.SelectBuilder) sq.SelectBuilder {
 }
 
 var _ datastore.Datastore = &pgDatastore{}
+
+func registerAndReturnPrometheusCollectors(replicaIndex int, isPrimary bool, readPool, writePool *pgxpool.Pool, enablePrometheusStats bool) ([]prometheus.Collector, error) {
+	collectors := []prometheus.Collector{}
+	if !enablePrometheusStats {
+		return collectors, nil
+	}
+
+	replicaIndexStr := strconv.Itoa(replicaIndex)
+	dbname := "spicedb"
+	if replicaIndex != primaryInstanceID {
+		dbname = "spicedb_replica_" + replicaIndexStr
+	}
+
+	readCollector := pgxpoolprometheus.NewCollector(readPool, map[string]string{
+		"db_name":    dbname,
+		"pool_usage": "read",
+	})
+	if err := prometheus.Register(readCollector); err != nil {
+		return collectors, err
+	}
+	collectors = append(collectors, readCollector)
+
+	if isPrimary {
+		writeCollector := pgxpoolprometheus.NewCollector(writePool, map[string]string{
+			"db_name":    "spicedb",
+			"pool_usage": "write",
+		})
+
+		if err := prometheus.Register(writeCollector); err != nil {
+			return collectors, nil
+		}
+		collectors = append(collectors, writeCollector)
+
+		gcCollectors, err := common.RegisterGCMetrics()
+		if err != nil {
+			return collectors, err
+		}
+		collectors = append(collectors, gcCollectors...)
+	}
+
+	return collectors, nil
+}
diff --git a/internal/datastore/postgres/revisions.go b/internal/datastore/postgres/revisions.go
index 6a03fc98..043c2938 100644
--- a/internal/datastore/postgres/revisions.go
+++ b/internal/datastore/postgres/revisions.go
@@ -404,9 +404,11 @@ func (pr postgresRevision) MarshalBinary() ([]byte, error) {
 		return nil, spiceerrors.MustBugf("could not safely cast snapshot xmax to int64: %v", err)
 	}
 	protoRevision := implv1.PostgresRevision{
-		Xmin:         pr.snapshot.xmin,
-		RelativeXmax: relativeXmax - xminInt,
-		RelativeXips: relativeXips,
+		Xmin:              pr.snapshot.xmin,
+		RelativeXmax:      relativeXmax - xminInt,
+		RelativeXips:      relativeXips,
+		OptionalTxid:      pr.optionalTxID.Uint64,
+		OptionalTimestamp: pr.optionalNanosTimestamp,
 	}
 
 	return protoRevision.MarshalVT()
diff --git a/pkg/cmd/datastore.go b/pkg/cmd/datastore.go
index 74970add..fcbb269e 100644
--- a/pkg/cmd/datastore.go
+++ b/pkg/cmd/datastore.go
@@ -70,6 +70,7 @@ func NewGCDatastoreCommand(programName string, cfg *datastore.Config) *cobra.Com
 			if err != nil {
 				return fmt.Errorf("failed to create datastore: %w", err)
 			}
+			defer ds.Close()
 
 			gcds := dspkg.UnwrapAs[common.GarbageCollectableDatastore](ds)
 			if gcds == nil {
@@ -109,6 +110,7 @@ func NewRepairDatastoreCommand(programName string, cfg *datastore.Config) *cobra
 			if err != nil {
 				return fmt.Errorf("failed to create datastore: %w", err)
 			}
+			defer ds.Close()
 
 			repairable := dspkg.UnwrapAs[dspkg.RepairableDatastore](ds)
 			if repairable == nil {



Test patch:
diff --git a/internal/datastore/mysql/datastore_test.go b/internal/datastore/mysql/datastore_test.go
index c7d49cd9..4ba2cc4e 100644
--- a/internal/datastore/mysql/datastore_test.go
+++ b/internal/datastore/mysql/datastore_test.go
@@ -74,6 +74,7 @@ var defaultOptions = []Option{
 	GCInterval(0 * time.Second),
 	DebugAnalyzeBeforeStatistics(),
 	OverrideLockWaitTimeout(1),
+	WithEnablePrometheusStats(true),
 }
 
 type datastoreTestFunc func(t *testing.T, ds datastore.Datastore)
@@ -103,7 +104,7 @@ func createMultiDatastoreTest(b testdatastore.RunningEngineForTest, tf multiData
 			ds, err := newMySQLDatastore(ctx, uri, primaryInstanceID, options...)
 			require.NoError(t, err)
 
-			ds2, err := newMySQLDatastore(ctx, uri, primaryInstanceID, options...)
+			ds2, err := newMySQLDatastore(ctx, uri, 1, options...)
 			require.NoError(t, err)
 
 			secondDS = ds2
@@ -137,12 +138,8 @@ func additionalMySQLTests(t *testing.T, b testdatastore.RunningEngineForTest) {
 	prometheus.DefaultGatherer = reg
 	prometheus.DefaultRegisterer = reg
 
-	t.Run("DatabaseSeeding", createDatastoreTest(b, DatabaseSeedingTest))
-	t.Run("PrometheusCollector", createDatastoreTest(
-		b,
-		PrometheusCollectorTest,
-		WithEnablePrometheusStats(true),
-	))
+	t.Run("DatabaseSeeding", createDatastoreTest(b, DatabaseSeedingTest, defaultOptions...))
+	t.Run("PrometheusCollector", createDatastoreTest(b, PrometheusCollectorTest, defaultOptions...))
 	t.Run("GarbageCollection", createDatastoreTest(b, GarbageCollectionTest, defaultOptions...))
 	t.Run("GarbageCollectionByTime", createDatastoreTest(b, GarbageCollectionByTimeTest, defaultOptions...))
 	t.Run("ChunkedGarbageCollection", createDatastoreTest(b, ChunkedGarbageCollectionTest, defaultOptions...))
@@ -779,6 +776,9 @@ func TestMySQLMigrations(t *testing.T) {
 
 	db := datastoreDB(t, false)
 	migrationDriver := migrations.NewMySQLDriverFromDB(db, "")
+	t.Cleanup(func() {
+		migrationDriver.Close(t.Context())
+	})
 
 	version, err := migrationDriver.Version(context.Background())
 	req.NoError(err)
diff --git a/internal/datastore/postgres/postgres_shared_test.go b/internal/datastore/postgres/postgres_shared_test.go
index 32a5f6d3..bea5589c 100644
--- a/internal/datastore/postgres/postgres_shared_test.go
+++ b/internal/datastore/postgres/postgres_shared_test.go
@@ -1215,7 +1215,7 @@ func ConcurrentRevisionWatchTest(t *testing.T, ds datastore.Datastore) {
 	require.Eventually(func() bool {
 		seenWatchRevisionsLock.Lock()
 		defer seenWatchRevisionsLock.Unlock()
-		return len(seenWatchRevisions) == 3 && seenWatchRevisions[len(seenWatchRevisions)-1].String() == afterRev.String()
+		return len(seenWatchRevisions) == 3 && seenWatchRevisions[len(seenWatchRevisions)-1].Equal(afterRev)
 	}, 2*time.Second, 5*time.Millisecond)
 }
 
diff --git a/internal/datastore/postgres/revisions_test.go b/internal/datastore/postgres/revisions_test.go
index 11eabf1c..105397fb 100644
--- a/internal/datastore/postgres/revisions_test.go
+++ b/internal/datastore/postgres/revisions_test.go
@@ -57,18 +57,22 @@ func TestRevisionSerDe(t *testing.T) {
 	}
 
 	testCases := []struct {
-		snapshot    pgSnapshot
-		expectedStr string
+		snapshot       pgSnapshot
+		expectedStr    string
+		optionalTxID   uint64
+		optionalNanoTS uint64
 	}{
-		{snap(0, 0), ""},
-		{snap(0, 5, 1), "EAUaAQE="},
-		{snap(1, 1), "CAE="},
-		{snap(1, 3, 1), "CAEQAhoBAA=="},
-		{snap(1, 2, 1), "CAEQARoBAA=="},
-		{snap(2, 2), "CAI="},
-		{snap(123, 123), "CHs="},
-		{snap(100, 150, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110), "CGQQMhoKAQIDBAUGBwgJCg=="},
-		{snap(maxSizeList[0], maxSizeList[len(maxSizeList)-1], maxSizeList...), "COv/////////fxATGhQAAQIDBAUGBwgJCgsMDQ4PEBESEw=="},
+		{snapshot: snap(0, 0), expectedStr: ""},
+		{snapshot: snap(0, 5, 1), expectedStr: "EAUaAQE="},
+		{snapshot: snap(1, 1), expectedStr: "CAE="},
+		{snapshot: snap(1, 3, 1), expectedStr: "CAEQAhoBAA=="},
+		{snapshot: snap(1, 2, 1), expectedStr: "CAEQARoBAA=="},
+		{snapshot: snap(2, 2), expectedStr: "CAI="},
+		{snapshot: snap(123, 123), expectedStr: "CHs="},
+		{snapshot: snap(100, 150, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110), expectedStr: "CGQQMhoKAQIDBAUGBwgJCg=="},
+		{snapshot: snap(maxSizeList[0], maxSizeList[len(maxSizeList)-1], maxSizeList...), expectedStr: "COv/////////fxATGhQAAQIDBAUGBwgJCgsMDQ4PEBESEw=="},
+		{snapshot: snap(1, 1), expectedStr: "CAEgASi4o46CjtKKvBg=", optionalTxID: 1, optionalNanoTS: 1763206055841731000},
+		{snapshot: snap(1, 3, 1), expectedStr: "CAEQAhoBACAKKIitk9OQ5Iq8GA==", optionalTxID: 10, optionalNanoTS: 1763206675023845000},
 	}
 
 	for _, tc := range testCases {
@@ -76,7 +80,11 @@ func TestRevisionSerDe(t *testing.T) {
 		t.Run(tc.snapshot.String(), func(t *testing.T) {
 			require := require.New(t)
 
-			rev := postgresRevision{snapshot: tc.snapshot}
+			rev := postgresRevision{
+				snapshot:               tc.snapshot,
+				optionalTxID:           xid8{Uint64: tc.optionalTxID, Valid: tc.optionalTxID > 0},
+				optionalNanosTimestamp: tc.optionalNanoTS,
+			}
 			serialized := rev.String()
 			require.Equal(tc.expectedStr, serialized)
 
diff --git a/internal/testserver/datastore/mysql.go b/internal/testserver/datastore/mysql.go
index 9e1bf5a8..c94af2b3 100644
--- a/internal/testserver/datastore/mysql.go
+++ b/internal/testserver/datastore/mysql.go
@@ -124,6 +124,7 @@ func (mb *mysqlTester) runMigrate(t testing.TB, dsn string) error {
 	if err != nil {
 		return fmt.Errorf("failed to create mysql driver: %w", err)
 	}
+	defer driver.Close(t.Context())
 
 	err = migrations.Manager.Run(context.Background(), driver, migrate.Head, migrate.LiveRun)
 	if err != nil {
diff --git a/pkg/schema/v2/testing/generator.go b/pkg/schema/v2/testing/generator.go
index 43549503..9f91a1f4 100644
--- a/pkg/schema/v2/testing/generator.go
+++ b/pkg/schema/v2/testing/generator.go
@@ -22,10 +22,14 @@ type RelationshipGenerator struct {
 	subjectTypeNames  []string
 }
 
+// See parsing.go for reference regexes. max length is 64. We subtract 4 due to "o_" and first and last character
+// expressed in the regex first and last segments.
+const objectExpr = "[a-z0-9_][a-z0-9_]{0,59}[a-z0-9]+"
+
 // GenerateRelationships generates an infinite sequence of relationships for the schema.
 // Relationships are randomly generated but valid according to the schema.
 func (rg *RelationshipGenerator) GenerateRelationships(t *rapid.T) iter.Seq[tuple.Relationship] {
-	rapidObjectString := rapid.StringMatching("o_\\w\\w\\w+")
+	rapidObjectString := rapid.StringMatching("o_" + objectExpr)
 
 	return func(yield func(tuple.Relationship) bool) {
 		for {
@@ -76,9 +80,9 @@ func (rg *RelationshipGenerator) GenerateRelationships(t *rapid.T) iter.Seq[tupl
 func CheckWithSchema(t *testing.T, handler func(t *rapid.T, schema *schema.Schema, relationshipGenerator RelationshipGenerator)) {
 	t.Helper()
 	rapid.Check(t, func(t *rapid.T) {
-		rapidRelationString := rapid.StringMatching("r_\\w\\w\\w+")
-		rapidPermissionString := rapid.StringMatching("p_\\w\\w\\w+")
-		rapidDefinitionString := rapid.StringMatching("d_\\w\\w\\w+")
+		rapidRelationString := rapid.StringMatching("r_" + objectExpr)
+		rapidPermissionString := rapid.StringMatching("p_" + objectExpr)
+		rapidDefinitionString := rapid.StringMatching("d_" + objectExpr)
 
 		builder := schema.NewSchemaBuilder()
 
diff --git a/pkg/schema/v2/testing/generator_test.go b/pkg/schema/v2/testing/generator_test.go
index 0801f4a0..b7ee3902 100644
--- a/pkg/schema/v2/testing/generator_test.go
+++ b/pkg/schema/v2/testing/generator_test.go
@@ -9,6 +9,7 @@ import (
 	"github.com/authzed/spicedb/pkg/schema/v2"
 	"github.com/authzed/spicedb/pkg/schemadsl/compiler"
 	"github.com/authzed/spicedb/pkg/schemadsl/generator"
+	"github.com/authzed/spicedb/pkg/tuple"
 )
 
 func TestExampleRunWithSchemaForTesting(t *testing.T) {
@@ -40,3 +41,19 @@ func TestExampleRunWithSchemaForTesting(t *testing.T) {
 		}
 	})
 }
+
+func TestGenerateRelationshipsConformsToRegex(t *testing.T) {
+	CheckWithSchema(t, func(t *rapid.T, schema *schema.Schema, relGenerator RelationshipGenerator) {
+		require.NotNil(t, schema)
+
+		counter := 0
+		total := 1000
+		for relationship := range relGenerator.GenerateRelationships(t) {
+			_ = tuple.MustParse(relationship.String())
+			counter++
+			if counter >= total {
+				break
+			}
+		}
+	})
+}