Repository: pingcap/tiproxy
Pull request #771 (2025-04-24 14:37:39)

balance: add debug log for updating balance count
<!--

Thank you for contributing to TiProxy!

PR Title Format:
1. pkg [, pkg2, pkg3]: what's changed
2. *: what's changed

-->

### What problem does this PR solve?
<!--

Please create an issue first to describe the problem.

There MUST be one line starting with "Issue Number:  " and 
linking the relevant issues via the "close: #xxx" or "ref: #xxx".

For more info, check https://pingcap.github.io/tidb-dev-guide/contribute-to-tidb/contribute-code.html#referring-to-an-issue.

-->

Issue Number: ref #768 

Problem Summary:
Sometimes TiDB is unbalanced but it's hard to troubleshoot. Adding debug log will help.

What is changed and how it works:
Add some debug logs.

### Check List

Tests <!-- At least one of them must be included. -->

- [ ] Unit test
- [ ] Integration test
- [ ] Manual test (add detailed scripts or steps below)
- [x] No code

Only debug logs are added.

Notable changes

- [ ] Has configuration change
- [ ] Has HTTP API interfaces change
- [ ] Has tiproxyctl change
- [ ] Other user behavior changes

### Release note

<!-- compatibility change, improvement, bugfix, and new feature need a release note -->

Please refer to [Release Notes Language Style Guide](https://pingcap.github.io/tidb-dev-guide/contribute-to-tidb/release-notes-style-guide.html) to write a quality release note.

```release-note
- Add debug log for balancing factors.
```


codecov-commenter: ## [Codecov](https://app.codecov.io/gh/pingcap/tiproxy/pull/771?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=pingcap) Report
All modified and coverable lines are covered by tests :white_check_mark:
> Please [upload](https://docs.codecov.com/docs/codecov-uploader) report for BASE (`main@7747829`). [Learn more](https://docs.codecov.io/docs/error-reference?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=pingcap#section-missing-base-commit) about missing BASE report.

<details><summary>Additional details and impacted files</summary>


```diff
@@           Coverage Diff           @@
##             main     #771   +/-   ##
=======================================
  Coverage        ?   66.42%           
=======================================
  Files           ?      126           
  Lines           ?    11643           
  Branches        ?        0           
=======================================
  Hits            ?     7734           
  Misses          ?     3349           
  Partials        ?      560           
```

| [Flag](https://app.codecov.io/gh/pingcap/tiproxy/pull/771/flags?src=pr&el=flags&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=pingcap) | Coverage Δ | |
|---|---|---|
| [unit](https://app.codecov.io/gh/pingcap/tiproxy/pull/771/flags?src=pr&el=flag&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=pingcap) | `66.42% <100.00%> (?)` | |

Flags with carried forward coverage won't be shown. [Click here](https://docs.codecov.io/docs/carryforward-flags?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=pingcap#carryforward-flags-in-the-pull-request-comment) to find out more.

</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/pingcap/tiproxy/pull/771?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=pingcap).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=pingcap).

<details><summary> :rocket: New features to boost your workflow: </summary>

- :snowflake: [Test Analytics](https://docs.codecov.com/docs/test-analytics): Detect flaky tests, report on failures, and find test suite problems.
</details>
ti-chi-bot[bot]: [APPROVALNOTIFIER] This PR is **APPROVED**

This pull-request has been approved by: *<a href="https://github.com/pingcap/tiproxy/pull/771#pullrequestreview-2792940514" title="Approved">xhebox</a>*

The full list of commands accepted by this bot can be found [here](https://prow.tidb.net/command-help?repo=pingcap%2Ftiproxy).

The pull request process is described [here](https://book.prow.tidb.net/#/workflows/pr)

<details >
Needs approval from an approver in each of these files:

- ~~[OWNERS](https://github.com/pingcap/tiproxy/blob/main/OWNERS)~~ [xhebox]

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":[]} -->

Patch:
diff --git a/pkg/balance/factor/factor_balance.go b/pkg/balance/factor/factor_balance.go
index 530f5b9..84c0dd9 100644
--- a/pkg/balance/factor/factor_balance.go
+++ b/pkg/balance/factor/factor_balance.go
@@ -59,7 +59,7 @@ func (fbb *FactorBasedBalance) setFactors(cfg *config.Config) {
 	fbb.factors = fbb.factors[:0]
 
 	if fbb.factorStatus == nil {
-		fbb.factorStatus = NewFactorStatus()
+		fbb.factorStatus = NewFactorStatus(fbb.lg.Named("status"))
 	}
 	fbb.factors = append(fbb.factors, fbb.factorStatus)
 
@@ -79,13 +79,13 @@ func (fbb *FactorBasedBalance) setFactors(cfg *config.Config) {
 			fbb.factorLocation = NewFactorLocation()
 		}
 		if fbb.factorHealth == nil {
-			fbb.factorHealth = NewFactorHealth(fbb.mr)
+			fbb.factorHealth = NewFactorHealth(fbb.mr, fbb.lg.Named("health"))
 		}
 		if fbb.factorMemory == nil {
-			fbb.factorMemory = NewFactorMemory(fbb.mr)
+			fbb.factorMemory = NewFactorMemory(fbb.mr, fbb.lg.Named("memory"))
 		}
 		if fbb.factorCPU == nil {
-			fbb.factorCPU = NewFactorCPU(fbb.mr)
+			fbb.factorCPU = NewFactorCPU(fbb.mr, fbb.lg.Named("cpu"))
 		}
 	default:
 		if fbb.factorLocation != nil {
diff --git a/pkg/balance/factor/factor_cpu.go b/pkg/balance/factor/factor_cpu.go
index a4709d1..7d5a3b9 100644
--- a/pkg/balance/factor/factor_cpu.go
+++ b/pkg/balance/factor/factor_cpu.go
@@ -11,6 +11,7 @@ import (
 	"github.com/pingcap/tiproxy/pkg/balance/metricsreader"
 	dto "github.com/prometheus/client_model/go"
 	"github.com/prometheus/common/model"
+	"go.uber.org/zap"
 )
 
 const (
@@ -88,13 +89,15 @@ type FactorCPU struct {
 	usagePerConn float64
 	mr           metricsreader.MetricsReader
 	bitNum       int
+	lg           *zap.Logger
 }
 
-func NewFactorCPU(mr metricsreader.MetricsReader) *FactorCPU {
+func NewFactorCPU(mr metricsreader.MetricsReader, lg *zap.Logger) *FactorCPU {
 	fc := &FactorCPU{
 		mr:       mr,
 		bitNum:   5,
 		snapshot: make(map[string]cpuBackendSnapshot),
+		lg:       lg,
 	}
 	mr.AddQueryExpr(fc.Name(), cpuQueryExpr, cpuQueryRule)
 	return fc
@@ -216,6 +219,7 @@ func (fc *FactorCPU) updateCpuPerConn() {
 		} else {
 			fc.usagePerConn = usagePerConn
 		}
+		fc.lg.Debug("updateCpuPerConn", zap.Float64("usagePerConn", fc.usagePerConn), zap.Int("totalConns", totalConns), zap.Float64("totalUsage", totalUsage))
 	}
 	if fc.usagePerConn <= 0 {
 		fc.usagePerConn = minCpuPerConn
@@ -248,7 +252,12 @@ func (fc *FactorCPU) BalanceCount(from, to scoredBackend) float64 {
 	// E.g. 10% vs 25% don't need rebalance, but 80% vs 95% need rebalance.
 	// Use the average usage to avoid thrash when CPU jitters too much and use the latest usage to avoid migrate too many connections.
 	if 1.3-toAvgUsage > (1.3-fromAvgUsage)*cpuBalancedRatio && 1.3-toLatestUsage > (1.3-fromLatestUsage)*cpuBalancedRatio {
-		return 1 / fc.usagePerConn / balanceRatio4Cpu
+		balanceCount := 1 / fc.usagePerConn / balanceRatio4Cpu
+		fc.lg.Debug("update cpu balance", zap.String("from", from.Addr()), zap.String("to", to.Addr()), zap.Float64("fromAvgUsage", fromAvgUsage),
+			zap.Float64("fromLatestUsage", fromLatestUsage), zap.Float64("toAvgUsage", toAvgUsage), zap.Float64("toLatestUsage", toLatestUsage),
+			zap.Int("fromConnScore", from.ConnScore()), zap.Int("toConnScore", to.ConnScore()), zap.Float64("balanceCount", balanceCount),
+			zap.Float64("usagePerConn", fc.usagePerConn))
+		return balanceCount
 	}
 	return 0
 }
diff --git a/pkg/balance/factor/factor_health.go b/pkg/balance/factor/factor_health.go
index 436bb08..5ce1723 100644
--- a/pkg/balance/factor/factor_health.go
+++ b/pkg/balance/factor/factor_health.go
@@ -11,6 +11,7 @@ import (
 	"github.com/pingcap/tiproxy/pkg/balance/metricsreader"
 	dto "github.com/prometheus/client_model/go"
 	"github.com/prometheus/common/model"
+	"go.uber.org/zap"
 )
 
 const (
@@ -165,14 +166,16 @@ type FactorHealth struct {
 	indicators []errIndicator
 	mr         metricsreader.MetricsReader
 	bitNum     int
+	lg         *zap.Logger
 }
 
-func NewFactorHealth(mr metricsreader.MetricsReader) *FactorHealth {
+func NewFactorHealth(mr metricsreader.MetricsReader, lg *zap.Logger) *FactorHealth {
 	return &FactorHealth{
 		mr:         mr,
 		snapshot:   make(map[string]healthBackendSnapshot),
 		indicators: initErrIndicator(mr),
 		bitNum:     2,
+		lg:         lg,
 	}
 }
 
@@ -233,7 +236,7 @@ func (fh *FactorHealth) updateSnapshot(backends []scoredBackend) {
 	snapshots := make(map[string]healthBackendSnapshot, len(fh.snapshot))
 	for _, backend := range backends {
 		// Get the current value range.
-		updatedTime, valueRange := time.Time{}, valueRangeNormal
+		updatedTime, valueRange, indicator, value := time.Time{}, valueRangeNormal, "", 0
 		for i := 0; i < len(fh.indicators); i++ {
 			ts := fh.indicators[i].queryResult.UpdateTime
 			if time.Since(ts) > errMetricExpDuration {
@@ -247,6 +250,8 @@ func (fh *FactorHealth) updateSnapshot(backends []scoredBackend) {
 			vr := calcValueRange(sample, fh.indicators[i])
 			if vr > valueRange {
 				valueRange = vr
+				indicator = fh.indicators[i].key
+				value = int(sample.Value)
 			}
 		}
 		// If the metric is unavailable, try to reuse the latest one.
@@ -267,7 +272,11 @@ func (fh *FactorHealth) updateSnapshot(backends []scoredBackend) {
 				balanceCount = float64(backend.ConnScore()) / balanceSeconds4Health
 			}
 		}
-
+		// Log it whenever the health changes, even from unhealthy to healthy and no connections.
+		if balanceCount != snapshot.balanceCount {
+			fh.lg.Info("update health risk", zap.String("addr", addr), zap.String("indicator", indicator), zap.Int("value", value), zap.Int("lastRisk", int(snapshot.valueRange)),
+				zap.Int("curRisk", int(valueRange)), zap.Float64("balanceCount", balanceCount), zap.Int("connScore", backend.ConnScore()))
+		}
 		snapshots[addr] = healthBackendSnapshot{
 			updatedTime:  updatedTime,
 			valueRange:   valueRange,
diff --git a/pkg/balance/factor/factor_memory.go b/pkg/balance/factor/factor_memory.go
index ddee12e..ba5d967 100644
--- a/pkg/balance/factor/factor_memory.go
+++ b/pkg/balance/factor/factor_memory.go
@@ -11,6 +11,7 @@ import (
 	"github.com/pingcap/tiproxy/pkg/balance/metricsreader"
 	dto "github.com/prometheus/client_model/go"
 	"github.com/prometheus/common/model"
+	"go.uber.org/zap"
 )
 
 const (
@@ -100,9 +101,10 @@ type FactorMemory struct {
 	lastMetricTime time.Time
 	mr             metricsreader.MetricsReader
 	bitNum         int
+	lg             *zap.Logger
 }
 
-func NewFactorMemory(mr metricsreader.MetricsReader) *FactorMemory {
+func NewFactorMemory(mr metricsreader.MetricsReader, lg *zap.Logger) *FactorMemory {
 	bitNum := 0
 	for levels := len(oomRiskLevels); ; bitNum++ {
 		if levels == 0 {
@@ -114,6 +116,7 @@ func NewFactorMemory(mr metricsreader.MetricsReader) *FactorMemory {
 		mr:       mr,
 		bitNum:   bitNum,
 		snapshot: make(map[string]memBackendSnapshot),
+		lg:       lg,
 	}
 	mr.AddQueryExpr(fm.Name(), memQueryExpr, memoryQueryRule)
 	return fm
@@ -164,13 +167,22 @@ func (fm *FactorMemory) updateSnapshot(qr metricsreader.QueryResult, backends []
 			if !existsSnapshot || snapshot.updatedTime.Before(updateTime) {
 				latestUsage, timeToOOM := calcMemUsage(pairs)
 				if latestUsage >= 0 {
+					balanceCount := fm.calcBalanceCount(backend, latestUsage, timeToOOM)
 					snapshots[addr] = memBackendSnapshot{
 						updatedTime:  updateTime,
 						memUsage:     latestUsage,
 						timeToOOM:    timeToOOM,
-						balanceCount: fm.calcBalanceCount(backend, latestUsage, timeToOOM),
+						balanceCount: balanceCount,
 					}
 					valid = true
+					// Log it whenever the risk changes, even from high risk to no risk and no connections.
+					lastRisk, _ := getRiskLevel(snapshot.memUsage, snapshot.timeToOOM)
+					curRisk, _ := getRiskLevel(latestUsage, timeToOOM)
+					if lastRisk != curRisk {
+						fm.lg.Info("update memory risk", zap.String("addr", addr), zap.Float64("memUsage", latestUsage), zap.Duration("timeToOOM", timeToOOM),
+							zap.Int("lastRisk", lastRisk), zap.Float64("lastMemUsage", snapshot.memUsage), zap.Duration("lastTimeToOOM", snapshot.timeToOOM),
+							zap.Int("curRisk", curRisk), zap.Float64("balanceCount", balanceCount), zap.Int("connScore", backend.ConnScore()))
+					}
 				}
 			}
 		}
diff --git a/pkg/balance/factor/factor_status.go b/pkg/balance/factor/factor_status.go
index 21dbe44..adb1e51 100644
--- a/pkg/balance/factor/factor_status.go
+++ b/pkg/balance/factor/factor_status.go
@@ -3,7 +3,10 @@
 
 package factor
 
-import "github.com/pingcap/tiproxy/lib/config"
+import (
+	"github.com/pingcap/tiproxy/lib/config"
+	"go.uber.org/zap"
+)
 
 const (
 	// balanceSeconds4Status indicates the time (in seconds) to migrate all the connections.
@@ -24,12 +27,14 @@ var _ Factor = (*FactorStatus)(nil)
 type FactorStatus struct {
 	snapshot map[string]statusBackendSnapshot
 	bitNum   int
+	lg       *zap.Logger
 }
 
-func NewFactorStatus() *FactorStatus {
+func NewFactorStatus(lg *zap.Logger) *FactorStatus {
 	return &FactorStatus{
 		bitNum:   1,
 		snapshot: make(map[string]statusBackendSnapshot),
+		lg:       lg,
 	}
 }
 
@@ -59,6 +64,10 @@ func (fs *FactorStatus) updateSnapshot(backends []scoredBackend) {
 				balanceCount = snapshot.balanceCount
 			} else {
 				balanceCount = float64(backends[i].ConnScore()) / balanceSeconds4Status
+				// Do not log it when the balance counts are both 0.
+				if balanceCount != snapshot.balanceCount {
+					fs.lg.Info("update status risk", zap.String("addr", addr), zap.Float64("balanceCount", balanceCount), zap.Int("connScore", backends[i].ConnScore()))
+				}
 			}
 		}
 		snapshots[addr] = statusBackendSnapshot{


Test patch:
diff --git a/pkg/balance/factor/factor_cpu_test.go b/pkg/balance/factor/factor_cpu_test.go
index 3d90b01..a84c1c0 100644
--- a/pkg/balance/factor/factor_cpu_test.go
+++ b/pkg/balance/factor/factor_cpu_test.go
@@ -14,6 +14,7 @@ import (
 	"github.com/prometheus/common/expfmt"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
+	"go.uber.org/zap"
 )
 
 func TestCPUBalanceOnce(t *testing.T) {
@@ -89,7 +90,7 @@ func TestCPUBalanceOnce(t *testing.T) {
 				},
 			},
 		}
-		fc := NewFactorCPU(mmr)
+		fc := NewFactorCPU(mmr, zap.NewNop())
 		updateScore(fc, backends)
 		sortedIdx := make([]int, 0, len(test.cpus))
 		for _, backend := range backends {
@@ -186,7 +187,7 @@ func TestCPUBalanceContinuously(t *testing.T) {
 	}
 
 	mmr := newMockMetricsReader()
-	fc := NewFactorCPU(mmr)
+	fc := NewFactorCPU(mmr, zap.NewNop())
 	curTime := model.Now().Add(-10 * time.Second)
 	for i, test := range tests {
 		backends := make([]scoredBackend, 0, len(test.cpus))
@@ -242,7 +243,7 @@ func TestNoCPUMetric(t *testing.T) {
 		},
 	}
 	mmr := newMockMetricsReader()
-	fc := NewFactorCPU(mmr)
+	fc := NewFactorCPU(mmr, zap.NewNop())
 	backends := []scoredBackend{createBackend(0, 0, 0), createBackend(1, 0, 0)}
 	for i, test := range tests {
 		values := make([]*model.SampleStream, 0, len(test.cpus))
@@ -284,7 +285,7 @@ func TestCPUResultNotUpdated(t *testing.T) {
 	}
 
 	mmr := newMockMetricsReader()
-	fc := NewFactorCPU(mmr)
+	fc := NewFactorCPU(mmr, zap.NewNop())
 	backends := []scoredBackend{createBackend(0, 0, 0), createBackend(1, 0, 0)}
 	for i, test := range tests {
 		array := []float64{test.cpu}
diff --git a/pkg/balance/factor/factor_health_test.go b/pkg/balance/factor/factor_health_test.go
index c1610c2..9517bd3 100644
--- a/pkg/balance/factor/factor_health_test.go
+++ b/pkg/balance/factor/factor_health_test.go
@@ -14,6 +14,7 @@ import (
 	"github.com/prometheus/common/expfmt"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
+	"go.uber.org/zap"
 )
 
 func TestHealthScore(t *testing.T) {
@@ -75,7 +76,7 @@ func TestHealthScore(t *testing.T) {
 			},
 		},
 	}
-	fh := NewFactorHealth(mmr)
+	fh := NewFactorHealth(mmr, zap.NewNop())
 	fh.UpdateScore(backends)
 	for i, test := range tests {
 		require.Equal(t, test.score, backends[i].score(), "test index %d", i)
@@ -151,7 +152,7 @@ func TestHealthBalance(t *testing.T) {
 				},
 			},
 		}
-		fh := NewFactorHealth(mmr)
+		fh := NewFactorHealth(mmr, zap.NewNop())
 		fh.UpdateScore(backends)
 		scores := make([]uint64, 0, len(backends))
 		for _, backend := range backends {
@@ -186,7 +187,7 @@ func TestNoHealthMetrics(t *testing.T) {
 	}
 
 	mmr := newMockMetricsReader()
-	fh := NewFactorHealth(mmr)
+	fh := NewFactorHealth(mmr, zap.NewNop())
 	backends := make([]scoredBackend, 0, 2)
 	for i := 0; i < 2; i++ {
 		backends = append(backends, createBackend(i, i*100, i*100))
@@ -257,7 +258,7 @@ func TestHealthBalanceCount(t *testing.T) {
 	mmr := &mockMetricsReader{
 		qrs: map[string]metricsreader.QueryResult{},
 	}
-	fh := NewFactorHealth(mmr)
+	fh := NewFactorHealth(mmr, zap.NewNop())
 	updateMmr := func(healthy bool) {
 		number := 0.0
 		if !healthy {
diff --git a/pkg/balance/factor/factor_memory_test.go b/pkg/balance/factor/factor_memory_test.go
index 7587b6d..e4f42e8 100644
--- a/pkg/balance/factor/factor_memory_test.go
+++ b/pkg/balance/factor/factor_memory_test.go
@@ -14,6 +14,7 @@ import (
 	"github.com/prometheus/common/expfmt"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
+	"go.uber.org/zap"
 )
 
 func TestMemoryScore(t *testing.T) {
@@ -101,7 +102,7 @@ func TestMemoryScore(t *testing.T) {
 			},
 		},
 	}
-	fm := NewFactorMemory(mmr)
+	fm := NewFactorMemory(mmr, zap.NewNop())
 	fm.UpdateScore(backends)
 	for i, test := range tests {
 		require.Equal(t, test.score, backends[i].score(), "test index %d", i)
@@ -213,7 +214,7 @@ func TestMemoryBalance(t *testing.T) {
 				},
 			},
 		}
-		fm := NewFactorMemory(mmr)
+		fm := NewFactorMemory(mmr, zap.NewNop())
 		fm.UpdateScore(backends)
 		scores := make([]uint64, 0, len(backends))
 		for _, backend := range backends {
@@ -248,7 +249,7 @@ func TestNoMemMetrics(t *testing.T) {
 	}
 
 	mmr := newMockMetricsReader()
-	fm := NewFactorMemory(mmr)
+	fm := NewFactorMemory(mmr, zap.NewNop())
 	backends := make([]scoredBackend, 0, 2)
 	for i := 0; i < 2; i++ {
 		backends = append(backends, createBackend(i, i*100, i*100))
@@ -331,7 +332,7 @@ func TestMemoryBalanceCount(t *testing.T) {
 			Value:      model.Matrix(values),
 		}
 	}
-	fs := NewFactorMemory(mmr)
+	fs := NewFactorMemory(mmr, zap.NewNop())
 	backends := make([]scoredBackend, 0, 2)
 	backends = append(backends, createBackend(0, 0, 0))
 	backends = append(backends, createBackend(1, 0, 0))
diff --git a/pkg/balance/factor/factor_status_test.go b/pkg/balance/factor/factor_status_test.go
index be4c2ad..cfd7f92 100644
--- a/pkg/balance/factor/factor_status_test.go
+++ b/pkg/balance/factor/factor_status_test.go
@@ -7,6 +7,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
+	"go.uber.org/zap"
 )
 
 func TestFactorStatus(t *testing.T) {
@@ -24,7 +25,7 @@ func TestFactorStatus(t *testing.T) {
 		},
 	}
 
-	fs := NewFactorStatus()
+	fs := NewFactorStatus(zap.NewNop())
 	backends := make([]scoredBackend, 0, len(tests))
 	for _, test := range tests {
 		backend := scoredBackend{
@@ -72,7 +73,7 @@ func TestStatusBalanceCount(t *testing.T) {
 	backends = append(backends, createBackend(0, 0, 0))
 	backends = append(backends, createBackend(1, 0, 0))
 	unhealthyBackend := backends[0].BackendCtx.(*mockBackend)
-	fs := NewFactorStatus()
+	fs := NewFactorStatus(zap.NewNop())
 	for i, test := range tests {
 		unhealthyBackend.healthy = test.healthy
 		unhealthyBackend.connScore = test.conn