Repository: pingcap/tidb (Go)

Issue:
Lightning panic when host ec2 is stopped
## Bug Report

Please answer these questions before submitting your issue. Thanks!

### 1. Minimal reproduce step (Required)
1. Trigger a lightning import in DBaaS
2. While import is in progress, stop the host ec2 of the lighting pod

### 2. What did you expect to see? (Required)
Lightning should not panic

### 3. What did you see instead (Required)
Lightning panic, and it results in lightning pod can't be started after host ec2 is started.

lightning received terminated signal when ec2 was stopped
``` 
[2022/03/28 08:04:24.699 +00:00] [INFO] [main.go:49] ["got signal to exit"] [signal=terminated]
[2022/03/28 08:04:24.700 +00:00] [WARN] [restore.go:1123] ["stopping periodic actions"] [error="context canceled"]
```

then lightning panic
```
[2022/03/28 08:04:24.729 +00:00] [ERROR] [local.go:2078] ["split & scatter ranges failed"] [uuid=132c73ad-f7fe-550a-bf5f-6270674024c4] [error="context canceled"]
[2022/03/28 08:04:24.729 +00:00] [INFO] [pd.go:462] ["resume scheduler"] [schedulers="[balance-leader-scheduler,balance-hot-region-scheduler,balance-region-scheduler]"]
[2022/03/28 08:04:24.730 +00:00] [INFO] [pd.go:482] ["resume scheduler successful"] [scheduler=balance-leader-scheduler]
[2022/03/28 08:04:24.730 +00:00] [INFO] [pd.go:482] ["resume scheduler successful"] [scheduler=balance-hot-region-scheduler]
[2022/03/28 08:04:24.731 +00:00] [INFO] [pd.go:482] ["resume scheduler successful"] [scheduler=balance-region-scheduler]
[2022/03/28 08:04:24.731 +00:00] [INFO] [pd.go:573] ["restoring config"] [config="{\"enable-location-replacement\":\"true\",\"leader-schedule-limit\":4,\"max-merge-region-keys\":200000,\"max-merge-region-size\":20,\"max-pending-peer-count\":64,\"max-snapshot-count\":64,\"region-schedule-limit\":2048}"]
[2022/03/28 08:04:24.736 +00:00] [INFO] [restore.go:1375] ["add back PD leader&region schedulers"]
[2022/03/28 08:04:24.736 +00:00] [INFO] [restore.go:1378] ["cleanup task metas"]
[2022/03/28 08:04:24.736 +00:00] [INFO] [restore.go:1227] ["cancel periodic actions"] [do=true]
[2022/03/28 08:04:24.737 +00:00] [INFO] [restore.go:1681] ["switch import mode"] [mode=Normal]
[2022/03/28 08:04:24.740 +00:00] [INFO] [restore.go:459] ["task canceled"] [step=4] [error="restore table `gotpc3000`.`order_line` failed: context canceled"]
[2022/03/28 08:04:24.740 +00:00] [INFO] [restore.go:474] ["the whole procedure completed"] [takeTime=14m54.044151364s] []
[2022/03/28 08:04:25.348 +00:00] [INFO] [local.go:3271] ["compact sst"] [fileCount=6] [size=668405745] [count=10609615] [cost=7.652965611s] [file=/etc/endpoint/feace5ee-67f2-55c0-81b9-8990a337bc8c.sst/40d8e331-2577-4503-a59f-a441404be79d.sst]
[2022/03/28 08:04:25.348 +00:00] [INFO] [local.go:689] ["write data to local DB"] [size=668405745] [kvs=10609615] [files=1] [sstFileSize=70829138] [file=/etc/endpoint/feace5ee-67f2-55c0-81b9-8990a337bc8c.sst/40d8e331-2577-4503-a59f-a441404be79d.sst] [firstKey=7480000000000000495F698000000000000001038000000000000580038000000000000004038000000000000001038000000000000001] [lastKey=7480000000000000495F6980000000000000010380000000000006AD038000000000000006038000000000000BB803800000000000000B]
panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x160 pc=0x33a225a]

goroutine 11489 [running]:
github.com/cockroachdb/pebble.(*DB).Ingest(0x0, 0xc002803650, 0x1, 0x1, 0x0, 0xc0e0f03a88)
	/nfs/cache/mod/github.com/cockroachdb/pebble@v0.0.0-20210719141320-8c3bd06debb5/ingest.go:505 +0x3a
github.com/pingcap/tidb/br/pkg/lightning/backend/local.dbSSTIngester.ingest(0xc000708820, 0xc0005a4880, 0x1, 0x1, 0x7, 0x7)
	/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/lightning/backend/local/local.go:3298 +0x14c
github.com/pingcap/tidb/br/pkg/lightning/backend/local.(*File).ingestSSTs(0xc000708820, 0xc0005a4880, 0x1, 0x1, 0x0, 0x0)
	/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/lightning/backend/local/local.go:697 +0x6ec
github.com/pingcap/tidb/br/pkg/lightning/backend/local.(*File).batchIngestSSTs(0xc000708820, 0xc0005a4870, 0x1, 0x1, 0xc0415328c0, 0x36)
	/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/lightning/backend/local/local.go:667 +0x45a
github.com/pingcap/tidb/br/pkg/lightning/backend/local.(*File).ingestSSTLoop.func1(0xc000708820, 0xc0090d22d0, 0xc0014860a8, 0xc0008c8000, 0xc0090d22c8, 0xc0014860c0)
	/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/lightning/backend/local/local.go:478 +0x3bb
created by github.com/pingcap/tidb/br/pkg/lightning/backend/local.(*File).ingestSSTLoop
	/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/br/pkg/lightning/backend/local/local.go:446 +0x259

```
[lightning.log](https://github.com/pingcap/tidb/files/8367483/lightning.log)


### 4. What is your TiDB version? (Required)
[release-version=v5.3.1] [git-hash=459917c6f83df155edcc03f6ebde24942ff73b0e] [git-branch=heads/refs/tags/v5.3.1] [go-version=go1.16.4] [utc-build-time="2022-03-02 08:34:21"]




Discussion:
/unassign gozssky
/assign lichunzhu
test



Patch:
diff --git a/br/pkg/lightning/backend/local/engine.go b/br/pkg/lightning/backend/local/engine.go
index f14a9a06c8e0a..82ebc4c4c3e65 100644
--- a/br/pkg/lightning/backend/local/engine.go
+++ b/br/pkg/lightning/backend/local/engine.go
@@ -426,8 +426,15 @@ func getSizeProperties(logger log.Logger, db *pebble.DB, keyAdapter KeyAdapter)
 }
 
 func (e *Engine) getEngineFileSize() backend.EngineFileSize {
-	metrics := e.db.Metrics()
-	total := metrics.Total()
+	e.mutex.RLock()
+	db := e.db
+	e.mutex.RUnlock()
+
+	var total pebble.LevelMetrics
+	if db != nil {
+		metrics := db.Metrics()
+		total = metrics.Total()
+	}
 	var memSize int64
 	e.localWriters.Range(func(k, v interface{}) bool {
 		w := k.(*Writer)
@@ -524,7 +531,6 @@ func (e *Engine) ingestSSTLoop() {
 	for i := 0; i < concurrency; i++ {
 		e.wg.Add(1)
 		go func() {
-			defer e.wg.Done()
 			defer func() {
 				if e.ingestErr.Get() != nil {
 					seqLock.Lock()
@@ -534,6 +540,7 @@ func (e *Engine) ingestSSTLoop() {
 					flushQueue = flushQueue[:0]
 					seqLock.Unlock()
 				}
+				e.wg.Done()
 			}()
 			for {
 				select {
@@ -1471,5 +1478,8 @@ func (i dbSSTIngester) ingest(metas []*sstMeta) error {
 	for _, m := range metas {
 		paths = append(paths, m.path)
 	}
+	if i.e.db == nil {
+		return errorEngineClosed
+	}
 	return i.e.db.Ingest(paths)
 }


Test patch:
diff --git a/br/pkg/lightning/backend/local/engine_test.go b/br/pkg/lightning/backend/local/engine_test.go
new file mode 100644
index 0000000000000..d78aa29ee1c36
--- /dev/null
+++ b/br/pkg/lightning/backend/local/engine_test.go
@@ -0,0 +1,85 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package local
+
+import (
+	"context"
+	"fmt"
+	"math"
+	"os"
+	"path"
+	"path/filepath"
+	"testing"
+
+	"github.com/cockroachdb/pebble"
+	"github.com/cockroachdb/pebble/sstable"
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/require"
+
+	"github.com/pingcap/tidb/br/pkg/lightning/backend"
+)
+
+func TestIngestSSTWithClosedEngine(t *testing.T) {
+	dir := t.TempDir()
+	opt := &pebble.Options{
+		MemTableSize:             1024 * 1024,
+		MaxConcurrentCompactions: 16,
+		L0CompactionThreshold:    math.MaxInt32, // set to max try to disable compaction
+		L0StopWritesThreshold:    math.MaxInt32, // set to max try to disable compaction
+		DisableWAL:               true,
+		ReadOnly:                 false,
+	}
+	db, err := pebble.Open(filepath.Join(dir, "test"), opt)
+	require.NoError(t, err)
+	tmpPath := filepath.Join(dir, "test.sst")
+	err = os.Mkdir(tmpPath, 0o755)
+	require.NoError(t, err)
+
+	_, engineUUID := backend.MakeUUID("ww", 0)
+	engineCtx, cancel := context.WithCancel(context.Background())
+	f := &Engine{
+		db:           db,
+		UUID:         engineUUID,
+		sstDir:       tmpPath,
+		ctx:          engineCtx,
+		cancel:       cancel,
+		sstMetasChan: make(chan metaOrFlush, 64),
+		keyAdapter:   noopKeyAdapter{},
+	}
+	f.sstIngester = dbSSTIngester{e: f}
+	sstPath := path.Join(tmpPath, uuid.New().String()+".sst")
+	file, err := os.Create(sstPath)
+	require.NoError(t, err)
+	w := sstable.NewWriter(file, sstable.WriterOptions{})
+	for i := 0; i < 10; i++ {
+		require.NoError(t, w.Add(sstable.InternalKey{
+			Trailer: uint64(sstable.InternalKeyKindSet),
+			UserKey: []byte(fmt.Sprintf("key%d", i)),
+		}, nil))
+	}
+	require.NoError(t, w.Close())
+
+	require.NoError(t, f.ingestSSTs([]*sstMeta{
+		{
+			path: sstPath,
+		},
+	}))
+	require.NoError(t, f.Close())
+	require.ErrorIs(t, f.ingestSSTs([]*sstMeta{
+		{
+			path: sstPath,
+		},
+	}), errorEngineClosed)
+}