Change:
won't output logs after kubelet rotate log filer of the container. kubectl output logs correctly after the log file is rotated. the config to . So kubelet will rotate the log soon. Build a image with this program a deploy with the previous image. the command No response <details</details<detailsNone </details<details</details<details</details<details</details<details</details
/sig node
/area kubelet
/triage accepted /priority important-longterm
/assign since you sent a PR already
Ran into this issue!

Discussion:
found := true writer := newLogWriter(stdout, stderr, opts) msg := &logMessage{} <ins> baseName := filepath.Base(path) dir := filepath.Dir(path) </ins> for { if stop || (limitedMode && limitedNum == 0) { klog.V(2).InfoS("Finished parsing log file", "path", path)

return fmt.Errorf("failed to create fsnotify watcher: %v", err) } defer watcher.Close() <del> if err := watcher.Add(f.Name()); err != nil { return fmt.Errorf("failed to watch file %q: %v", f.Name(), err) </del> <ins> if err := watcher.Add(dir); err != nil { return fmt.Errorf("failed to watch directory %q: %w", dir, err) </ins> } // If we just created the watcher, try again to read as we might have missed // the event.

} var recreated bool // Wait until the next log change. <del> found, recreated, err = waitLogs(ctx, containerID, watcher, runtimeService) </del> <ins> found, recreated, err = waitLogs(ctx, containerID, baseName, watcher, runtimeService) </ins> if err != nil { return err }

} defer newF.Close() f.Close() <del> if err := watcher.Remove(f.Name()); err != nil && !os.IsNotExist(err) { klog.ErrorS(err, "Failed to remove file watch", "path", f.Name()) } </del> f = newF <del> if err := watcher.Add(f.Name()); err != nil { return fmt.Errorf("failed to watch file %q: %v", f.Name(), err) } </del> r = bufio.NewReader(f) } // If the container exited consume data until the next EOF

// waitLogs wait for the next log write. It returns two booleans and an error. The first boolean // indicates whether a new log is found; the second boolean if the log file was recreated; // the error is error happens during waiting new logs. <del> func waitLogs(ctx context.Context, id string, w *fsnotify.Watcher, runtimeService internalapi.RuntimeService) (bool, bool, error) { </del> <ins> func waitLogs(ctx context.Context, id string, logName string, w *fsnotify.Watcher, runtimeService internalapi.RuntimeService) (bool, bool, error) { </ins> // no need to wait if the pod is not running if running, err := isContainerRunning(ctx, id, runtimeService); !running { return false, false, err

return false, false, fmt.Errorf("context cancelled") case e := <-w.Events: switch e.Op { <del> case fsnotify.Write: </del> <ins> case fsnotify.Write, fsnotify.Rename, fsnotify.Remove, fsnotify.Chmod: </ins> return true, false, nil case fsnotify.Create: <del> fallthrough case fsnotify.Rename: fallthrough case fsnotify.Remove: fallthrough case fsnotify.Chmod: return true, true, nil </del> <ins> return true, filepath.Base(e.Name) == logName, nil </ins> default: klog.ErrorS(nil, "Received unexpected fsnotify event, retrying", "event", e) }

"io" "io/ioutil" "os" <ins> "path/filepath" </ins> "testing" "time" v1 "k8s.io/api/core/v1" apitesting "k8s.io/cri-api/pkg/apis/testing" <ins> "k8s.io/kubernetes/pkg/kubelet/types" </ins> "k8s.io/utils/pointer" "github.com/stretchr/testify/assert"

} } <ins> func TestReadRotatedLog(t *testing.T) { tmpDir := t.TempDir() file, err := os.CreateTemp(tmpDir, "logfile") if err != nil { assert.NoErrorf(t, err, "unable to create temp file") } stdoutBuf := &bytes.Buffer{} stderrBuf := &bytes.Buffer{} containerID := "fake-container-id" fakeRuntimeService := &apitesting.FakeRuntimeService{ Containers: map[string]*apitesting.FakeContainer{ containerID: { ContainerStatus: runtimeapi.ContainerStatus{ State: runtimeapi.ContainerState_CONTAINER_RUNNING, }, }, }, } ctx, cancel := context.WithCancel(context.Background()) defer cancel() // Start to follow the container's log. go func(ctx context.Context) { podLogOptions := v1.PodLogOptions{ Follow: true, } opts := NewLogOptions(&podLogOptions, time.Now()) ReadLogs(ctx, file.Name(), containerID, opts, fakeRuntimeService, stdoutBuf, stderrBuf) }(ctx) // log in stdout expectedStdout := "line0nline2nline4nline6nline8n" // log in stderr expectedStderr := "line1nline3nline5nline7nline9n" dir := filepath.Dir(file.Name()) baseName := filepath.Base(file.Name()) // Write 10 lines to log file. // Let ReadLogs start. time.Sleep(50 * time.Millisecond) for line := 0; line < 10; line++ { // Write the first three lines to log file now := time.Now().Format(types.RFC3339NanoLenient) if line%2 == 0 { file.WriteString(fmt.Sprintf( `{"log":"line%dn","stream":"stdout","time":"%s"}`+"n", line, now)) } else { file.WriteString(fmt.Sprintf( `{"log":"line%dn","stream":"stderr","time":"%s"}`+"n", line, now)) } time.Sleep(1 * time.Millisecond) if line == 5 { file.Close() // Pretend to rotate the log. rotatedName := fmt.Sprintf("%s.%s", baseName, time.Now().Format("220060102-150405")) rotatedName = filepath.Join(dir, rotatedName) if err := os.Rename(filepath.Join(dir, baseName), rotatedName); err != nil { assert.NoErrorf(t, err, "failed to rotate log %q to %q", file.Name(), rotatedName) return } newF := filepath.Join(dir, baseName) if file, err = os.Create(newF); err != nil { assert.NoError(t, err, "unable to create new log file") return } time.Sleep(20 * time.Millisecond) } } time.Sleep(20 * time.Millisecond) // Make the function ReadLogs end. fakeRuntimeService.Lock() fakeRuntimeService.Containers[containerID].State = runtimeapi.ContainerState_CONTAINER_EXITED fakeRuntimeService.Unlock() assert.Equal(t, expectedStdout, stdoutBuf.String()) assert.Equal(t, expectedStderr, stderrBuf.String()) } </ins> func TestParseLog(t *testing.T) { timestamp, err := time.Parse(timeFormatIn, "2016-10-20T18:39:20.57606443Z") assert.NoError(t, err)