Change:
Pods should be run in parallel with checking the logs from the backend to fit into the timeout.

Discussion:
framework.ExpectNoError(err, "Fluentd deployed incorrectly") By("Running synthetic logger") <del> pod := createLoggingPod(f, podName, "", 10*60, 10*time.Minute) </del> <ins> pod := startNewLoggingPod(f, podName, "", 10*60, 10*time.Minute) </ins> defer f.PodClient().Delete(podName, &meta_v1.DeleteOptions{}) err = framework.WaitForPodNameRunningInNamespace(f.ClientSet, podName, f.Namespace.Name) framework.ExpectNoError(err, fmt.Sprintf("Should've successfully waited for pod %s to be running", podName))

By("Running pods to generate events while waiting for some of them to be ingested") wait.PollUntil(eventCreationInterval, func() (bool, error) { podName := "synthlogger" <del> createLoggingPod(f, podName, "", 1, 1*time.Second) </del> <ins> startNewLoggingPod(f, podName, "", 1, 1*time.Second) </ins> defer f.PodClient().Delete(podName, &meta_v1.DeleteOptions{}) err = framework.WaitForPodSuccessInNamespace(f.ClientSet, podName, f.Namespace.Name) if err != nil {

// considered acceptable. Once per hour is fine for now, as long as it // doesn't loose too much logs. maxAllowedRestartsPerHour = 1.0 <ins> // lastPodIngestionSlack is the amount of time to wait for the last pod's // logs to be ingested by the logging agent. lastPodIngestionSlack = 5 * time.Minute </ins> ) var _ = framework.KubeDescribe("Cluster level logging implemented by Stackdriver [Feature:StackdriverLogging] [Soak]", func() {

float64(time.Hour) * maxAllowedRestartsPerHour)) podRunDelay := time.Duration(int64(jobDuration) / int64(maxPodCount)) <del> podRunCount := int(testDuration.Seconds())/int(podRunDelay.Seconds()) - 1 </del> <ins> podRunCount := maxPodCount*(int(testDuration/jobDuration)-1) + 1 </ins> linesPerPod := linesPerPodPerSecond * int(jobDuration.Seconds()) <del> By("Running short-living pods") </del> pods := []*loggingPod{} for runIdx := 0; runIdx < podRunCount; runIdx++ { for nodeIdx, node := range nodes { podName := fmt.Sprintf("job-logs-generator-%d-%d-%d-%d", maxPodCount, linesPerPod, runIdx, nodeIdx) <del> pods = append(pods, createLoggingPod(f, podName, node.Name, linesPerPod, jobDuration)) defer f.PodClient().Delete(podName, &meta_v1.DeleteOptions{}) </del> <ins> pods = append(pods, newLoggingPod(podName, node.Name, linesPerPod, jobDuration)) </ins> } <del> time.Sleep(podRunDelay) </del> } <ins> By("Running short-living pods") go func() { for _, pod := range pods { pod.Start(f) time.Sleep(podRunDelay) defer f.PodClient().Delete(pod.Name, &meta_v1.DeleteOptions{}) } // Waiting until the last pod has completed time.Sleep(jobDuration - podRunDelay + lastPodIngestionSlack) }() </ins> By("Waiting for all log lines to be ingested") config := &loggingTestConfig{ LogsProvider:              sdLogsProvider,

logEntryMessageRegex = regexp.MustCompile("(?:Id+ d+:d+:d+.d+       d+ logs_generator.go:67] )?(d+) .*") ) <del> // Type to track the progress of logs generating pod type loggingPod struct { // Name of the pod Name string // Cache of ingested and read entries Occurrences map[int]logEntry // Number of lines expected to be ingested from this pod ExpectedLinesNumber int } </del> type logEntry struct { Payload string } <ins> func (entry logEntry) getLogEntryNumber() (int, bool) { submatch := logEntryMessageRegex.FindStringSubmatch(entry.Payload) if submatch == nil || len(submatch) < 2 { return 0, false } lineNumber, err := strconv.Atoi(submatch[1]) return lineNumber, err == nil } </ins> type logsProvider interface { Init() error Cleanup()

MaxAllowedFluentdRestarts int } <del> func (entry logEntry) getLogEntryNumber() (int, bool) { submatch := logEntryMessageRegex.FindStringSubmatch(entry.Payload) if submatch == nil || len(submatch) < 2 { return 0, false } lineNumber, err := strconv.Atoi(submatch[1]) return lineNumber, err == nil </del> <ins> // Type to track the progress of logs generating pod type loggingPod struct { // Name equals to the pod name and the container name. Name string // NodeName is the name of the node this pod will be // assigned to. Can be empty. NodeName string // Occurrences is a cache of ingested and read entries. Occurrences map[int]logEntry // ExpectedLinesNumber is the number of lines that are // expected to be ingested from this pod. ExpectedLinesNumber int // RunDuration is how long the pod will live. RunDuration time.Duration </ins> } <del> func createLoggingPod(f *framework.Framework, podName string, nodeName string, totalLines int, loggingDuration time.Duration) *loggingPod { framework.Logf("Starting pod %s", podName) createLogsGeneratorPod(f, podName, nodeName, totalLines, loggingDuration) </del> <ins> func newLoggingPod(podName string, nodeName string, totalLines int, loggingDuration time.Duration) *loggingPod { </ins> return &loggingPod{ Name:                podName, <ins> NodeName:            nodeName, </ins> Occurrences:         make(map[int]logEntry), ExpectedLinesNumber: totalLines, <ins> RunDuration:         loggingDuration, </ins> } } <del> func createLogsGeneratorPod(f *framework.Framework, podName string, nodeName string, linesCount int, duration time.Duration) { </del> <ins> func (p *loggingPod) Start(f *framework.Framework) { framework.Logf("Starting pod %s", p.Name) </ins> f.PodClient().Create(&api_v1.Pod{ ObjectMeta: meta_v1.ObjectMeta{ <del> Name: podName, </del> <ins> Name: p.Name, </ins> }, Spec: api_v1.PodSpec{ RestartPolicy: api_v1.RestartPolicyNever,

Env: []api_v1.EnvVar{ { Name:  "LOGS_GENERATOR_LINES_TOTAL", <del> Value: strconv.Itoa(linesCount), </del> <ins> Value: strconv.Itoa(p.ExpectedLinesNumber), </ins> }, { Name:  "LOGS_GENERATOR_DURATION", <del> Value: duration.String(), </del> <ins> Value: p.RunDuration.String(), </ins> }, }, Resources: api_v1.ResourceRequirements{

}, }, }, <del> NodeName: nodeName, </del> <ins> NodeName: p.NodeName, </ins> }, }) } <ins> func startNewLoggingPod(f *framework.Framework, podName string, nodeName string, totalLines int, loggingDuration time.Duration) *loggingPod { pod := newLoggingPod(podName, nodeName, totalLines, loggingDuration) pod.Start(f) return pod } </ins> func waitForSomeLogs(f *framework.Framework, config *loggingTestConfig) error { podHasIngestedLogs := make([]bool, len(config.Pods)) podWithIngestedLogsCount := 0