Repository: firecracker-microvm/firecracker
File: tests/framework/microvm.py (Python)

Code before:

            self._dump_debug_information(msg)

            raise

        # if microvm was spawned then check if it gets killed
        if self._spawned:
            # Wait until the Firecracker process is actually dead
            utils.wait_process_termination(self.firecracker_pid)

            # The following logic guards us against the case where `firecracker_pid` for some
            # reason is the wrong PID, e.g. this is a regression test for
            # https://github.com/firecracker-microvm/firecracker/pull/4442/commits/d63eb7a65ffaaae0409d15ed55d99ecbd29bc572

            # filter ps results for the jailer's unique id
            _, stdout, stderr = utils.run_cmd(
                f"ps ax -o pid,cmd -ww | grep {self.jailer.jailer_id}"
            )

            offenders = []
            for proc in stdout.splitlines():
                if "firecracker" in proc:
                    offenders.append(proc)

            # make sure firecracker was killed
            assert not stderr and not offenders, (
                f"Firecracker reported its pid {self.firecracker_pid}, which was killed, but there still exist processes using the supposedly dead Firecracker's jailer_id: \n"
                + "\n".join(offenders)
            )

        if self.uffd_handler and self.uffd_handler.is_running():
            self.uffd_handler.kill()

        # Mark the microVM as not spawned, so we avoid trying to kill twice.
        self._spawned = False
        self._killed = True

        if self.time_api_requests:
            self._validate_api_response_times()

        if self.memory_monitor:
            self.memory_monitor.check_samples()

    def _validate_api_response_times(self):
        """
        Parses the firecracker logs for information regarding api server request processing times, and asserts they
        are within acceptable bounds.
        """
        # Log messages are either
        # 2023-06-16T07:45:41.767987318 [fc44b23e-ce47-4635-9549-5779a6bd9cee:fc_api] The API server received a Get request on "/mmds".
        # or

Review comment (bug):
Since you're at it, could you split this in two assertions?:

```python
assert not stderr. "Error while looking for Firecracker process"
```

and 

```python
assert not offenders, "..."
```

so we know which of the two actually failed?

Code after:

            self._dump_debug_information(msg)

            raise

        # if microvm was spawned then check if it gets killed
        if self._spawned:
            # Wait until the Firecracker process is actually dead
            utils.wait_process_termination(self.firecracker_pid)

            # The following logic guards us against the case where `firecracker_pid` for some
            # reason is the wrong PID, e.g. this is a regression test for
            # https://github.com/firecracker-microvm/firecracker/pull/4442/commits/d63eb7a65ffaaae0409d15ed55d99ecbd29bc572

            # filter ps results for the jailer's unique id
            _, stdout, stderr = utils.run_cmd(
                f"ps ax -o pid,cmd -ww | grep {self.jailer.jailer_id}"
            )

            assert not stderr, f"error querying processes using `ps`: {stderr}"

            offenders = []
            for proc in stdout.splitlines():
                _, cmd = proc.lower().split(maxsplit=1)
                if "firecracker" in proc and not cmd.startswith("screen"):
                    offenders.append(proc)

            # make sure firecracker was killed
            assert not offenders, (
                f"Firecracker reported its pid {self.firecracker_pid}, which was killed, but there still exist processes using the supposedly dead Firecracker's jailer_id: \n"
                + "\n".join(offenders)
            )

        if self.uffd_handler and self.uffd_handler.is_running():
            self.uffd_handler.kill()

        # Mark the microVM as not spawned, so we avoid trying to kill twice.
        self._spawned = False
        self._killed = True

        if self.time_api_requests:
            self._validate_api_response_times()

        if self.memory_monitor:
            self.memory_monitor.check_samples()

    def _validate_api_response_times(self):
        """
        Parses the firecracker logs for information regarding api server request processing times, and asserts they
        are within acceptable bounds.
        """