Repository: ClickHouse/ClickHouse
File: ci/jobs/scripts/clickhouse_proc.py (Python)

Code before:
        if self.minio_proc:
            # remove the webhook so it doesn't spam with errors once we stop ClickHouse
            Shell.check(
                "/mc admin config reset clickminio logger_webhook:ch_server_webhook",
                verbose=True,
            )
            Shell.check(
                "/mc admin config reset clickminio audit_webhook:ch_audit_webhook",
                verbose=True,
            )

        self._flush_system_logs()

        self.save_system_metadata_files_from_remote_database_disk()

        print("Terminate ClickHouse processes")

        Shell.check(f"ps -ef | grep  clickhouse")
        for proc, pid_file, pid, run_path in (
            (self.proc, self.pid_file, self.pid_0, self.run_path0),
            (self.proc_1, self.pid_file_replica_1, self.pid_1, self.run_path1),
            (self.proc_2, self.pid_file_replica_2, self.pid_2, self.run_path2),
        ):
            if proc and pid:
                if not Shell.check(
                    f"cd {run_path} && clickhouse stop --pid-path {Path(pid_file).parent} --max-tries 5 --do-not-kill >/dev/null",
                    verbose=True,
                ):
                    print(
                        "Failed to stop ClickHouse process gracefully - send ABRT signal to generate core file"
                    )
                    Shell.check(f"kill -ABRT {pid}")

        return self

    @staticmethod
    def _chmod(files):
        for file in files:
            try:
                os.chmod(file, 0o666)
            except Exception as ex:
                print(f"WARNING: Failed to chmod {file}: {ex}")

    def prepare_logs(self, info, all=False):
        res = []
        try:
            res = self._get_logs_archives_server()
            res += self._get_jemalloc_profiles()
            if Path(self.GDB_LOG).exists():
                res.append(self.GDB_LOG)
            if all:

Review comment (suggestion):
Reducing max-tries from 300 to 5 significantly decreases the grace period for ClickHouse to stop cleanly. The original setting allowed up to 5 minutes (300 tries * 1 second each) for graceful shutdown, while the new setting only allows 5 seconds. This aggressive timeout may cause ClickHouse to not shut down properly, potentially leading to data corruption or loss of in-flight operations. Additionally, redirecting output to /dev/null may hide important shutdown errors. Consider whether this timeout reduction is appropriate for all scenarios, especially under heavy load.
```suggestion
                    f"cd {run_path} && clickhouse stop --pid-path {Path(pid_file).parent} --max-tries 300 --do-not-kill",
```

Code after:
clickhouse-client --query "SELECT count() FROM test.visits"
"""
        if with_s3_storage:
            command = "USE_S3_STORAGE_FOR_MERGE_TREE=1\n" + command
        return Shell.check(command)

    def insert_system_zookeeper_config(self):
        for _ in range(10):
            res = Shell.check(
                f"clickhouse-client --query \"insert into system.zookeeper (name, path, value) values ('auxiliary_zookeeper2', '{temp_dir}/chroot/', '')\"",
                verbose=True,
            )
            time.sleep(1)
            if res:
                return True
        else:
            return False

    def run_fast_test(self, test=""):
        if Path(self.test_output_file).exists():
            Path(self.test_output_file).unlink()
        exit_code = Shell.run(self.fast_test_command.format(TEST=test), verbose=True)
        return exit_code == 0

    def terminate(self):
        if self.minio_proc:
            # remove the webhook so it doesn't spam with errors once we stop ClickHouse
            Shell.check(
                "/mc admin config reset clickminio logger_webhook:ch_server_webhook",
                verbose=True,
            )
            Shell.check(
                "/mc admin config reset clickminio audit_webhook:ch_audit_webhook",
                verbose=True,
            )

        if self.kafka_proc:
            print("Stopping Kafka broker")
            Shell.check("kafka-server-stop.sh", verbose=True)
            try:
                self.kafka_proc.wait(timeout=30)
            except subprocess.TimeoutExpired:
                self.kafka_proc.kill()

        self._flush_system_logs()

        self.save_system_metadata_files_from_remote_database_disk()

        print("Terminate ClickHouse processes")

        Shell.check(f"ps -ef | grep  clickhouse")