Repository: bitcoin/bitcoin
File: ci/test/02_run_container.py (Python)

Code before:
                file.write(f"{k}={v}\n")
    run(["cat", env_file])

    if not os.getenv("DANGER_RUN_CI_ON_HOST"):
        CI_IMAGE_LABEL = "bitcoin-ci-test"

        # Use buildx unconditionally
        # Using buildx is required to properly load the correct driver, for use with registry caching. Neither build, nor BUILDKIT=1 currently do this properly
        cmd_build = ["docker", "buildx", "build"]
        cmd_build += [
            f"--file={os.environ['BASE_READ_ONLY_DIR']}/ci/test_imagefile",
            f"--build-arg=CI_IMAGE_NAME_TAG={os.environ['CI_IMAGE_NAME_TAG']}",
            f"--build-arg=FILE_ENV={os.environ['FILE_ENV']}",
            f"--build-arg=BASE_ROOT_DIR={os.environ['BASE_ROOT_DIR']}",
            f"--platform={os.environ['CI_IMAGE_PLATFORM']}",
            f"--label={CI_IMAGE_LABEL}",
            f"--tag={os.environ['CONTAINER_NAME']}",
        ]
        cmd_build += shlex.split(os.getenv("DOCKER_BUILD_CACHE_ARG", ""))
        cmd_build += [os.environ["BASE_READ_ONLY_DIR"]]

        print(f"Building {os.environ['CONTAINER_NAME']} image tag to run in")
        if run(cmd_build, check=False).returncode != 0:
            print(f"Retry building {os.environ['CONTAINER_NAME']} image tag after failure")
            time.sleep(3)
            run(cmd_build)

    run(["./ci/test/02_run_container.sh"])  # run the remainder


if __name__ == "__main__":
    main()


Review comment (bug):
if I remember correctly, whenever this error occurred, all concurrent CI builds failed so 3 seconds and a single retry may not suffice in this case - but I don't mind trying as such and add some exponential backoff in a future PR if this isn't enough (e.g. 5 runs with `2 ** retry_count` sleep).

Q: do we need to clean up anything after the first try to make sure the second call has more chance of success?

Code after:
            if k in settings:
                file.write(f"{k}={v}\n")
    run(["cat", env_file])

    if not os.getenv("DANGER_RUN_CI_ON_HOST"):
        CI_IMAGE_LABEL = "bitcoin-ci-test"

        # Use buildx unconditionally
        # Using buildx is required to properly load the correct driver, for use with registry caching. Neither build, nor BUILDKIT=1 currently do this properly
        cmd_build = ["docker", "buildx", "build"]
        cmd_build += [
            f"--file={os.environ['BASE_READ_ONLY_DIR']}/ci/test_imagefile",
            f"--build-arg=CI_IMAGE_NAME_TAG={os.environ['CI_IMAGE_NAME_TAG']}",
            f"--build-arg=FILE_ENV={os.environ['FILE_ENV']}",
            f"--build-arg=BASE_ROOT_DIR={os.environ['BASE_ROOT_DIR']}",
            f"--platform={os.environ['CI_IMAGE_PLATFORM']}",
            f"--label={CI_IMAGE_LABEL}",
            f"--tag={os.environ['CONTAINER_NAME']}",
        ]
        cmd_build += shlex.split(os.getenv("DOCKER_BUILD_CACHE_ARG", ""))
        cmd_build += [os.environ["BASE_READ_ONLY_DIR"]]

        print(f"Building {os.environ['CONTAINER_NAME']} image tag to run in")
        if run(cmd_build, check=False).returncode != 0:
            print(f"Retry building {os.environ['CONTAINER_NAME']} image tag after failure")
            time.sleep(3)
            run(cmd_build)

        for suffix in ["ccache", "depends", "depends_sources", "previous_releases"]:
            run(["docker", "volume", "create", f"{os.environ['CONTAINER_NAME']}_{suffix}"], check=False)

        CI_CCACHE_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_ccache,dst={os.environ['CCACHE_DIR']}"
        CI_DEPENDS_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_depends,dst={os.environ['DEPENDS_DIR']}/built"
        CI_DEPENDS_SOURCES_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_depends_sources,dst={os.environ['DEPENDS_DIR']}/sources"
        CI_PREVIOUS_RELEASES_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_previous_releases,dst={os.environ['PREVIOUS_RELEASES_DIR']}"
        CI_BUILD_MOUNT = []

        if os.getenv("DANGER_CI_ON_HOST_FOLDERS"):
            # ensure the directories exist
            for create_dir in [
                    os.environ["CCACHE_DIR"],
                    f"{os.environ['DEPENDS_DIR']}/built",
                    f"{os.environ['DEPENDS_DIR']}/sources",
                    os.environ["PREVIOUS_RELEASES_DIR"],
                    os.environ["BASE_BUILD_DIR"],  # Unset by default, must be defined externally
            ]:
                Path(create_dir).mkdir(parents=True, exist_ok=True)

            CI_CCACHE_MOUNT = f"type=bind,src={os.environ['CCACHE_DIR']},dst={os.environ['CCACHE_DIR']}"
            CI_DEPENDS_MOUNT = f"type=bind,src={os.environ['DEPENDS_DIR']}/built,dst={os.environ['DEPENDS_DIR']}/built"
            CI_DEPENDS_SOURCES_MOUNT = f"type=bind,src={os.environ['DEPENDS_DIR']}/sources,dst={os.environ['DEPENDS_DIR']}/sources"