Repository: ROCm/TheRock

Issue:
[Windows] Upload artifacts in build_windows_packages.yml
See https://github.com/ROCm/TheRock/blob/main/.github/workflows/build_linux_packages.yml and https://github.com/ROCm/TheRock/blob/main/.github/workflows/build_windows_packages.yml.

The Windows workflow should have the same "report" and "upload" steps. This might involve creating a [python?] script and sharing that script between both workflows.

Artifact uploading will enable test workflows to pull the artifacts from the build jobs and run tests. Users/developers will also be able to use these artifacts - see https://github.com/ROCm/TheRock/blob/main/RELEASES.md#from-per-commit-ci-builds.

We'll need to decide on naming / namespacing. Should Windows artifacts go in the existing `s3://therock-artifacts` bucket with Windows file names, or should they go in a new bucket like "therock-windows-artifacts"?

Discussion:
@ScottTodd: Anything left to do here @geomin12 ?

Looks like the artifact uploads are working well now. For example, https://github.com/ROCm/TheRock/actions/runs/15031073217#summary-42243505016 produced https://therock-artifacts.s3.us-east-2.amazonaws.com/15031073217-windows/index-gfx110X-dgpu.html@geomin12: This issue is complete! Windows upload works great

Solution patch:
diff --git a/.github/workflows/build_windows_packages.yml b/.github/workflows/build_windows_packages.yml
index 28f9fab..39e2f66 100644
--- a/.github/workflows/build_windows_packages.yml
+++ b/.github/workflows/build_windows_packages.yml
@@ -39,6 +39,7 @@ jobs:
       CCACHE_DIR: "${{github.workspace}}/.cache/ccache"
       CCACHE_MAXSIZE: "4000M"
       TEATIME_FORCE_INTERACTIVE: 0
+      AMDGPU_FAMILIES: ${{ inputs.amdgpu_families }}
     steps:
       - name: "Create build dir"
         shell: powershell
@@ -71,6 +72,8 @@ jobs:
           choco install --no-progress -y ninja
           choco install --no-progress -y strawberryperl
           echo "$PATH;C:\Strawberry\c\bin" >> $GITHUB_PATH
+          choco install --no-progress -y awscli
+          echo "$PATH;C:\Program Files\Amazon\AWSCLIV2" >> $GITHUB_PATH
 
       # After other installs, so MSVC get priority in the PATH.
       - name: Configure MSVC
@@ -155,7 +158,7 @@ jobs:
             ${extra_cmake_options}
 
       - name: Build Projects
-        run: cmake --build "${{ env.BUILD_DIR_BASH }}" -j $(nproc --all)
+        run: cmake --build "${{ env.BUILD_DIR_BASH }}" -j $(nproc --all) --target therock-archives therock-dist
 
       - name: Report
         if: ${{ !cancelled() }}
@@ -163,6 +166,12 @@ jobs:
           echo "Build dir:"
           echo "------------"
           ls -lh "${{ env.BUILD_DIR_BASH }}"
+          echo "Artifact Archives:"
+          echo "------------------"
+          ls -lh "${{ env.BUILD_DIR_BASH }}"/artifacts/*.tar.xz
+          echo "Artifacts:"
+          echo "----------"
+          du -h -d 1 "${{ env.BUILD_DIR_BASH }}"/artifacts
           echo "CCache Stats:"
           echo "-------------"
           ccache -s
@@ -176,9 +185,53 @@ jobs:
 
       - name: "Build size report"
         if: always()
+        shell: powershell
         run: |
           $fs = Get-PSDrive -PSProvider "FileSystem"
           $fsout = $fs | Select-Object -Property Name,Used,Free,Root
           $fsout | % {$_.Used/=1GB;$_.Free/=1GB;$_} | Write-Host
           get-disk | Select-object @{Name="Size(GB)";Expression={$_.Size/1GB}} | Write-Host
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 # v4.1.0
+        with:
+          aws-region: us-east-2
+          role-to-assume: arn:aws:iam::692859939525:role/therock-artifacts
+
+      # TODO: Move to script
+      - name: Create Index Files
+        run: |
+          curl --silent --fail --show-error --location \
+            https://raw.githubusercontent.com/joshbrunty/Indexer/6d8cbfd15d3853b482e6a49f2d875ded9188b721/indexer.py \
+            --output ${{ env.BUILD_DIR_BASH }}/indexer.py
+          python ${{ env.BUILD_DIR_BASH }}/indexer.py -f '*.tar.xz*' ${{ env.BUILD_DIR_BASH }}/artifacts/
+          python ${{ env.BUILD_DIR_BASH }}/indexer.py -f '*.log' ${{ env.BUILD_DIR_BASH }}/logs/
+          sed -i 's,a href="..",a href="../../index-${{env.AMDGPU_FAMILIES}}.html",g' ${{ env.BUILD_DIR_BASH }}/logs/index.html
+
+      # TODO: Move to script
+      - name: Upload Artifacts
         shell: powershell
+        run: |
+          $Env:PATH += ";C:\Program Files\Amazon\AWSCLIV2"
+          aws s3 cp ${{ env.BASE_BUILD_DIR_POWERSHELL }}\artifacts s3://therock-artifacts/${{github.run_id}}-windows/ `
+            --recursive --no-follow-symlinks `
+            --exclude "*" `
+            --include "*.tar.xz*"
+          aws s3 cp ${{ env.BASE_BUILD_DIR_POWERSHELL }}\artifacts\index.html s3://therock-artifacts/${{github.run_id}}-windows/index-${{env.AMDGPU_FAMILIES}}.html
+
+      # TODO: Move to script
+      - name: Upload Logs
+        shell: powershell
+        run: |
+          $Env:PATH += ";C:\Program Files\Amazon\AWSCLIV2"
+          aws s3 cp ${{ env.BASE_BUILD_DIR_POWERSHELL }}\logs s3://therock-artifacts/${{github.run_id}}-windows/logs/${{env.AMDGPU_FAMILIES}}/ `
+            --recursive `
+            --content-type="text/plain" `
+            --exclude "*" `
+            --include "*.log"
+          aws s3 cp ${{ env.BASE_BUILD_DIR_POWERSHELL }}\logs\index.html s3://therock-artifacts/${{github.run_id}}-windows/logs/${{env.AMDGPU_FAMILIES}}/
+
+      - name: Add Links to Job Summary
+        run: |
+          echo '* [Artifacts](https://therock-artifacts.s3.us-east-2.amazonaws.com/${{github.run_id}}-windows/index-${{env.AMDGPU_FAMILIES}}.html)' >> $GITHUB_STEP_SUMMARY
+          echo '* [Build logs](https://therock-artifacts.s3.us-east-2.amazonaws.com/${{github.run_id}}-windows/logs/${{env.AMDGPU_FAMILIES}}/index.html)' >> $GITHUB_STEP_SUMMARY
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 513abfd..a8de27c 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -92,14 +92,12 @@ jobs:
       fail-fast: false
       matrix:
         families: ${{ fromJSON(needs.setup.outputs.windows_amdgpu_families) }}
-        extra_cmake_options:
-          - "-DBUILD_TESTING=ON"
-          - "-DBUILD_TESTING=OFF"
     uses: ./.github/workflows/ci_windows.yml
     with:
       amdgpu_families: ${{ matrix.families.family }}
-      windows_use_prebuilt_artifacts: ${{ inputs.windows_use_prebuilt_artifacts == 'true' && 'true' || 'false' }}
-      extra_cmake_options: ${{ matrix.extra_cmake_options }}
+      windows_use_prebuilt_artifacts: ${{ inputs.windows_use_prebuilt_artifacts == true && 'true' || 'false' }}
+    permissions:
+      id-token: write
 
   # build_python_packages:
   #   name: Build Python Packages
diff --git a/.github/workflows/ci_windows.yml b/.github/workflows/ci_windows.yml
index 14bd0d7..46b0b1f 100644
--- a/.github/workflows/ci_windows.yml
+++ b/.github/workflows/ci_windows.yml
@@ -23,3 +23,5 @@ jobs:
     with:
       amdgpu_families: ${{ inputs.amdgpu_families }}
       extra_cmake_options: ${{ inputs.extra_cmake_options }}
+    permissions:
+      id-token: write
diff --git a/.github/workflows/publish_pytorch_dev_docker.yml b/.github/workflows/publish_pytorch_dev_docker.yml
index e09b4d3..ad5dd95 100644
--- a/.github/workflows/publish_pytorch_dev_docker.yml
+++ b/.github/workflows/publish_pytorch_dev_docker.yml
@@ -53,9 +53,6 @@ jobs:
           git config --global --add safe.directory $PWD
           git config fetch.parallel 10
 
-      - name: Generate unique dispatch ID
-        run: echo "DISPATCH_ID=dispatch-${{ matrix.targets.amdgpu_family }}-${{ github.run_id }}" >> $GITHUB_ENV
-
       - name: Fetch sources
         run: |
           python3 ./build_tools/fetch_sources.py --jobs 10
@@ -105,7 +102,7 @@ jobs:
       - name: Trigger verification workflow
         uses: benc-uk/workflow-dispatch@e2e5e9a103e331dad343f381a29e654aea3cf8fc #1.2.4
         with:
-          workflow: verify_docker_image.yml
+          workflow: test_pytorch_docker.yml
           inputs: |
             {
               "amdgpu_target": "${{ matrix.targets.amdgpu_family }}",
@@ -116,6 +113,5 @@ jobs:
                 (matrix.targets.amdgpu_family == 'gfx1100' && 'linux-rx7900-gpu-rocm') ||
                 'linux-mi300-1gpu-ossci-rocm'
               }}",
-              "dispatch_id": "${{ env.DISPATCH_ID }}",
               "version": "${{ env.VERSION }}"
             }
diff --git a/build_tools/github_action/configure_ci.py b/build_tools/github_action/configure_ci.py
index 16c8744..8dbf3d8 100755
--- a/build_tools/github_action/configure_ci.py
+++ b/build_tools/github_action/configure_ci.py
@@ -127,12 +127,12 @@ def check_for_non_skippable_path(paths: Optional[Iterable[str]]) -> bool:
     return any(not is_path_skippable(p) for p in paths)
 
 
-# TODO(#199): rename all of these to `ci_*.yml` so this is easier to understand?
 GITHUB_WORKFLOWS_CI_PATTERNS = [
-    "ci.yml",
     "setup.yml",
-    "build_*_packages.yml",
-    "test_*_packages.yml",
+    "ci*.yml",
+    "build*package*.yml",
+    "test*packages.yml",
+    "test*.yml",  # This may be too broad, but there are many test workflows.
 ]
 
 


Test patch:
diff --git a/.github/workflows/verify_docker_image.yml b/.github/workflows/test_pytorch_docker.yml
similarity index 87%
rename from .github/workflows/verify_docker_image.yml
rename to .github/workflows/test_pytorch_docker.yml
index b139857..3af632f 100644
--- a/.github/workflows/verify_docker_image.yml
+++ b/.github/workflows/test_pytorch_docker.yml
@@ -1,15 +1,15 @@
-name: Pull and Verify Docker Image
+name: Pull and Test Docker Image
 
 on:
   workflow_dispatch:
     inputs:
       amdgpu_target:
-        description: "AMDGPU target to verify"
+        description: "AMDGPU target to test"
         required: true
         default: "gfx942"
         type: string
       image_digest:
-        description: "Digest of image to verify"
+        description: "Digest of image to test"
         required: true
         type: string
       runner_type:
@@ -17,10 +17,6 @@ on:
         required: true
         default: "linux-mi300-1gpu-ossci-rocm"
         type: string
-      dispatch_id:
-        description: "Auto generated id for a dispatch"
-        type: string
-        required: true
       version:
         description: "Version number string"
         type: string
@@ -36,9 +32,6 @@ on:
       runner_type:
         type: string
         required: true
-      dispatch_id:
-        type: string
-        required: true
       version:
         type: string
         required: true
@@ -61,8 +54,8 @@ jobs:
       run: |
         echo "image_ref=$IMAGE_REF" >> "$GITHUB_OUTPUT"
 
-  pull-and-verify-image:
-    name: Verify ${{ inputs.version }}/${{ needs.setup_metadata.outputs.image_ref }} Docker Image
+  pull-and-test-image:
+    name: Test ${{ inputs.version }}/${{ needs.setup_metadata.outputs.image_ref }} Docker Image
     runs-on: ${{ inputs.runner_type }}
     needs: [setup_metadata]
     steps:
diff --git a/build_tools/github_action/configure_ci_test.py b/build_tools/github_action/configure_ci_test.py
index c668944..e59ba99 100644
--- a/build_tools/github_action/configure_ci_test.py
+++ b/build_tools/github_action/configure_ci_test.py
@@ -17,6 +17,12 @@ class ConfigureCITest(TestCase):
         paths = [".github/workflows/ci.yml"]
         run_ci = configure_ci.should_ci_run_given_modified_paths(paths)
         self.assertTrue(run_ci)
+        paths = [".github/workflows/build_package.yml"]
+        run_ci = configure_ci.should_ci_run_given_modified_paths(paths)
+        self.assertTrue(run_ci)
+        paths = [".github/workflows/test_some_subproject.yml"]
+        run_ci = configure_ci.should_ci_run_given_modified_paths(paths)
+        self.assertTrue(run_ci)
 
     def test_dont_run_ci_if_unrelated_workflow_file_edited(self):
         paths = [".github/workflows/publish_pytorch_dev_docker.yml"]