Repository: opensearch-project/opensearch-build

Issue:
[Bug]: Distribution builds can have path collisions
Distribution builds are not namespace'd based on the type of artifact produced so an OpenSearch and OpenSearch Dashboard build could collide.

### Actual
The current path generated is `https://ci.opensearch.org/ci/dbc/builds/1.1.0/328/x64/manifest.yml`.

### Expected
It should have the distribution type in the path like `https://ci.opensearch.org/ci/dbc/opensearch/builds/1.1.0/328/x64/manifest.yml` and `https://ci.opensearch.org/ci/dbc/opensearch-dashboards/builds/1.1.0/328/x64/manifest.yml`

### Required changes
- [x] Update the build jobs to include the build type in the url for the generated manifests, see [bundle_recorder.py](https://github.com/opensearch-project/opensearch-build/blob/main/src/assemble_workflow/bundle_recorder.py).
- [x] Update the jenkinsfile to upload to a destination based on the build type, see [Jenkinsfile](https://github.com/opensearch-project/opensearch-build/blob/main/Jenkinsfile#L127).

### Other changes
- [ ] Migrate/ update existing artifacts to the new destination
- [x] Update the [roles](https://github.com/opensearch-project/opensearch-build/blob/main/deployment/lib/identities.ts#L11) to be separate between jobs to prevent any possible conflicts, then update the jenkins jobs to build to the correct destination.
- [x] https://github.com/opensearch-project/opensearch-build/issues/661
- [x] https://github.com/opensearch-project/opensearch-build/issues/714



Discussion:
What is `dbc`, btw?
Let's make sure to address https://github.com/opensearch-project/opensearch-build/issues/661 when we change these.
Looks like https://github.com/opensearch-project/opensearch-build/issues/714 is related to this change too
Should we consider merging the output of build and assemble and renaming bundle as well? #692 and #479?

We could end up with a single folder that contains build output and bundle:

https://ci.opensearch.org/ci/dbc/opensearch/builds/1.1.0/328/x64/min/openseach-min-1.1.0.tar.gz (currently bundle/...)
https://ci.opensearch.org/ci/dbc/opensearch/builds/1.1.0/328/x64/dist/openseach-1.1.0.tar.gz (now separate under bundles)
https://ci.opensearch.org/ci/dbc/opensearch/builds/1.1.0/328/x64/maven/...


Resolving patch:
diff --git a/src/assemble_workflow/bundle_recorder.py b/src/assemble_workflow/bundle_recorder.py
--- a/src/assemble_workflow/bundle_recorder.py
+++ b/src/assemble_workflow/bundle_recorder.py
@@ -11,10 +11,10 @@
 
 
 class BundleRecorder:
-    def __init__(self, build, output_dir, artifacts_dir):
+    def __init__(self, build, output_dir, artifacts_dir, base_url):
         self.output_dir = output_dir
         self.build_id = build.id
-        self.public_url = os.getenv("PUBLIC_ARTIFACT_URL", None)
+        self.base_url = base_url
         self.version = build.version
         self.tar_name = self.__get_tar_name(build)
         self.artifacts_dir = artifacts_dir
@@ -38,18 +38,18 @@ def __get_tar_name(self, build):
         return "-".join(parts) + ".tar.gz"
 
     def __get_public_url_path(self, folder, rel_path):
-        path = "/".join((folder, self.version, self.build_id, self.architecture, rel_path))
-        return urljoin(self.public_url + "/", path)
+        path = "/".join((folder, rel_path))
+        return urljoin(self.base_url + "/", path)
 
     def __get_location(self, folder_name, rel_path, abs_path):
-        if self.public_url:
+        if self.base_url:
             return self.__get_public_url_path(folder_name, rel_path)
         return abs_path
 
     # Assembled bundles are expected to be served from a separate "bundles" folder
     # Example: https://artifacts.opensearch.org/bundles/1.0.0/<build-id
     def __get_tar_location(self):
-        return self.__get_location("bundles", self.tar_name, os.path.join(self.output_dir, self.tar_name))
+        return self.__get_location("dist", self.tar_name, os.path.join(self.output_dir, self.tar_name))
 
     # Build artifacts are expected to be served from a "builds" folder
     # Example: https://artifacts.opensearch.org/builds/1.0.0/<build-id>
diff --git a/src/run_assemble.py b/src/run_assemble.py
--- a/src/run_assemble.py
+++ b/src/run_assemble.py
@@ -30,6 +30,7 @@ def main():
         const=logging.DEBUG,
         dest="logging_level",
     )
+    parser.add_argument("-b", "--base-url", dest='base_url', help="The base url to download the artifacts.")
     args = parser.parse_args()
 
     console.configure(level=args.logging_level)
@@ -37,7 +38,7 @@ def main():
     build_manifest = BuildManifest.from_file(args.manifest)
     build = build_manifest.build
     artifacts_dir = os.path.dirname(os.path.realpath(args.manifest.name))
-    output_dir = os.path.join(os.getcwd(), "bundle")
+    output_dir = os.path.join(os.getcwd(), "dist")
     os.makedirs(output_dir, exist_ok=True)
 
     with TemporaryDirectory() as work_dir:
@@ -45,7 +46,7 @@ def main():
 
         os.chdir(work_dir.name)
 
-        bundle_recorder = BundleRecorder(build, output_dir, artifacts_dir)
+        bundle_recorder = BundleRecorder(build, output_dir, artifacts_dir, args.base_url)
 
         bundle = Bundles.create(build_manifest, artifacts_dir, bundle_recorder)
 
diff --git a/src/run_build.py b/src/run_build.py
--- a/src/run_build.py
+++ b/src/run_build.py
@@ -26,7 +26,7 @@ def main():
     manifest = InputManifest.from_file(args.manifest)
 
     with TemporaryDirectory(keep=args.keep) as work_dir:
-        output_dir = os.path.join(os.getcwd(), "artifacts")
+        output_dir = os.path.join(os.getcwd(), "builds")
 
         logging.info(f"Building in {work_dir.name}")
 


Test patch:
diff --git a/tests/test_run_assemble.py b/tests/test_run_assemble.py
--- a/tests/test_run_assemble.py
+++ b/tests/test_run_assemble.py
@@ -47,6 +47,6 @@ def test_main(self, mock_copy, mock_temp, mock_recorder, mock_bundles, *mocks):
         mock_bundle.install_min.assert_called()
         mock_bundle.install_plugins.assert_called()
 
-        mock_bundle.build_tar.assert_called_with(os.path.join("curdir", "bundle"))
+        mock_bundle.build_tar.assert_called_with(os.path.join("curdir", "dist"))
 
-        mock_recorder.return_value.write_manifest.assert_has_calls([call("path"), call(os.path.join("curdir", "bundle"))])  # manifest included in tar
+        mock_recorder.return_value.write_manifest.assert_has_calls([call("path"), call(os.path.join("curdir", "dist"))])  # manifest included in tar
diff --git a/tests/tests_assemble_workflow/test_bundle_recorder.py b/tests/tests_assemble_workflow/test_bundle_recorder.py
--- a/tests/tests_assemble_workflow/test_bundle_recorder.py
+++ b/tests/tests_assemble_workflow/test_bundle_recorder.py
@@ -20,7 +20,7 @@ def setUp(self):
         self.maxDiff = None
         manifest_path = os.path.join(os.path.dirname(__file__), "data", "opensearch-build-1.1.0.yml")
         manifest = BuildManifest.from_path(manifest_path)
-        self.bundle_recorder = BundleRecorder(manifest.build, "output_dir", "artifacts_dir")
+        self.bundle_recorder = BundleRecorder(manifest.build, "output_dir", "artifacts_dir", None)
 
     def test_record_component(self):
         component = BuildManifest.Component(
@@ -87,7 +87,7 @@ def test_write_manifest(self):
                 self.assertEqual(yaml.safe_load(f), data)
 
     def test_record_component_public(self):
-        self.bundle_recorder.public_url = "https://ci.opensearch.org/ci/os-distro-prod"
+        self.bundle_recorder.base_url = "https://ci.opensearch.org/ci/ci-env-prod/job-name-opensearch/1.2.0/build-123/platform-mac/arch-amd64/"
         component = BuildManifest.Component(
             {
                 "name": "job_scheduler",
@@ -114,7 +114,7 @@ def test_record_component_public(self):
                 "components": [
                     {
                         "commit_id": "3913d7097934cbfe1fdcf919347f22a597d00b76",
-                        "location": "https://ci.opensearch.org/ci/os-distro-prod/builds/1.1.0/c3ff7a232d25403fa8cc14c97799c323/x64/plugins",
+                        "location": "https://ci.opensearch.org/ci/ci-env-prod/job-name-opensearch/1.2.0/build-123/platform-mac/arch-amd64/builds/plugins",
                         "name": component.name,
                         "ref": "main",
                         "repository": "https://github.com/opensearch-project/job_scheduler",
@@ -125,8 +125,8 @@ def test_record_component_public(self):
         )
 
     def test_get_location_scenarios(self):
-        def get_location(public_url):
-            self.bundle_recorder.public_url = public_url
+        def get_location(base_url):
+            self.bundle_recorder.base_url = base_url
             return self.bundle_recorder._BundleRecorder__get_location("builds", "dir1/dir2/file", "/tmp/builds/foo/dir1/dir2/file")
 
         # No public URL - Fallback to ABS Path
@@ -134,14 +134,14 @@ def get_location(public_url):
 
         # Public URL - No trailing slash
         self.assertEqual(
-            get_location("https://ci.opensearch.org/ci/os-distro-prod"),
-            "https://ci.opensearch.org/ci/os-distro-prod/builds/1.1.0/c3ff7a232d25403fa8cc14c97799c323/x64/dir1/dir2/file",
+            get_location("https://ci.opensearch.org/ci/ci-env-prod/job-name-opensearch/1.2.0/build-123/platform-mac/arch-amd64"),
+            "https://ci.opensearch.org/ci/ci-env-prod/job-name-opensearch/1.2.0/build-123/platform-mac/arch-amd64/builds/dir1/dir2/file",
         )
 
         # Public URL - Trailing slash
         self.assertEqual(
-            get_location("https://ci.opensearch.org/ci/os-distro-prod/"),
-            "https://ci.opensearch.org/ci/os-distro-prod/builds/1.1.0/c3ff7a232d25403fa8cc14c97799c323/x64/dir1/dir2/file",
+            get_location("https://ci.opensearch.org/ci/ci-env-prod/job-name-opensearch/1.2.0/build-123/platform-mac/arch-amd64/"),
+            "https://ci.opensearch.org/ci/ci-env-prod/job-name-opensearch/1.2.0/build-123/platform-mac/arch-amd64/builds/dir1/dir2/file",
         )
 
     def test_tar_name(self):
@@ -152,7 +152,7 @@ class TestBundleRecorderDashboards(unittest.TestCase):
     def setUp(self):
         manifest_path = os.path.join(os.path.dirname(__file__), "data", "opensearch-dashboards-build-1.1.0.yml")
         manifest = BuildManifest.from_path(manifest_path)
-        self.bundle_recorder = BundleRecorder(manifest.build, "output_dir", "artifacts_dir")
+        self.bundle_recorder = BundleRecorder(manifest.build, "output_dir", "artifacts_dir", None)
 
     def test_record_component(self):
         component = BuildManifest.Component(
@@ -219,7 +219,7 @@ def test_write_manifest(self):
                 self.assertEqual(yaml.safe_load(f), data)
 
     def test_record_component_public(self):
-        self.bundle_recorder.public_url = "https://ci.opensearch.org/ci/os-distro-prod"
+        self.bundle_recorder.base_url = "https://ci.opensearch.org/ci/ci-env-prod/job-name-dashboards/1.2.0/build-123/platform-mac/arch-amd64/"
         component = BuildManifest.Component(
             {
                 "name": "alertingDashboards",
@@ -246,7 +246,7 @@ def test_record_component_public(self):
                 "components": [
                     {
                         "commit_id": "ae789280740d7000d1f13245019414abeedfc286",
-                        "location": "https://ci.opensearch.org/ci/os-distro-prod/builds/1.1.0/c94ebec444a94ada86a230c9297b1d73/x64/plugins",
+                        "location": "https://ci.opensearch.org/ci/ci-env-prod/job-name-dashboards/1.2.0/build-123/platform-mac/arch-amd64/builds/plugins",
                         "name": component.name,
                         "ref": "main",
                         "repository": "https://github.com/opensearch-project/alerting-dashboards-plugin",
@@ -257,8 +257,8 @@ def test_record_component_public(self):
         )
 
     def test_get_location_scenarios(self):
-        def get_location(public_url):
-            self.bundle_recorder.public_url = public_url
+        def get_location(base_url):
+            self.bundle_recorder.base_url = base_url
             return self.bundle_recorder._BundleRecorder__get_location("builds", "dir1/dir2/file", "/tmp/builds/foo/dir1/dir2/file")
 
         # No public URL - Fallback to ABS Path
@@ -266,14 +266,14 @@ def get_location(public_url):
 
         # Public URL - No trailing slash
         self.assertEqual(
-            get_location("https://ci.opensearch.org/ci/os-distro-prod"),
-            "https://ci.opensearch.org/ci/os-distro-prod/builds/1.1.0/c94ebec444a94ada86a230c9297b1d73/x64/dir1/dir2/file",
+            get_location("https://ci.opensearch.org/ci/ci-env-prod/job-name-dashboards/1.2.0/build-123/platform-mac/arch-amd64"),
+            "https://ci.opensearch.org/ci/ci-env-prod/job-name-dashboards/1.2.0/build-123/platform-mac/arch-amd64/builds/dir1/dir2/file",
         )
 
         # Public URL - Trailing slash
         self.assertEqual(
-            get_location("https://ci.opensearch.org/ci/os-distro-prod/"),
-            "https://ci.opensearch.org/ci/os-distro-prod/builds/1.1.0/c94ebec444a94ada86a230c9297b1d73/x64/dir1/dir2/file",
+            get_location("https://ci.opensearch.org/ci/ci-env-prod/job-name-dashboards/1.2.0/build-123/platform-mac/arch-amd64/"),
+            "https://ci.opensearch.org/ci/ci-env-prod/job-name-dashboards/1.2.0/build-123/platform-mac/arch-amd64/builds/dir1/dir2/file",
         )
 
     def test_tar_name(self):