Repository: opensearch-project/opensearch-build

Issue:
Generated manifests for OpenSearch 2.x need custom JAVA_HOME
### Is your feature request related to a problem? Please describe

Automation opened a PR to generate the OpenSearch 3.0 manifest. That manifest was missing `JAVA_HOME` that was added manually in https://github.com/opensearch-project/opensearch-build/pull/2042.

### Describe the solution you'd like

Add a way to customize generated manifests for various products/versions, add `JAVA_HOME` to the OpenSearch ones.

### Describe alternatives you've considered

Keep doing it manually.

### Additional context

_No response_


Discussion:
We just need the manifest workflow to add this line for OpenSearch BE.
The problem being we can only default to the highest java version tho.
There's some hard-coding going on in https://github.com/opensearch-project/opensearch-build/blob/3b65a0a9803f53d4ad1bb4c51763748150fb9bd0/src/manifests_workflow/input_manifests.py#L108 that will need refactoring.
[Triage] just a thought since the the build container already has multiple java versions added, we might need to add an entry to manifest file that has JDK version specification, just like we have one for [checks:](https://github.com/opensearch-project/opensearch-build/blob/main/manifests/2.0.0/opensearch-2.0.0.yml#L14), so that way each component can use the desired java version that is passed, if not it will go withe default JDK version.

Resolving patch:
diff --git a/src/manifests_workflow/input_manifests.py b/src/manifests_workflow/input_manifests.py
--- a/src/manifests_workflow/input_manifests.py
+++ b/src/manifests_workflow/input_manifests.py
@@ -11,7 +11,7 @@
 from abc import abstractmethod
 from typing import Dict, List, Type, Union
 
-from manifests.input_manifest import InputManifest
+from manifests.input_manifest import InputComponents, InputManifest
 from manifests.manifests import Manifests
 from manifests_workflow.component_opensearch import ComponentOpenSearch
 from manifests_workflow.component_opensearch_dashboards_min import ComponentOpenSearchDashboardsMin
@@ -48,7 +48,12 @@ def files(self, name: str) -> List:
         return results
 
     @abstractmethod
-    def update(self, min_klass: Union[Type[ComponentOpenSearchMin], Type[ComponentOpenSearchDashboardsMin]], component_klass: Type[ComponentOpenSearch], keep: bool = False) -> None:
+    def update(
+        self,
+        min_klass: Union[Type[ComponentOpenSearchMin], Type[ComponentOpenSearchDashboardsMin]],
+        component_klass: Type[ComponentOpenSearch],
+        keep: bool = False,
+    ) -> None:
         known_versions = self.versions
         logging.info(f"Known versions: {known_versions}")
         main_versions: Dict = {}
@@ -106,30 +111,23 @@ def update(self, min_klass: Union[Type[ComponentOpenSearchMin], Type[ComponentOp
                 self.add_to_cron(release_version)
 
     def create_manifest(self, version: str, components: List = []) -> InputManifest:
-        image_map = {
-            "opensearch": "opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2",
-            "opensearch-dashboards": "opensearchstaging/ci-runner:ci-runner-centos7-opensearch-dashboards-build-v2"
-        }
-
-        data: Dict = {
-            "schema-version": "1.0",
-            "build": {
-                "name": self.name,
-                "version": version
-            },
-            "ci": {
-                "image": {
-                    "name": image_map[self.prefix]
-                }
-            },
-            "components": [],
-        }
+        templates_base_path = os.path.join(self.manifests_path(), "templates")
+        template_version_folder = version.split(".")[0] + ".x"
+        template_full_path = os.path.join(templates_base_path, self.prefix, template_version_folder, "manifest.yml")
+        if not os.path.exists(template_full_path):
+            template_full_path = os.path.join(templates_base_path, self.prefix, "default", "manifest.yml")
+
+        manifest = InputManifest.from_file(open(template_full_path))
+
+        manifest.build.version = version
+        manifests_components = []
 
         for component in components:
             logging.info(f" Adding {component.name}")
-            data["components"].append(component.to_dict())
+            manifests_components.append(component.to_dict())
 
-        return InputManifest(data)
+        manifest.components = InputComponents(manifests_components)  # type: ignore
+        return manifest
 
     def write_manifest(self, version: str, components: List = []) -> None:
         logging.info(f"Creating new version: {version}")
@@ -151,10 +149,7 @@ def add_to_cron(self, version: str) -> None:
         if cron_entry in data:
             raise ValueError(f"{jenkinsfile} already contains an entry for {self.prefix} {version}")
 
-        data = data.replace(
-            "parameterizedCron '''\n",
-            f"parameterizedCron '''\n{' ' * 12}{cron_entry}"
-        )
+        data = data.replace("parameterizedCron '''\n", f"parameterizedCron '''\n{' ' * 12}{cron_entry}")
 
         with open(jenkinsfile, "w") as f:
             f.write(data)


Test patch:
diff --git a/tests/tests_manifests_workflow/test_input_manifests.py b/tests/tests_manifests_workflow/test_input_manifests.py
--- a/tests/tests_manifests_workflow/test_input_manifests.py
+++ b/tests/tests_manifests_workflow/test_input_manifests.py
@@ -17,26 +17,52 @@ def test_manifests_path(self) -> None:
         self.assertEqual(path, InputManifests.manifests_path())
 
     def test_create_manifest_opensearch(self) -> None:
-        input_manifests = InputManifests("opensearch")
+        input_manifests = InputManifests("OpenSearch")
         input_manifest = input_manifests.create_manifest("1.2.3", [])
         self.assertEqual(
             input_manifest.to_dict(),
             {
                 "schema-version": "1.0",
-                "build": {"name": "opensearch", "version": "1.2.3"},
-                "ci": {"image": {"name": "opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2"}},
+                "build": {"name": "OpenSearch", "version": "1.2.3"},
+                "ci": {"image": {"name": "opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2",
+                                 "args": "-e JAVA_HOME=/opt/java/openjdk-11"}},
+            },
+        )
+
+    def test_create_manifest_opensearch_from_default(self) -> None:
+        input_manifests = InputManifests("OpenSearch")
+        input_manifest = input_manifests.create_manifest("0.2.3", [])
+        self.assertEqual(
+            input_manifest.to_dict(),
+            {
+                "schema-version": "1.0",
+                "build": {"name": "OpenSearch", "version": "0.2.3"},
+                "ci": {"image": {"name": "opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2",
+                                 "args": "-e JAVA_HOME=/opt/java/openjdk-17"}},
             },
         )
 
     def test_create_manifest_opensearch_dashboards(self) -> None:
-        input_manifests = InputManifests("opensearch-dashboards")
+        input_manifests = InputManifests("OpenSearch Dashboards")
         input_manifest = input_manifests.create_manifest("1.2.3", [])
         self.assertEqual(
             input_manifest.to_dict(),
             {
                 "schema-version": "1.0",
-                "build": {"name": "opensearch-dashboards", "version": "1.2.3"},
-                "ci": {"image": {"name": "opensearchstaging/ci-runner:ci-runner-centos7-opensearch-dashboards-build-v2"}},
+                "build": {"name": "OpenSearch Dashboards", "version": "1.2.3"},
+                "ci": {"image": {"name": "opensearchstaging/ci-runner:ci-runner-centos7-opensearch-dashboards-build-v2", }},
+            },
+        )
+
+    def test_create_manifest_opensearch_dashboards_from_default(self) -> None:
+        input_manifests = InputManifests("OpenSearch Dashboards")
+        input_manifest = input_manifests.create_manifest("4.2.3", [])
+        self.assertEqual(
+            input_manifest.to_dict(),
+            {
+                "schema-version": "1.0",
+                "build": {"name": "OpenSearch Dashboards", "version": "4.2.3"},
+                "ci": {"image": {"name": "opensearchstaging/ci-runner:ci-runner-centos7-opensearch-dashboards-build-v2", }},
             },
         )
 
diff --git a/tests/tests_manifests_workflow/test_input_manifests_opensearch.py b/tests/tests_manifests_workflow/test_input_manifests_opensearch.py
--- a/tests/tests_manifests_workflow/test_input_manifests_opensearch.py
+++ b/tests/tests_manifests_workflow/test_input_manifests_opensearch.py
@@ -30,6 +30,8 @@ def test_files(self) -> None:
 
     @patch("os.makedirs")
     @patch("os.chdir")
+    @patch("manifests_workflow.input_manifests.InputComponents")
+    @patch("manifests_workflow.input_manifests.InputManifest.from_file")
     @patch("manifests_workflow.input_manifests.InputManifests.add_to_cron")
     @patch("manifests_workflow.input_manifests.InputManifest.from_path")
     @patch("manifests_workflow.input_manifests_opensearch.ComponentOpenSearchMin")
@@ -37,7 +39,8 @@ def test_files(self) -> None:
     @patch("manifests_workflow.input_manifests.InputManifest")
     def test_update(self, mock_input_manifest: MagicMock, mock_component_opensearch: MagicMock,
                     mock_component_opensearch_min: MagicMock, mock_input_manifest_from_path: MagicMock,
-                    mock_add_to_cron: MagicMock, *mocks: MagicMock) -> None:
+                    mock_add_to_cron: MagicMock, mock_input_manifest_from_file: MagicMock,
+                    mock_input_manifest_component: MagicMock, *mocks: MagicMock) -> None:
         mock_component_opensearch_min.return_value = MagicMock(name="OpenSearch")
         mock_component_opensearch_min.branches.return_value = ["main", "0.9.0"]
         mock_component_opensearch_min.checkout.return_value = MagicMock(version="0.9.0")
@@ -48,7 +51,7 @@ def test_update(self, mock_input_manifest: MagicMock, mock_component_opensearch:
         }
         manifests = InputManifestsOpenSearch()
         manifests.update()
-        self.assertEqual(mock_input_manifest().to_file.call_count, 2)
+        self.assertEqual(mock_input_manifest_from_file().to_file.call_count, 2)
         calls = [
             call(
                 os.path.join(
@@ -65,7 +68,7 @@ def test_update(self, mock_input_manifest: MagicMock, mock_component_opensearch:
                 )
             ),
         ]
-        mock_input_manifest().to_file.assert_has_calls(calls)
+        mock_input_manifest_from_file().to_file.assert_has_calls(calls)
         mock_add_to_cron.assert_has_calls([
             call('0.10.0'),
             call('0.9.0')
diff --git a/tests/tests_manifests_workflow/test_input_manifests_opensearch_dashboards.py b/tests/tests_manifests_workflow/test_input_manifests_opensearch_dashboards.py
--- a/tests/tests_manifests_workflow/test_input_manifests_opensearch_dashboards.py
+++ b/tests/tests_manifests_workflow/test_input_manifests_opensearch_dashboards.py
@@ -29,19 +29,24 @@ def test_files(self) -> None:
 
     @patch("os.makedirs")
     @patch("os.chdir")
+    @patch("manifests_workflow.input_manifests.InputComponents")
+    @patch("manifests_workflow.input_manifests.InputManifest.from_file")
     @patch("manifests_workflow.input_manifests.InputManifests.add_to_cron")
     @patch("manifests_workflow.input_manifests.InputManifest.from_path")
     @patch("manifests_workflow.input_manifests_opensearch_dashboards.ComponentOpenSearchDashboardsMin")
     @patch("manifests_workflow.input_manifests.InputManifest")
     def test_update(self, mock_input_manifest: MagicMock, mock_component_opensearch_min: MagicMock,
-                    mock_input_manifest_from_path: MagicMock, mock_add_to_cron: MagicMock, *mocks: MagicMock) -> None:
+                    mock_input_manifest_from_path: MagicMock, mock_add_to_cron: MagicMock,
+                    mock_input_manifest_from_file: MagicMock, mock_input_manifest_component: MagicMock,
+                    *mocks: MagicMock) -> None:
         mock_component_opensearch_min.return_value = MagicMock(name="OpenSearch-Dashboards")
         mock_component_opensearch_min.branches.return_value = ["main", "0.9.0"]
         mock_component_opensearch_min.checkout.return_value = MagicMock(version="0.9.0")
         mock_input_manifest_from_path.return_value = MagicMock(components=[])
+
         manifests = InputManifestsOpenSearchDashboards()
         manifests.update()
-        self.assertEqual(mock_input_manifest().to_file.call_count, 1)
+        self.assertEqual(mock_input_manifest_from_file().to_file.call_count, 1)
         calls = [
             call(
                 os.path.join(
@@ -51,7 +56,7 @@ def test_update(self, mock_input_manifest: MagicMock, mock_component_opensearch_
                 )
             )
         ]
-        mock_input_manifest().to_file.assert_has_calls(calls)
+        mock_input_manifest_from_file().to_file.assert_has_calls(calls)
         mock_add_to_cron.assert_has_calls([
             call('0.9.0')
         ])