Repository: dell/omnia
Pull request #2550:  Restructure the code for k8s cluster reset playbooks in a non-HA config mode
Author: login: pullan1
Created: 2025-03-05T16:18:38Z | Merged: 2025-03-08T03:53:18Z



Diff:
diff --git a/local_repo/roles/parse_and_download/module_utils/software_utils.py b/local_repo/roles/parse_and_download/module_utils/software_utils.py
index 4da9f8f0f7..1075085677 100644
--- a/local_repo/roles/parse_and_download/module_utils/software_utils.py
+++ b/local_repo/roles/parse_and_download/module_utils/software_utils.py
@@ -203,7 +203,7 @@ def parse_repo_urls(local_repo_config_path, version_variables):
     parsed_repos = []
 
     for url_ in rhel_repo_entry:
-        name = url_.get("name","unkown")
+        name = url_.get("name","unknown")
         url = url_.get("url","")
         gpgkey = url_.get("gpgkey")
         if not is_remote_url_reachable(url):
@@ -231,7 +231,7 @@ def parse_repo_urls(local_repo_config_path, version_variables):
         if name in ["amdgpu", "rocm", "beegfs"] and (version is None or version == "null"):
             continue
 
-        # Edge case for oneapi, snoopy
+        # Edge case for oneapi, snoopy, nvidia-repo
         elif not is_remote_url_reachable(rendered_url) and name not in ["oneapi","snoopy","nvidia-repo"]:
             return rendered_url, False
 
diff --git a/scheduler/roles/cluster_validation/tasks/fetch_omnia_inputs.yml b/scheduler/roles/cluster_validation/tasks/fetch_omnia_inputs.yml
index dab374c8ce..2da7190198 100644
--- a/scheduler/roles/cluster_validation/tasks/fetch_omnia_inputs.yml
+++ b/scheduler/roles/cluster_validation/tasks/fetch_omnia_inputs.yml
@@ -102,14 +102,6 @@
 - name: Validate k8s_support variable
   when: k8s_support
   block:
-    - name: Assert supported kubernetes version
-      ansible.builtin.assert:
-        that:
-          - k8s_version | default("", false) | length > 1
-          - " ('{{ omnia20_k8s_version }}' in k8s_version)" # noqa: yaml[line-length]
-        success_msg: "{{ success_msg_k8s_version }}"
-        fail_msg: "{{ fail_msg_k8s_version }}"
-
     - name: Assert kubernetes cni
       ansible.builtin.assert:
         that:
diff --git a/scheduler/roles/cluster_validation/tasks/validate_network_spec.yml b/scheduler/roles/cluster_validation/tasks/validate_network_spec.yml
index 71ac5f70bf..3d69fa5093 100644
--- a/scheduler/roles/cluster_validation/tasks/validate_network_spec.yml
+++ b/scheduler/roles/cluster_validation/tasks/validate_network_spec.yml
@@ -34,7 +34,7 @@
 - name: Set admin network nic and ip
   ansible.builtin.set_fact:
     admin_nic_ip: "{{ network_data.admin_network.primary_oim_admin_ip }}"
-    admin_nic: "{{ network_data.admin_network.nic_name }}"
+    admin_nic: "{{ network_data.admin_network.oim_nic_name }}"
 
 - name: Get all IPv4 addresses assigned to the NIC
   ansible.builtin.set_fact:
diff --git a/scheduler/roles/cluster_validation/templates/containerd_vars_template.j2 b/scheduler/roles/cluster_validation/templates/containerd_vars_template.j2
index 0979c128a0..57b11ab38e 100644
--- a/scheduler/roles/cluster_validation/templates/containerd_vars_template.j2
+++ b/scheduler/roles/cluster_validation/templates/containerd_vars_template.j2
@@ -1,10 +1,10 @@
 containerd_registries_mirrors:
-  - prefix: "{{ hostvars['localhost']['admin_nic_ip'] }}:2225"
+  - prefix: "{{ hostvars['localhost']['admin_nic_ip'] }}:{{ pulp_registry_port }}"
     mirrors:
-      - host: "http://{{ hostvars['localhost']['admin_nic_ip'] }}:2225"
+      - host: "http://{{ hostvars['localhost']['admin_nic_ip'] }}:{{ pulp_registry_port }}"
         capabilities: ["pull", "resolve"]
         skip_verify: true
  
 containerd_registry_auth:
-  - registry: "{{ hostvars['localhost']['admin_nic_ip'] }}:2225"
+  - registry: "{{ hostvars['localhost']['admin_nic_ip'] }}:{{ pulp_registry_port }}"
     password: "{{ hostvars['localhost']['pulp_password'] }}"
diff --git a/scheduler/roles/cluster_validation/templates/download_urls_template.j2 b/scheduler/roles/cluster_validation/templates/download_urls_template.j2
index e1859576c5..34452e44f6 100644
--- a/scheduler/roles/cluster_validation/templates/download_urls_template.j2
+++ b/scheduler/roles/cluster_validation/templates/download_urls_template.j2
@@ -10,10 +10,10 @@ runc_download_url: "{{ hostvars['localhost']['offline_tarball_path'] }}/{{ hostv
 etcd_download_url: "{{ hostvars['localhost']['offline_tarball_path'] }}/{{ hostvars['localhost']['etcd_package'] }}/{{ hostvars['localhost']['etcd_package'] }}.tar.gz"
 containerd_download_url: "{{ hostvars['localhost']['offline_tarball_path'] }}/{{ hostvars['localhost']['containerd_package'] }}/{{ hostvars['localhost']['containerd_package'] }}.tar.gz"
 helm_download_url: "{{ hostvars['localhost']['offline_tarball_path'] }}/{{ hostvars['localhost']['helm_package'] }}/{{ hostvars['localhost']['helm_package'] }}.tar.gz"
-registry_host: "{{ hostvars['localhost']['admin_nic_ip'] }}:2225"
-kube_image_repo: "{{ hostvars['localhost']['admin_nic_ip'] }}:2225"
-docker_image_repo: "{{ hostvars['localhost']['admin_nic_ip'] }}:2225"
-quay_image_repo: "{{ hostvars['localhost']['admin_nic_ip'] }}:2225"
+registry_host: "{{ hostvars['localhost']['admin_nic_ip'] }}:{{ pulp_registry_port }}"
+kube_image_repo: "{{ hostvars['localhost']['admin_nic_ip'] }}:{{ pulp_registry_port }}"
+docker_image_repo: "{{ hostvars['localhost']['admin_nic_ip'] }}:{{ pulp_registry_port }}"
+quay_image_repo: "{{ hostvars['localhost']['admin_nic_ip'] }}:{{ pulp_registry_port }}"
 docker_rh_repo_base_url: "{{ hostvars['localhost']['docker_rh_repo_base_url'] | default('') }}"
 docker_rh_repo_gpgkey: "{{ hostvars['localhost']['docker_rh_repo_gpgkey'] | default('') }}"
 
diff --git a/scheduler/roles/cluster_validation/vars/main.yml b/scheduler/roles/cluster_validation/vars/main.yml
index 386149a109..3253c31dbc 100644
--- a/scheduler/roles/cluster_validation/vars/main.yml
+++ b/scheduler/roles/cluster_validation/vars/main.yml
@@ -62,6 +62,7 @@ k8s_var_common_src: "../templates/common_vars_template.j2"
 k8s_var_final_src: "../templates/final_template.j2"
 k8s_var_dest: "/opt/omnia/kubespray/k8s_all_vars.yml"
 file_permission: "0644"
+pulp_registry_port: "2225"
 
 # Usage: fetch_omnia_inputs.yml
 config_filename: "omnia_config.yml"
@@ -69,13 +70,9 @@ config_vaultname: .omnia_vault_key
 vault_key_permission: "0644"
 min_length: 8
 max_length: 30
-omnia20_k8s_version: '1.31.4'
 omnia_config_syntax_fail_msg: "Failed. Syntax errors present in omnia_config.yml. Fix errors and re-run playbook again."
 fail_msg_mariadb_password: "maria_db password not given in correct format."
 success_msg_mariadb_password: "mariadb_password validated"
-success_msg_k8s_version: "Kubernetes Version Validated"
-fail_msg_k8s_version: "Failed. Kubernetes Version is unsupported or incorrect in software_config.json for this virtual environment. Supported versions for omnia17_venv: 1.29.5, and for omnia161_venv: 1.26.12." # noqa: yaml[line-length]
-kube_version_on_unsupported_os: "Failed. On RHEL/Rocky 8.6 or 8.7 OS, supported kubernetes version is 1.26.12 and suppported virtual environment is omnia161_venv only"  # noqa: yaml[line-length]
 success_msg_k8s_cni: "Kubernetes CNI Validated"
 fail_msg_k8s_cni: "Kubernetes CNI not correct."
 supported_topology_manager_policy: ['none', 'best-effort', 'restricted', 'single-numa-node']
diff --git a/scheduler/scheduler.yml b/scheduler/scheduler.yml
index e282c12c2c..619a513143 100644
--- a/scheduler/scheduler.yml
+++ b/scheduler/scheduler.yml
@@ -81,7 +81,7 @@
 
     - name: Execute ansible-playbook for Kubernetes setup asynchronously
       ansible.builtin.shell:
-        cmd: ansible-playbook {{ k8s_nfs_share }}/k8s_start_setup.yml -i {{ k8s_nfs_share }}/inv_k8s --extra-vars "@{{ k8s_nfs_share }}/k8s_all_vars.yml" -vvv | /usr/bin/tee {{ k8s_logs_dir }}//k8s_deployment.log
+        cmd: /venv/bin/ansible-playbook {{ k8s_nfs_share }}/k8s_start_setup.yml -i {{ k8s_nfs_share }}/inv_k8s --extra-vars "@{{ k8s_nfs_share }}/k8s_all_vars.yml" -vvv | /usr/bin/tee {{ k8s_logs_dir }}//k8s_deployment.log
       async: 3600  # Set async timeout (e.g., 1 hour)
       poll: 0  # Non-blocking (continue the playbook without waiting for completion)
       register: result  # Register the result to capture job ID
diff --git a/utils/reset_cluster_configuration.yml b/utils/reset_cluster_configuration.yml
index 9f47e8df0d..eb6590deb7 100644
--- a/utils/reset_cluster_configuration.yml
+++ b/utils/reset_cluster_configuration.yml
@@ -13,9 +13,13 @@
 #  limitations under the License.
 ---
 
-- name: Check if virtual environment is active
-  ansible.builtin.import_playbook: check_venv.yml
-  when: not ( check_venv_executed | default(false) | bool )
+- name: Include input project directory
+  ansible.builtin.import_playbook: ../utils/include_input_dir.yml
+
+- name: Create kubespray container group
+  ansible.builtin.import_playbook: ../utils/create_container_group.yml
+  vars:
+    omnia_kubespray_group: true
 
 - name: Warning and User confirmation for removing cluster
   hosts: localhost
@@ -43,9 +47,40 @@
   roles:
     - { role: remove_cluster/remove_k8s_cluster } # noqa: role-name[path]
 
-- name: Remove kubernetes cluster
-  tags: k8s_cluster
-  ansible.builtin.import_playbook: "{{ playbook_dir }}/playbooks/reset_k8s_cluster.yml"
+- name: Reset kubernetes cluster
+  hosts: omnia_kubespray
+  vars:
+    k8s_nfs_share: "/opt/omnia/kubespray"
+    k8s_log_dir: "/opt/omnia/log/kubespray"
+    dir_mode: '0755'
+  tasks:
+    - name: Execute tasks for Reset kubernetes setup
+      block:
+        - name: Copy reset_k8s_cluste to kubespray nfs share
+          ansible.builtin.copy:
+            src: "{{ playbook_dir }}/playbooks/reset_k8s_cluster.yml"
+            dest: "{{ k8s_nfs_share }}/reset_k8s_cluster.yml"
+
+        - name: Ensure /logs directory exists
+          ansible.builtin.file:
+            path: "{{ k8s_log_dir }}"
+            state: directory
+            mode: "{{ dir_mode }}"
+
+        - name: Execute ansible-playbook for Kubernetes setup asynchronously
+          ansible.builtin.shell:
+            cmd: /venv/bin/ansible-playbook {{ k8s_nfs_share }}/reset_k8s_cluster.yml -i {{ k8s_nfs_share }}/inv_k8s --extra-vars "@{{ k8s_nfs_share }}/k8s_all_vars.yml" -vvv | /usr/bin/tee {{ k8s_log_dir }}/k8s_reset_cluster.log
+          async: 3600  # Set async timeout (e.g., 1 hour)
+          poll: 0  # Non-blocking (continue the playbook without waiting for completion)
+          register: k8s_reset_result  # Register the result to capture job ID
+        
+        - name: Wait for the Kubernetes rest cluster to finish. Logs can be checked at /opt/omnia/log/kubespray/k8s_reset_cluster.log
+          ansible.builtin.async_status:
+            jid: "{{ k8s_reset_result.ansible_job_id }}"  # Job ID from the previous task
+          register: job_result
+          until: job_result.finished
+          retries: 60  # Retry the task 60 times (1 hour total)
+          delay: 10  # Wait 10 seconds between retries
 
 # manually removing kube service address from /etc/resolv.conf
 - name: Remove kube service address post kubernetes cluster removal


Reviews:
user: login: snarthan
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-06T04:17:20Z
user: login: pullan1
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-06T04:39:39Z
user: login: snarthan
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-03-06T07:49:23Z
user: login: abhishek-sa1
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-03-07T07:22:05Z
user: login: priti-parate
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-03-07T13:27:29Z

Review comments:
diff_hunk: @@ -81,7 +81,7 @@
 
     - name: Execute ansible-playbook for Kubernetes setup asynchronously
       ansible.builtin.shell:
-        cmd: ansible-playbook {{ k8s_nfs_share }}/k8s_start_setup.yml -i {{ k8s_nfs_share }}/inv_k8s --extra-vars "@{{ k8s_nfs_share }}/k8s_all_vars.yml" -vvv | /usr/bin/tee {{ k8s_logs_dir }}//k8s_deployment.log
+        cmd: /venv/bin/ansible-playbook {{ k8s_nfs_share }}/k8s_start_setup.yml -i {{ k8s_nfs_share }}/inv_k8s --extra-vars "@{{ k8s_nfs_share }}/k8s_all_vars.yml" -vvv | /usr/bin/tee {{ k8s_logs_dir }}//k8s_deployment.log
path: scheduler/scheduler.yml
user: login: snarthan
body: is this /venc/bin/ansible-playbook will activate the virtual env ?
created_at: 2025-03-06T04:17:20Z
author_association: COLLABORATOR
diff_hunk: @@ -81,7 +81,7 @@
 
     - name: Execute ansible-playbook for Kubernetes setup asynchronously
       ansible.builtin.shell:
-        cmd: ansible-playbook {{ k8s_nfs_share }}/k8s_start_setup.yml -i {{ k8s_nfs_share }}/inv_k8s --extra-vars "@{{ k8s_nfs_share }}/k8s_all_vars.yml" -vvv | /usr/bin/tee {{ k8s_logs_dir }}//k8s_deployment.log
+        cmd: /venv/bin/ansible-playbook {{ k8s_nfs_share }}/k8s_start_setup.yml -i {{ k8s_nfs_share }}/inv_k8s --extra-vars "@{{ k8s_nfs_share }}/k8s_all_vars.yml" -vvv | /usr/bin/tee {{ k8s_logs_dir }}//k8s_deployment.log
path: scheduler/scheduler.yml
user: login: pullan1
body: No it won't activate venv , It simply calls the ansible-playbook executable located in /venv/bin/ . Inside kubespray container ansible is getting installed in venv. 
created_at: 2025-03-06T04:39:39Z
author_association: CONTRIBUTOR

PR comments:
user: login: Omnia-svc
created_at: 2025-03-05T16:20:03Z
body: Can one of the admins verify this patch?
author_association: COLLABORATOR
user: login: snarthan
created_at: 2025-03-06T04:22:14Z
body: run omnia-checkers
author_association: COLLABORATOR
user: login: abhishek-sa1
created_at: 2025-03-07T05:42:02Z
body: @pullan1  can you update PR with title and description
author_association: COLLABORATOR
user: login: priti-parate
created_at: 2025-03-07T13:27:44Z
body: run omnia-checkers
author_association: COLLABORATOR