Repository: cablelabs/snaps-kubernetes

Issue:
Authentication and authorization support
Sachin to fill in details

Discussion:

Patch:
diff --git a/ci/README.md b/ci/README.md
index 5bdafea..a14b65b 100644
--- a/ci/README.md
+++ b/ci/README.md
@@ -57,6 +57,7 @@ Please begin with the file ci/snaps/snaps-env.yaml.tmplt. Below is a
 short explanation of each variable:
 
 * build_id - Used for appending to the names of any shared OpenStack resources being spawned by this process
+* k8s_version - Kubernetes version to lay down (default 1.11.0). Do not prefix with 'v'
 * admin_user - The name of an OpenStack user that is part of the 'admin' group
 * admin_proj - The name of an OpenStack project that has been associated with the 'admin_user'
 * admin_pass - The associated OpenStack password for the 'admin_user'
@@ -74,6 +75,11 @@ short explanation of each variable:
 * admin_iface - 'ens3' must be the value with the image being used (TODO - consider removing this configuration option as it is directly tied to the ubuntu image)
 * deployment_yaml_target_path - The location where to copy the deployment.yaml file (TODO - consider removing this configuration option as it really does not have to be configurable)
 * local_snaps_k8_dir - The location to the local snaps-kubernetes source directory which will be copied to the build VM
+* flavor_metadata - snaps-oo formatted flavor extra metadata generally used for CPU pinning in this context
+* run_build - When True, deployment operations will be executed (default True)
+* run_validation - When True, validation operations will be executed (default True)
+* run_conformance - When True, the CNCF tests will be setup and executed on the build server (default False). Note: script will exit after CNCF has been installed and kicked-off but will not wait for completion which is ~ 2 hours
+* inject_keys - Denotes whether or not the CI scripts should inject ssh keys into the new VMs prior to executing the iaas_launch.py script
 
 ### Launch
 note: The entire process can take up to an hour to complete.
diff --git a/ci/playbooks/deploy_k8.yaml b/ci/playbooks/deploy_k8.yaml
index f3f2611..3dd67e2 100644
--- a/ci/playbooks/deploy_k8.yaml
+++ b/ci/playbooks/deploy_k8.yaml
@@ -14,6 +14,10 @@
 # limitations under the License.
 ---
 - hosts: all
+
+  vars:
+    inject_keys: "{{ inject_keys }}"
+
   tasks:
   - name: Install apt dependencies
     become: yes
@@ -55,14 +59,17 @@
     file:
       state: absent
       dest: ~/.ssh/id_rsa
+    when: inject_keys == True
 
   - name: Delete user's public key - ~/.ssh/id_rsa.pub
     file:
       state: absent
       dest: ~/.ssh/id_rsa.pub - ~/.ssh/id_rsa
+    when: inject_keys == True
 
   - name: Generate user's key
     command: ssh-keygen -P '' -f ~/.ssh/id_rsa
+    when: inject_keys == True
 
   - name: Copy user's key to root@master node @{{ master_admin_ip }}
     command: "sshpass -p \"{{ node_host_pass }}\" ssh-copy-id -i ~/.ssh/id_rsa.pub -o StrictHostKeyChecking=no root@{{ master_admin_ip }}"
@@ -70,6 +77,7 @@
     delay: 3
     register: result
     until: result.rc == 0
+    when: inject_keys == True
 
   - name: Copy root's key to minion node @{{ minion_admin_ip }}
     command: "sshpass -p \"{{ node_host_pass }}\" ssh-copy-id -i ~/.ssh/id_rsa.pub -o StrictHostKeyChecking=no root@{{ minion_admin_ip }}"
@@ -77,6 +85,7 @@
     delay: 3
     register: result
     until: result.rc == 0
+    when: inject_keys == True
 
   - name: Apply template and copy deployment.yaml.tmplt to {{ deployment_yaml_path }}
     action: template src=templates/deployment.yaml.tmplt dest={{ deployment_yaml_path }}
diff --git a/ci/playbooks/templates/deployment.yaml.tmplt b/ci/playbooks/templates/deployment.yaml.tmplt
index 106c6fd..5eb25e0 100644
--- a/ci/playbooks/templates/deployment.yaml.tmplt
+++ b/ci/playbooks/templates/deployment.yaml.tmplt
@@ -17,7 +17,7 @@
 kubernetes:
   Project_name: {{ build_id }}
   Git_branch: {{ branch_name }}
-  version: v1.11.0
+  version: v{{ k8s_version }}
   enable_metrics_server: True
   Exclusive_CPU_alloc_support: True  
   enable_logging: True
diff --git a/ci/playbooks/validation.yaml b/ci/playbooks/validation.yaml
new file mode 100644
index 0000000..b9525ed
--- /dev/null
+++ b/ci/playbooks/validation.yaml
@@ -0,0 +1,42 @@
+# Copyright (c) 2018 Cable Television Laboratories, Inc. ("CableLabs")
+#                    and others.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+- hosts: all
+
+  tasks:
+  - name: Get masters
+    shell: "kubectl --kubeconfig=/tmp/snaps-k8s-projects/{{ project_name }}/node-kubeconfig.yaml get nodes | grep -v 'NAME     STATUS   ROLES    AGE   VERSION' | grep Ready | grep master"
+    register: masters
+
+  - name: Check that expected master count equal {{ num_masters }}
+    fail:
+    when: masters.stdout_lines|length != num_masters
+
+  - name: Get nodes
+    shell: "kubectl --kubeconfig=/tmp/snaps-k8s-projects/{{ project_name }}/node-kubeconfig.yaml get nodes | grep -v 'NAME     STATUS   ROLES    AGE   VERSION' | grep Ready | grep node"
+    register: minions
+
+  - name: Check that expected minion count equal expected {{ num_minions }}
+    fail:
+    when: minions.stdout_lines|length != num_minions
+
+  - name: Get Version
+    shell: "kubectl --kubeconfig=/tmp/snaps-k8s-projects/{{ project_name }}/node-kubeconfig.yaml get nodes | grep -v 'NAME     STATUS   ROLES    AGE   VERSION' | grep Ready | grep {{ k8s_version }}"
+    register: version
+
+  - name: Check number of nodes with the expected version {{ k8s_version }} equals the number of masters and minions
+    fail:
+    when: version.stdout_lines|length != num_masters + num_minions
+
diff --git a/ci/snaps/snaps-env.yaml.tmplt b/ci/snaps/snaps-env.yaml.tmplt
index 2b092c9..a7880a7 100644
--- a/ci/snaps/snaps-env.yaml.tmplt
+++ b/ci/snaps/snaps-env.yaml.tmplt
@@ -1,6 +1,8 @@
 ---
 build_id: {{ build_id }}
 
+k8s_version: {{ k8s_version | default('1.11.0') }}
+
 admin_user: {{ os_admin_user | default('admin') }}
 admin_proj: {{ os_admin_proj | default('admin') }}
 admin_pass: {{ os_admin_pass }}
@@ -28,3 +30,17 @@ admin_iface: ens3
 deployment_yaml_target_path: /tmp/deployment.yaml
 
 local_snaps_k8_dir: {{ local_snaps_k8s_dir }}
+
+# Optional Settings
+# For CPU Pinning
+flavor_metadata:
+  hw:cpu_policy: dedicated
+  hw:cpu_thread_policy: isolate
+
+# Optional CI job overrides
+run_build: {{ run_build | default(True) }}
+run_validation: {{ run_validation | default(True) }}
+
+# True will execute the CNCF conformance tests
+run_conformance: {{ run_conformance | default(False) }}
+inject_keys: {{ inject_keys | default(True) }}
\ No newline at end of file
diff --git a/ci/snaps/snaps_k8_tmplt.yaml b/ci/snaps/snaps_k8_tmplt.yaml
index 3396c0c..71dae13 100644
--- a/ci/snaps/snaps_k8_tmplt.yaml
+++ b/ci/snaps/snaps_k8_tmplt.yaml
@@ -284,6 +284,7 @@ openstack:
             - systemctl restart ssh
 ansible:
   # Install and configure snaps-boot to build host
+{% if run_build | default(True) %}
   - playbook_location: {{ local_snaps_k8_dir }}/ci/playbooks/deploy_k8.yaml
     hosts:
     - build-k8-vm
@@ -316,7 +317,34 @@ ansible:
       node_host_pass:
         type: string
         value: {{ node_host_pass }}
+      inject_keys:
+        type: string
+        value: {{ inject_keys | default(True) }}
+      k8s_version:
+        type: string
+        value: {{ k8s_version | default('1.11.0') }}
+{% endif %}
 
+{% if run_validation | default(True) %}
+  - playbook_location: {{ local_snaps_k8_dir }}/ci/playbooks/validation.yaml
+    hosts:
+    - build-k8-vm
+    variables:
+      project_name:
+        type: string
+        value: {{ build_id }}
+      k8s_version:
+        type: string
+        value: {{ k8s_version | default('1.11.0') }}
+      num_masters:
+        type: string
+        value: 1
+      num_minions:
+        type: string
+        value: 1
+{% endif %}
+
+{% if run_conformance | default(False) %}
   - playbook_location: {{ local_snaps_k8_dir }}/ci/playbooks/conformance.yaml
     hosts:
     - build-k8-vm
@@ -324,3 +352,4 @@ ansible:
       project_name:
         type: string
         value: {{ build_id }}
+{% endif %}
diff --git a/iaas_launch.py b/iaas_launch.py
index d3e293c..e35d414 100644
--- a/iaas_launch.py
+++ b/iaas_launch.py
@@ -23,7 +23,10 @@ import os
 from snaps_common.ansible_snaps import ansible_utils
 
 from snaps_common.file import file_utils
+from snaps_common.ssh import ssh_utils
+
 from snaps_k8s.common.consts import consts
+from snaps_k8s.common.utils import config_utils
 from snaps_k8s.common.utils.validation_utils import validate_deployment_file
 from snaps_k8s.provision import k8_utils
 
@@ -71,6 +74,23 @@ def __installation_logs(cmdln_args):
         logging.getLogger().addHandler(logging.StreamHandler())
 
 
+def __manage_keys(config):
+    """
+    Creates and pushes SSH keys when necessary
+    """
+    logger.info('Managing SSH keys')
+    nodes_info = config_utils.get_nodes_ip_name_type(config)
+    for hostname, ip, node_type in nodes_info:
+        ssh_client = ssh_utils.ssh_client(ip, 'root')
+        if not ssh_client:
+            logger.debug('Creating and injecting key to %s', ip)
+            password = config_utils.get_node_password(config, hostname)
+            ansible_utils.apply_playbook(consts.MANAGE_KEYS, variables={
+                'ip': ip, 'password': password})
+        else:
+            logger.debug('Key already exists')
+
+
 def __launcher_conf():
     """
     Performs build server setup
@@ -103,6 +123,8 @@ def run(arguments):
     config = file_utils.read_yaml(config_file)
     logger.info('Read configuration file - %s', config_file)
 
+    __manage_keys(config)
+
     if arguments.deploy_kubernetes:
         __launcher_conf()
         validate_deployment_file(config)
diff --git a/snaps_k8s/common/consts/consts.py b/snaps_k8s/common/consts/consts.py
index eae80a5..bc845e5 100644
--- a/snaps_k8s/common/consts/consts.py
+++ b/snaps_k8s/common/consts/consts.py
@@ -147,6 +147,8 @@ BUILD_PREREQS = pkg_resources.resource_filename(
     BUILD_ANSIBLE_PKG, 'build_prerequisites.yaml')
 SETUP_ETC_HOSTS = pkg_resources.resource_filename(
     BUILD_ANSIBLE_PKG, 'setup_etc_hosts.yaml')
+MANAGE_KEYS = pkg_resources.resource_filename(
+    BUILD_ANSIBLE_PKG, 'manage_keys.yaml')
 
 K8_CLEAN_UP = pkg_resources.resource_filename(
     K8_ANSIBLE_PKG, 'k8_clean_up.yaml')
diff --git a/snaps_k8s/common/utils/config_utils.py b/snaps_k8s/common/utils/config_utils.py
index bdd1fb5..83176a3 100644
--- a/snaps_k8s/common/utils/config_utils.py
+++ b/snaps_k8s/common/utils/config_utils.py
@@ -461,6 +461,20 @@ def get_minion_node_ips(k8s_conf):
     return out
 
 
+def get_node_password(k8s_conf, hostname):
+    """
+    Returns the configured password for a given hostname
+    :param k8s_conf: the configuration dict
+    :param hostname: the hostname to resolve
+    :return: the password or None
+    """
+    node_confs = get_node_configs(k8s_conf)
+    for node_conf in node_confs:
+        host_conf = node_conf[consts.HOST_KEY]
+        if hostname == host_conf[consts.HOSTNAME_KEY]:
+            return host_conf[consts.PASSWORD_KEY]
+
+
 def is_logging_enabled(k8s_conf):
     """
     Returns T/F based on the kubernetes.enable_logging value
diff --git a/temp_archive/playbooks/node_inventory_file.yaml b/snaps_k8s/playbooks/build_setup/manage_keys.yaml
similarity index 52%
rename from temp_archive/playbooks/node_inventory_file.yaml
rename to snaps_k8s/playbooks/build_setup/manage_keys.yaml
index c4efcb1..d9e2279 100644
--- a/temp_archive/playbooks/node_inventory_file.yaml
+++ b/snaps_k8s/playbooks/build_setup/manage_keys.yaml
@@ -1,5 +1,4 @@
-# Copyright 2018 ARICENT HOLDINGS LUXEMBOURG SARL and Cable Television
-# Laboratories, Inc.
+# Copyright 2018 ARICENT HOLDINGS LUXEMBOURG SARL and Cable Television Laboratories, Inc.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -11,20 +10,27 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-# This script is responsible for deploying Aricent_Iaas environments and
-# Kubernetes Services
 ---
-
 - hosts: localhost
-  become: true
-  vars_files:
-    - "{{ VARIABLE_FILE }}" 
+
   tasks:
-           
-    - name: inventory details
-      lineinfile:
-           dest: "{{KUBESPRAY_PATH}}kubespray-master/inventory/inventory.cfg"
-           line: "{{host_name}} ansible_ssh_host={{ip}}"
-           insertbefore: BOF
+  - name: Creates ssh keys if one does not exist
+    user:
+      name: "{{ ansible_user_id }}"
+      generate_ssh_key: yes
+      ssh_key_bits: 2048
+      ssh_key_file: "~/.ssh/id_rsa"
+
+  - name: Install sshpass
+    become: yes
+    become_method: sudo
+    become_user: root
+    apt:
+      name: sshpass
 
+  - name: Copy user's key to root@master node @{{ ip }}
+    command: "sshpass -p \"{{ password }}\" ssh-copy-id -i ~/.ssh/id_rsa.pub -o StrictHostKeyChecking=no root@{{ ip }}"
+    retries: 30
+    delay: 3
+    register: result
+    until: result.rc == 0
diff --git a/snaps_k8s/playbooks/k8/launcher_setup.yaml b/snaps_k8s/playbooks/k8/launcher_setup.yaml
index a48ff83..7288a04 100644
--- a/snaps_k8s/playbooks/k8/launcher_setup.yaml
+++ b/snaps_k8s/playbooks/k8/launcher_setup.yaml
@@ -62,6 +62,20 @@
       src: "{{ PROJ_ARTIFACT_DIR }}/all.yml"
       dest: "{{ KUBESPRAY_PATH }}/kubespray/inventory/sample/group_vars/all.yml"
 
+  - name: prepare HTTP proxy for Kubespray's all.yml
+    lineinfile:
+      dest: "{{ KUBESPRAY_PATH }}/kubespray/inventory/sample/group_vars/all.yml"
+      regexp: "^(.*)HTTP_PROXY: (.*)$"
+      line: "HTTP_PROXY: {{ http_proxy }}"
+      backrefs: yes
+
+  - name: prepare HTTPS proxy for Kubespray's all.yml
+    lineinfile:
+      dest: "{{ KUBESPRAY_PATH }}/kubespray/inventory/sample/group_vars/all.yml"
+      regexp: "^(.*)HTTPS_PROXY: (.*)$"
+      line: "HTTPS_PROXY: {{ https_proxy }}"
+      backrefs: yes
+
   - name: installating kubernetes|logs redirect to {{ PROJ_ARTIFACT_DIR }}/kubernetes_kubespray.log
     shell: "ansible-playbook -i {{ KUBESPRAY_PATH }}/kubespray/inventory/sample/inventory.cfg -u root -e ansible_ssh_user=root -b {{ KUBESPRAY_PATH }}/kubespray/cluster.yml >>{{ PROJ_ARTIFACT_DIR }}/kubernetes_kubespray.log"
     args:
diff --git a/snaps_k8s/playbooks/k8/new_inventory_file.yaml b/snaps_k8s/playbooks/k8/new_inventory_file.yaml
index a6a4220..1792ffe 100644
--- a/snaps_k8s/playbooks/k8/new_inventory_file.yaml
+++ b/snaps_k8s/playbooks/k8/new_inventory_file.yaml
@@ -22,6 +22,6 @@
     - name: Adding line "{{ host_name }} ansible_ssh_host={{ ip }}" to "{{ PROJ_ARTIFACT_DIR }}/inventory.cfg"
       lineinfile:
            dest: "{{ PROJ_ARTIFACT_DIR }}/inventory.cfg"
-           line: "{{ host_name }} ansible_ssh_host={{ ip }}"
+           line: "{{ host_name }} ansible_ssh_host={{ ip }} http_proxy={{ http_proxy }}"
            insertbefore: BOF
 
diff --git a/snaps_k8s/playbooks/k8/setup_k8.yaml b/snaps_k8s/playbooks/k8/setup_k8.yaml
index 1da346c..32a21be 100644
--- a/snaps_k8s/playbooks/k8/setup_k8.yaml
+++ b/snaps_k8s/playbooks/k8/setup_k8.yaml
@@ -27,87 +27,66 @@
     no_proxy: "{{ no_proxy | default('') }}"
 
   tasks:
-    - name: apt install python-pip
-      apt:
-        name: python-pip
+  - name: apt update
+    apt:
+      update_cache: yes
 
-    - name: pip install pyOpenSSL -U
-      pip:
-        name: pyOpenSSL
+  - name: apt install python-pip
+    apt:
+      name: python-pip
 
-    - name: apt-get install apt-transport-https
-      apt:
-        name: apt-transport-https
-        state: latest
+  - name: pip install pyOpenSSL
+    pip:
+      name: pyOpenSSL
 
-    - name: swapoff -a
-      command: swapoff -a
+  - name: swapoff -a
+    command: swapoff -a
 
-    - name: remove apt.conf file
-      file:
-        dest: "{{ APT_CONF_DEST }}"
-        state: absent 
-    - name: Create file apt.conf 
-      file:
-        dest: "{{ APT_CONF_DEST }}"
-        state: touch
-    - name: edit apt.conf file
-      lineinfile:
-        dest: "{{ APT_CONF_DEST }}"
-        line: '{{ item }}'
-      with_items:
-        - "Acquire::http::Proxy {{ http_proxy | default('') }};"
-        - "Acquire::https::Proxy {{ https_proxy | default('') }};"
-        - "Acquire::no::Proxy {{ no_proxy | default('') }};"
-    - name: apt-get update
-      apt:
-        update_cache: yes 
+  - name: Install remaining dependencies
+    apt:
+      name:
+      - apt-transport-https
+      - curl
+      - sshpass
+      - ceph-common
+      - build-essential
+      - autoconf
+      - libtool
+      - pkg-config
+      - python-opengl
+      - python-imaging
+      - python-pyrex
+      - python-pyside.qtopengl
+      - idle-python2.7
+      - qt4-dev-tools
+      - qt4-designer
+      - libqtgui4
+      - libqtcore4
+      - libqt4-xml
+      - libqt4-test
+      - libqt4-script
+      - libqt4-network
+      - libqt4-dbus
+      - python-qt4
+      - python-qt4-gl
+      - libgle3
+      - python-dev
+      - libssl-dev
+      - psmisc
+      - golang
 
-    - name: Install remaining dependencies
-      apt:
-        name:
-        - curl
-        - sshpass
-        - python-pip
-        - ceph-common
-        - build-essential
-        - autoconf
-        - libtool
-        - pkg-config
-        - python-opengl
-        - python-imaging
-        - python-pyrex 
-        - python-pyside.qtopengl
-        - idle-python2.7
-        - qt4-dev-tools
-        - qt4-designer
-        - libqtgui4 
-        - libqtcore4
-        - libqt4-xml
-        - libqt4-test
-        - libqt4-script
-        - libqt4-network
-        - libqt4-dbus 
-        - python-qt4
-        - python-qt4-gl
-        - libgle3
-        - python-dev
-        - libssl-dev
-        - psmisc
-        - golang
+  - name: Enable root login on all nodes
+    replace:
+      dest: /etc/ssh/sshd_config
+      regexp: 'PermitRootLogin no'
+      replace: 'PermitRootLogin yes'
+      backup: yes
 
-    - name: Enable root login on all nodes
-      replace:
-        dest: /etc/ssh/sshd_config
-        regexp: 'PermitRootLogin no'
-        replace: 'PermitRootLogin yes'
-        backup: yes
+  - name: copying docker-ce_17.03.2~ce-0~ubuntu-xenial_amd64.deb
+    copy:
+      src: "{{ SRC_PACKAGE_PATH }}/packages/source/docker-ce_17.03.2~ce-0~ubuntu-xenial_amd64.deb"
+      dest: "{{ APT_ARCHIVES_SRC }}"
 
-    - name: copying docker-ce_17.03.2~ce-0~ubuntu-xenial_amd64.deb 
-      copy:
-        src: "{{ SRC_PACKAGE_PATH }}/packages/source/docker-ce_17.03.2~ce-0~ubuntu-xenial_amd64.deb"
-        dest: "{{ APT_ARCHIVES_SRC }}"
-
-    - name: dpkg -i docker-ce_17.03.2~ce-0~ubuntu-xenial_amd64.deb
-      apt:
-         deb: "{{ APT_ARCHIVES_SRC }}/docker-ce_17.03.2~ce-0~ubuntu-xenial_amd64.deb"
+  - name: dpkg -i docker-ce_17.03.2~ce-0~ubuntu-xenial_amd64.deb
+    apt:
+       deb: "{{ APT_ARCHIVES_SRC }}/docker-ce_17.03.2~ce-0~ubuntu-xenial_amd64.deb"
diff --git a/snaps_k8s/provision/ansible_configuration.py b/snaps_k8s/provision/ansible_configuration.py
index 9e6691f..b401c80 100644
--- a/snaps_k8s/provision/ansible_configuration.py
+++ b/snaps_k8s/provision/ansible_configuration.py
@@ -202,6 +202,7 @@ def __kubespray(k8s_conf, base_pb_vars):
             'PROJ_ARTIFACT_DIR': config_utils.get_project_artifact_dir(
                 k8s_conf),
         }
+        pb_vars.update(config_utils.get_proxy_dict(k8s_conf))
         ansible_utils.apply_playbook(consts.KUBERNETES_NEW_INVENTORY,
                                      variables=pb_vars)
 


Test patch:
diff --git a/tests/common/utils/config_utils.py b/tests/common/utils/config_utils.py
index 068369c..bb9a64f 100644
--- a/tests/common/utils/config_utils.py
+++ b/tests/common/utils/config_utils.py
@@ -194,3 +194,10 @@ class ConfigUtilsTests(unittest.TestCase):
 
         this_cfg[consts.K8S_KEY][consts.METRICS_SERVER_KEY] = None
         self.assertFalse(config_utils.is_metrics_server_enabled(self.config))
+
+    def test_get_password(self):
+        node_confs = config_utils.get_node_configs(self.config)
+        for node_conf in node_confs:
+            password = config_utils.get_node_password(
+                self.config, node_conf[consts.HOST_KEY][consts.HOSTNAME_KEY])
+            self.assertEqual('password', password)