Mac で Kubernetes クラスタを構築する

お手軽に試すには minikube などの手段がありますが、本番環境に近いマルチノードがほしいと思い、構築してみました。
[Kubernetes the hard way](https://github.com/kelseyhightower/kubernetes-the-hard-way) のように根本を理解するための難しい手順もありますが、簡単に作れる方法で試そうと思います。

ここでは、
[Kubernetes Setup Using Ansible and Vagrant](https://kubernetes.io/blog/2019/03/15/kubernetes-setup-using-ansible-and-vagrant/)
に従って作成します。が、すんなりはいかずいくつかエラーの対処をしました。
また、Ubuntu のバージョンや、各ノードの memory/CPU を変更しています。

# 実行環境とツール

macOS Catalina 10.15.4
Vagrant 2.2.7
Ansible 2.9.6

# クラスタ構成

Ubuntu 18.04
kubernetes v1.18.3
master(x1): memory 1536MB, CPU 4
node(x2): memory 2048MB, CPU 4

# 構築する

### 必要なファイルを用意する

<details><summary>./Vagrantfile</summary><div>

```
IMAGE_NAME = "generic/ubuntu1804"
N = 2

Vagrant.configure("2") do |config|
    config.ssh.insert_key = false

    config.vm.provider "virtualbox" do |v|
        v.cpus = 4
        v.gui = false
    end
      
    config.vm.define "k8s-master" do |master|
        master.vm.box = IMAGE_NAME
        master.vm.network "private_network", ip: "192.168.50.10"
        master.vm.hostname = "k8s-master"
        master.vm.provision "ansible" do |ansible|
            ansible.playbook = "kubernetes-setup/master-playbook.yml"
            ansible.extra_vars = {
                node_ip: "192.168.50.10",
            }
        end
        master.vm.provider "virtualbox" do |v| 
          v.customize ["modifyvm", :id, "--memory", 1536 ] 
        end
    end

    (1..N).each do |i|
        config.vm.define "node-#{i}" do |node|
            node.vm.box = IMAGE_NAME
            node.vm.network "private_network", ip: "192.168.50.#{i + 10}"
            node.vm.hostname = "node-#{i}"
            node.vm.provision "ansible" do |ansible|
                ansible.playbook = "kubernetes-setup/node-playbook.yml"
                ansible.extra_vars = {
                    node_ip: "192.168.50.#{i + 10}",
                }
            end
            node.vm.provider "virtualbox" do |v| 
              v.customize ["modifyvm", :id, "--memory", 2048 ] 
            end
        end
    end
end
```
</div></details>

<details><summary>./kubernetes-setup/master-playbook.yml</summary><div>

```
---
- hosts: all
  become: true
  tasks:
  - name: Install packages that allow apt to be used over HTTPS
    apt:
      name: "{{ packages }}"
      state: present
      update_cache: yes
    vars:
      packages:
      - apt-transport-https
      - ca-certificates
      - curl
      - gnupg-agent
      - software-properties-common

  - name: Add an apt signing key for Docker
    apt_key:
      url: https://download.docker.com/linux/ubuntu/gpg
      state: present

  - name: Add apt repository for stable version
    apt_repository:
      repo: deb [arch=amd64] https://download.docker.com/linux/ubuntu xenial stable
      state: present

  - name: Install docker and its dependecies
    apt: 
      name: "{{ packages }}"
      state: present
      update_cache: yes
    vars:
      packages:
      - docker-ce 
      - docker-ce-cli 
      - containerd.io
    notify:
      - docker status

  - name: Add vagrant user to docker group
    user:
      name: vagrant
      group: docker

  - name: Remove swapfile from /etc/fstab
    mount:
      name: "{{ item }}"
      fstype: swap
      state: absent
    with_items:
      - swap
      - none

  - name: Disable swap
    command: swapoff -a
    when: ansible_swaptotal_mb > 0

  - name: Add an apt signing key for Kubernetes
    apt_key:
      url: https://packages.cloud.google.com/apt/doc/apt-key.gpg
      state: present

  - name: Adding apt repository for Kubernetes
    apt_repository:
      repo: deb https://apt.kubernetes.io/ kubernetes-xenial main
      state: present
      filename: kubernetes.list

  - name: Install Kubernetes binaries
    apt: 
      name: "{{ packages }}"
      state: present
      update_cache: yes
    vars:
      packages:
        - kubelet 
        - kubeadm 
        - kubectl

  - name: Configure node ip
    lineinfile:
      create: yes
      path: /etc/default/kubelet
      line: KUBELET_EXTRA_ARGS=--node-ip={{ node_ip }}

  - name: Restart kubelet
    service:
      name: kubelet
      daemon_reload: yes
      state: restarted

  - name: Initialize the Kubernetes cluster using kubeadm
    command: kubeadm init --apiserver-advertise-address="192.168.50.10" --apiserver-cert-extra-sans="192.168.50.10"  --node-name k8s-master --pod-network-cidr=192.168.0.0/16

  - name: Setup kubeconfig for vagrant user
    command: "{{ item }}"
    with_items:
     - mkdir -p /home/vagrant/.kube
     - cp -i /etc/kubernetes/admin.conf /home/vagrant/.kube/config
     - chown vagrant:vagrant /home/vagrant/.kube/config

  - name: Install calico pod network
    become: false
    command: kubectl create -f https://docs.projectcalico.org/manifests/calico.yaml

  - name: Generate join command
    command: kubeadm token create --print-join-command
    register: join_command

  - name: Copy join command to local file
    become: false
    local_action: copy content="{{ join_command.stdout_lines[0] }}" dest="./join-command"

  handlers:
    - name: docker status
      service: name=docker state=started
```
</div></details>

<details><summary>./kubernetes-setup/node-playbook.yml</summary><div>

```
---
- hosts: all
  become: true
  tasks:
  - name: Install packages that allow apt to be used over HTTPS
    apt:
      name: "{{ packages }}"
      state: present
      update_cache: yes
    vars:
      packages:
      - apt-transport-https
      - ca-certificates
      - curl
      - gnupg-agent
      - software-properties-common

  - name: Add an apt signing key for Docker
    apt_key:
      url: https://download.docker.com/linux/ubuntu/gpg
      state: present

  - name: Add apt repository for stable version
    apt_repository:
      repo: deb [arch=amd64] https://download.docker.com/linux/ubuntu xenial stable
      state: present

  - name: Install docker and its dependecies
    apt: 
      name: "{{ packages }}"
      state: present
      update_cache: yes
    vars:
      packages:
      - docker-ce 
      - docker-ce-cli 
      - containerd.io
    notify:
      - docker status

  - name: Add vagrant user to docker group
    user:
      name: vagrant
      group: docker

  - name: Remove swapfile from /etc/fstab
    mount:
      name: "{{ item }}"
      fstype: swap
      state: absent
    with_items:
      - swap
      - none

  - name: Disable swap
    command: swapoff -a
    when: ansible_swaptotal_mb > 0

  - name: Add an apt signing key for Kubernetes
    apt_key:
      url: https://packages.cloud.google.com/apt/doc/apt-key.gpg
      state: present

  - name: Adding apt repository for Kubernetes
    apt_repository:
      repo: deb https://apt.kubernetes.io/ kubernetes-xenial main
      state: present
      filename: kubernetes.list

  - name: Install Kubernetes binaries
    apt: 
      name: "{{ packages }}"
      state: present
      update_cache: yes
    vars:
      packages:
        - kubelet 
        - kubeadm 
        - kubectl

  - name: Configure node ip
    lineinfile:
      create: yes
      path: /etc/default/kubelet
      line: KUBELET_EXTRA_ARGS=--node-ip={{ node_ip }}

  - name: Restart kubelet
    service:
      name: kubelet
      daemon_reload: yes
      state: restarted

  - name: Copy the join command to server location
    copy: src=join-command dest=/tmp/join-command.sh mode=0777

  - name: Join the node to cluster
    command: sh /tmp/join-command.sh

  handlers:
    - name: docker status
      service: name=docker state=started
```
</div></details>

### 実行

```
$ vagrant up
```

### ログインして kubectl コマンドを打ってみる

```
% vagrant ssh k8s-master
Last login: Fri May 22 15:18:20 2020 from 10.0.2.2
vagrant@k8s-master:~$ kubectl get node
NAME         STATUS   ROLES    AGE     VERSION
k8s-master   Ready    master   13m     v1.18.3
node-1       Ready    <none>   6m58s   v1.18.3
node-2       Ready    <none>   2m21s   v1.18.3
vagrant@k8s-master:~$ kubectl get pod --all-namespaces
NAMESPACE     NAME                                       READY   STATUS    RESTARTS   AGE
kube-system   calico-kube-controllers-789f6df884-t476z   1/1     Running   0          12m
kube-system   calico-node-8qqpk                          1/1     Running   0          7m
kube-system   calico-node-jnb8w                          1/1     Running   0          2m23s
kube-system   calico-node-lss4w                          1/1     Running   0          12m
kube-system   coredns-66bff467f8-82l2z                   1/1     Running   0          12m
kube-system   coredns-66bff467f8-sz28s                   1/1     Running   0          12m
kube-system   etcd-k8s-master                            1/1     Running   0          13m
kube-system   kube-apiserver-k8s-master                  1/1     Running   0          13m
kube-system   kube-controller-manager-k8s-master         1/1     Running   1          13m
kube-system   kube-proxy-6gtft                           1/1     Running   0          2m23s
kube-system   kube-proxy-g4dgs                           1/1     Running   0          12m
kube-system   kube-proxy-j6drw                           1/1     Running   0          7m
kube-system   kube-scheduler-k8s-master                  1/1     Running   2          13m
vagrant@k8s-master:~$ 
```

無事動いているようです！
kubeadm 便利..

## 対応したエラーと詳細

(1) Configure node ip

```
TASK [Configure node ip] *******************************************************
fatal: [k8s-master]: FAILED! => {"changed": false, "msg": "Destination /etc/default/kubelet does not exist !", "rc": 257}
```
`/etc/default/kubelet` が存在しなかった。
`/etc/default/kubelet` は kubelet の Environment file だそうです。
[The kubelet drop-in file for systemd](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/kubelet-integration/#the-kubelet-drop-in-file-for-systemd)
-> create するように修正

(2) Install calico pod network

```
TASK [Install calico pod network] **********************************************
fatal: [k8s-master]: FAILED! => {"changed": true, "cmd": ["kubectl", "create", "-f", "https://docs.projectcalico.org/v3.4/getting-started/kubernetes/installation/hosted/calico.yaml"], "delta": "0:00:01.641120", "end": "2020-05-22 14:12:14.999861", "msg": "non-zero return code", "rc": 1, "start": "2020-05-22 14:12:13.358741", "stderr": "unable to recognize \"https://docs.projectcalico.org/v3.4/getting-started/kubernetes/installation/hosted/calico.yaml\": no matches for kind \"DaemonSet\" in version \"extensions/v1beta1\"\nunable to recognize \"https://docs.projectcalico.org/v3.4/getting-started/kubernetes/installation/hosted/calico.yaml\": no matches for kind \"Deployment\" in version \"extensions/v1beta1\"", "stderr_lines": ["unable to recognize \"https://docs.projectcalico.org/v3.4/getting-started/kubernetes/installation/hosted/calico.yaml\": no matches for kind \"DaemonSet\" in version \"extensions/v1beta1\"", "unable to recognize \"https://docs.projectcalico.org/v3.4/getting-started/kubernetes/installation/hosted/calico.yaml\": no matches for kind \"Deployment\" in version \"extensions/v1beta1\""], "stdout": "configmap/calico-config created\nsecret/calico-etcd-secrets created\nserviceaccount/calico-node created\nserviceaccount/calico-kube-controllers created\nclusterrole.rbac.authorization.k8s.io/calico-kube-controllers created\nclusterrolebinding.rbac.authorization.k8s.io/calico-kube-controllers created\nclusterrole.rbac.authorization.k8s.io/calico-node created\nclusterrolebinding.rbac.authorization.k8s.io/calico-node created", "stdout_lines": ["configmap/calico-config created", "secret/calico-etcd-secrets created", "serviceaccount/calico-node created", "serviceaccount/calico-kube-controllers created", "clusterrole.rbac.authorization.k8s.io/calico-kube-controllers created", "clusterrolebinding.rbac.authorization.k8s.io/calico-kube-controllers created", "clusterrole.rbac.authorization.k8s.io/calico-node created", "clusterrolebinding.rbac.authorization.k8s.io/calico-node created"]}
```
どうやら今回の Kubernetes のバージョンで扱えなくなった記述がされているようでした。
`apiVersion: extensions/v1beta1` は kuberenetes 1.16 で削除された模様。
[Deprecated APIs Removed In 1.16: Here’s What You Need To Know](https://kubernetes.io/blog/2019/07/18/api-deprecations-in-1-16/)
-> 最新の yaml に変更

(3) Join the node to cluster

```
TASK [Join the node to cluster] ************************************************
fatal: [node-1]: FAILED! => {"changed": true, "cmd": ["sh", "/tmp/join-command.sh"], "delta": "0:05:08.334251", "end": "2020-05-22 15:06:29.580596", "msg": "non-zero return code", "rc": 1, "start": "2020-05-22 15:01:21.246345", "stderr": "W0522 15:01:21.328886   26793 join.go:346] [preflight] WARNING: JoinControlPane.controlPlane settings will be ignored when control-plane flag is not set.\n\t[WARNING IsDockerSystemdCheck]: detected \"cgroupfs\" as the Docker cgroup driver. The recommended driver is \"systemd\". Please follow the guide at https://kubernetes.io/docs/setup/cri/\nerror execution phase preflight: couldn't validate the identity of the API Server: Get https://192.168.50.10:6443/api/v1/namespaces/kube-public/configmaps/cluster-info?timeout=10s: net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)\nTo see the stack trace of this error execute with --v=5 or higher", "stderr_lines": ["W0522 15:01:21.328886   26793 join.go:346] [preflight] WARNING: JoinControlPane.controlPlane settings will be ignored when control-plane flag is not set.", "\t[WARNING IsDockerSystemdCheck]: detected \"cgroupfs\" as the Docker cgroup driver. The recommended driver is \"systemd\". Please follow the guide at https://kubernetes.io/docs/setup/cri/", "error execution phase preflight: couldn't validate the identity of the API Server: Get https://192.168.50.10:6443/api/v1/namespaces/kube-public/configmaps/cluster-info?timeout=10s: net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)", "To see the stack trace of this error execute with --v=5 or higher"], "stdout": "[preflight] Running pre-flight checks", "stdout_lines": ["[preflight] Running pre-flight checks"]}
```

これはよくわかりませんでした。。
全く同じことをしている方がいらっしゃったので参考にさせて頂きました。
[AnsibleとVagrantでKubernetesをセットアップする](https://hi1280.hatenablog.com/entry/2019/09/25/230533)
-> `master-playbook.yml` の [Copy join command to local file] を 非 root 権限で実行

## 参考

- [Kubernetes Setup Using Ansible and Vagrant](https://kubernetes.io/blog/2019/03/15/kubernetes-setup-using-ansible-and-vagrant/)
- [The kubelet drop-in file for systemd](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/kubelet-integration/#the-kubelet-drop-in-file-for-systemd)
- [Deprecated APIs Removed In 1.16: Here’s What You Need To Know](https://kubernetes.io/blog/2019/07/18/api-deprecations-in-1-16/)
- [AnsibleとVagrantでKubernetesをセットアップする](https://hi1280.hatenablog.com/entry/2019/09/25/230533)