CentOS8でkubernetesを動かす。(2019/12時点)

2020/08/22
CentOS8からFireWallのバックエンドがnftableに変わったためCNIのkube-routerが正常に動かないです。
calicoに変更しました。

あとEL7向けのrpmパッケージを使用しない版で書き直した記事はこっちです。 -> <a href=https://qiita.com/ishida0503/items/f3b62b02dec4f6fef42f>書き直した記事はここ</a>

---
2018/12/06 swap無効化を忘れてたので追記


#1.dnf(yum)リポジトリ用意
~~~
# cat << EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
exclude=kube*
EOF
~~~
#2.swap無効化
~~~
# swapoff -a
# sed -i -e '/swap/d' /etc/fstab
~~~

#3.SELinux無効化
~~~
# setenforce 0
# sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config
~~~

#4.カーネルパラメタ設定
~~~
# cat << EOF >  /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF

# sysctl --system
~~~

#5.ipvsadmのインストール
~~~
# dnf install ipvsadm
~~~

#6.firewalldの無効化
~~~
# systemctl stop firewalld
# systemctl disable firewalld
~~~

#7.docker-ceのインストール
##7.1.podman、runcのアンインストール
~~~
# dnf remove podman runc
~~~

##7.2.バイナリパッケージの展開

~~~
# cd /var/tmp
# wget https://download.docker.com/linux/static/stable/x86_64/docker-19.03.5.tgz
# tar xzvf docker-19.03.5.tgz
# chown root:root docker/*
# cp docker/* /usr/bin/
~~~

##7.3.systemd unit作成
~~~
# cat << EOF > /usr/lib/systemd/system/docker.service
[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
BindsTo=containerd.service
After=network-online.target firewalld.service containerd.service
Wants=network-online.target
Requires=docker.socket

[Service]
Type=notify
ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock
ExecReload=/bin/kill -s HUP \$MAINPID
TimeoutSec=0
RestartSec=2
Restart=always

StartLimitBurst=3

StartLimitInterval=60s

LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity

TasksMax=infinity

Delegate=yes

KillMode=process

[Install]
WantedBy=multi-user.target
EOF
~~~

~~~
# cat << EOF > /usr/lib/systemd/system/docker.socket
[Unit]
Description=Docker Socket for the API
PartOf=docker.service

[Socket]
ListenStream=/var/run/docker.sock
SocketMode=0660
SocketUser=root
SocketGroup=docker

[Install]
WantedBy=sockets.target
EOF
~~~

~~~
# cat << EOF >/usr/lib/systemd/system/containerd.service
[Unit]
Description=containerd container runtime
Documentation=https://containerd.io
After=network.target

[Service]
ExecStartPre=-/sbin/modprobe overlay
ExecStart=/usr/bin/containerd
KillMode=process
Delegate=yes
LimitNOFILE=1048576
LimitNPROC=infinity
LimitCORE=infinity
TasksMax=infinity

[Install]
WantedBy=multi-user.target
EOF
~~~

##7.4.dockerグループ作成
~~~
# groupadd -g 2000 docker
~~~

##7.5.起動
~~~
# systemctl daemon-reload
# systemctl enable containerd.service
# systemctl enable docker.socket
# systemctl enable docker.service
# systemctl restart docker
~~~

#8.kubernetesのインストール
##8.1.インストール
~~~
# dnf install -y kubelet kubeadm kubectl --disableexcludes=kubernetes
~~~

##8.2.cgroup-driver設定
~~~
# cat << EOF > /etc/sysconfig/kubelet
KUBELET_EXTRA_ARGS="--cgroup-driver=cgroupfs"
EOF
~~~

##8.3.起動
~~~
# systemctl enable kubelet
# systemctl restart kubelet
~~~

#9.初期化
~~~
# kubeadm init --pod-network-cidr=10.244.0.0/16 --service-cidr=10.0.0.0/16
~~~

#10.config設定
~~~
# mkdir $HOME/.kube/
# sudo cp /etc/kubernetes/admin.conf $HOME/.kube/config
# sudo chown $(id -u):$(id -g) $HOME/.kube/config
# chmod 600 $HOME/.kube/config
~~~

#11.CNI(~~kube-router~~calico)のインストール
~~# kubectl apply -f https://raw.githubusercontent.com/cloudnativelabs/kube-router/master/daemonset/kubeadm-kuberouter.yaml~~
~~# kubectl apply -f https://raw.githubusercontent.com/cloudnativelabs/kube-router/master/daemonset/kubeadm-kuberouter-all-features.yaml~~

~~~
# export KUBECONFIG=/etc/kubernetes/admin.conf
# curl -L https://docs.projectcalico.org/manifests/calico.yaml | \
sed  '/            - name: CALICO_DISABLE_FILE_LOGGING/i\            # ADD' | \
sed  '/            - name: CALICO_DISABLE_FILE_LOGGING/i\            - name: FELIX_IPTABLESBACKEND' | \
sed  '/            - name: CALICO_DISABLE_FILE_LOGGING/i\              value: Auto'  | \
sed  '/            - name: CALICO_DISABLE_FILE_LOGGING/i\            # ADD' | \
sed  '/            - name: CALICO_DISABLE_FILE_LOGGING/i\            - name: CALICO_IPV4POOL_CIDR' | \
sed  '/            - name: CALICO_DISABLE_FILE_LOGGING/i\              value: \"10.244.0.0\/16\"' | \
kubectl apply -f -
~~~



#12.おまけ
##12.1.マスターノードへのデプロイを有効化
~~~
# kubectl taint nodes --all node-role.kubernetes.io/master-
~~~

##12.2.kubectlでの状態表示
~~~
# kubectl get node -o wide
NAME      STATUS   ROLES    AGE   VERSION   INTERNAL-IP    EXTERNAL-IP   OS-IMAGE                KERNEL-VERSION                CONTAINER-RUNTIME
centos8   Ready    master   37m   v1.16.3   172.16.0.101   <none>        CentOS Linux 8 (Core)   4.18.0-80.11.2.el8_0.x86_64   docker://19.3.5

# kubectl get all -A -o wide
NAMESPACE     NAME                                  READY   STATUS    RESTARTS   AGE   IP             NODE      NOMINATED NODE   READINESS GATES
kube-system   pod/coredns-5644d7b6d9-frjfp          1/1     Running   0          36m   10.244.0.4     centos8   <none>           <none>
kube-system   pod/coredns-5644d7b6d9-r9dnb          1/1     Running   0          36m   10.244.0.3     centos8   <none>           <none>
kube-system   pod/etcd-centos8                      1/1     Running   0          37m   172.16.0.101   centos8   <none>           <none>
kube-system   pod/kube-apiserver-centos8            1/1     Running   0          37m   172.16.0.101   centos8   <none>           <none>
kube-system   pod/kube-controller-manager-centos8   1/1     Running   0          37m   172.16.0.101   centos8   <none>           <none>
kube-system   pod/kube-proxy-tfc7n                  1/1     Running   0          38m   172.16.0.101   centos8   <none>           <none>
kube-system   pod/kube-router-2b4b2                 1/1     Running   0          35m   172.16.0.101   centos8   <none>           <none>
kube-system   pod/kube-scheduler-centos8            1/1     Running   0          37m   172.16.0.101   centos8   <none>           <none>

NAMESPACE     NAME                 TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)                  AGE   SELECTOR
default       service/kubernetes   ClusterIP   10.0.0.1     <none>        443/TCP                  39m   <none>
kube-system   service/kube-dns     ClusterIP   10.0.0.10    <none>        53/UDP,53/TCP,9153/TCP   39m   k8s-app=kube-dns

NAMESPACE     NAME                         DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE   NODE SELECTOR                 AGE   CONTAINERS    IMAGES                                  SELECTOR
kube-system   daemonset.apps/kube-proxy    1         1         1       1            1           beta.kubernetes.io/os=linux   39m   kube-proxy    k8s.gcr.io/kube-proxy:v1.16.3           k8s-app=kube-proxy
kube-system   daemonset.apps/kube-router   1         1         1       1            1           <none>                        38m   kube-router   docker.io/cloudnativelabs/kube-router   k8s-app=kube-router,tier=node

NAMESPACE     NAME                      READY   UP-TO-DATE   AVAILABLE   AGE   CONTAINERS   IMAGES                     SELECTOR
kube-system   deployment.apps/coredns   2/2     2            2           39m   coredns      k8s.gcr.io/coredns:1.6.2   k8s-app=kube-dns

NAMESPACE     NAME                                 DESIRED   CURRENT   READY   AGE   CONTAINERS   IMAGES                     SELECTOR
kube-system   replicaset.apps/coredns-5644d7b6d9   2         2         2       38m   coredns      k8s.gcr.io/coredns:1.6.2   k8s-app=kube-dns,pod-template-hash=5644d7b6d9
~~~