AWSにKubernetesクラスタをTerraformで構築する（デジャヴ）

#AWS上にマスターnodeをマルチゾーン化したKubernetesクラスタを構築する
イメージ図
![image.png](https://qiita-image-store.s3.ap-northeast-1.amazonaws.com/0/545831/b7432c9f-d53d-df5d-6898-14b113c17b2c.png)

マスターnodeをマルチゾーン化し、LBで束ねて高可用性なKubernetesクラスタを構築するのを、Terraformで自動化する。
Kubernetesのnodeは、あらかじめkubeadm,kubectl,kubelet,docker-ceをインストールしたイメージを作成しておく。

#作業場所
Teraformを実行する端末は何でもよく、手元のMacやLinux PC、Azure等パブリッククラウドのLinuxインスタンスでも構わない。今回は自宅のLinux PC(Ubuntu 18.04)から実行している。Terraformのバージョンは次の通り。

```sh
root@liva-z:~# terraform version
Terraform v0.12.21
```

#各種tfファイル

```tf:main.tf
# AWS Providerの設定
provider "aws" {
  region = var.aws_region
}

# 有効なゾーンを問い合わせlocal.all_zonesで参照する
data "aws_availability_zones" "available" {
  state = "available"
}

# local変数の設定
locals {
  all_zones        = data.aws_availability_zones.available.names
}

# sshキーペアの登録
resource "aws_key_pair" "deployer" {
  key_name   = "${var.cluster_name}-deployer-key"
  public_key = file(var.ssh_public_key_file)
}

# セキュリティグループの作成(common)
resource "aws_security_group" "common" {
  name        = "${var.cluster_name}-common"
  description = "cluster common rules"
  vpc_id      = aws_vpc.vpc.id
  tags        = map("Name", "${var.cluster_name}-sg-common", )
  ingress {
    from_port   = var.ssh_port
    to_port     = var.ssh_port
    protocol    = "tcp"
    cidr_blocks = ["0.0.0.0/0"]
  }

  ingress {
    from_port = 0
    to_port   = 0
    protocol  = "-1"
    self      = true
  }

  egress {
    from_port   = 0
    to_port     = 0
    protocol    = "-1"
    cidr_blocks = ["0.0.0.0/0"]
  }
}

# セキュリティグループの作成(master)
resource "aws_security_group" "master" {
  name        = "${var.cluster_name}-masters"
  description = "cluster masters"
  vpc_id      = aws_vpc.vpc.id
  tags        = map("Name", "${var.cluster_name}-sg-master", )

  ingress {
    from_port   = 6443
    to_port     = 6443
    protocol    = "tcp"
    cidr_blocks = ["0.0.0.0/0"]
  }
}

# VPCの作成
resource "aws_vpc" "vpc" {
  cidr_block = var.vpc_cidr
  tags       = map("Name", "${var.cluster_name}-vpc", )
}

# Subnetの作成(public)
resource "aws_subnet" "public" {
  count                   = length(local.all_zones)
  vpc_id                  = aws_vpc.vpc.id
  cidr_block              = cidrsubnet(var.vpc_cidr, var.subnet_netmask_bits, var.subnet_offset + count.index)
  availability_zone       = local.all_zones[count.index]
  map_public_ip_on_launch = true
  tags                    = map("Name", "${var.cluster_name}_public_${local.all_zones[count.index]}", )
}

# Subnetの作成(private)
resource "aws_subnet" "private" {
  count                   = length(local.all_zones)
  vpc_id                  = aws_vpc.vpc.id
  cidr_block              = cidrsubnet(var.vpc_cidr, var.subnet_netmask_bits, var.subnet_offset + length(local.all_zones) + count.index)
  availability_zone       = local.all_zones[count.index]
  map_public_ip_on_launch = false
  tags                    = map("Name", "${var.cluster_name}_private_${local.all_zones[count.index]}", )
}

# eipの作成
resource "aws_eip" "nat" {
  vpc = true
}

# インターネットゲートウェイの作成
resource "aws_internet_gateway" "igw" {
  vpc_id = aws_vpc.vpc.id
  tags   = map("Name", "${var.cluster_name}_igw", )
}

# NATゲートウェイの作成
resource "aws_nat_gateway" "ngw" {
  allocation_id = aws_eip.nat.id
  subnet_id     = aws_subnet.public.0.id
  tags          = map("Name", "${var.cluster_name}_nat_gateway-ngw", )
}

# 以下ルートテーブルの作成
resource "aws_route_table" "public" {
  vpc_id = aws_vpc.vpc.id
  tags   = map("Name", "${var.cluster_name}_public_route_table", )

  route {
    cidr_block = "0.0.0.0/0"
    gateway_id = aws_internet_gateway.igw.id
  }
}

resource "aws_route_table" "private" {
  vpc_id = aws_vpc.vpc.id
  tags   = map("Name", "${var.cluster_name}_private_route_table", )

  route {
    cidr_block     = "0.0.0.0/0"
    nat_gateway_id = aws_nat_gateway.ngw.id
  }
}

resource "aws_route_table_association" "public" {
  count          = length(local.all_zones)
  subnet_id      = element(aws_subnet.public.*.id, count.index)
  route_table_id = aws_route_table.public.id
}

resource "aws_route_table_association" "private" {
  count          = length(local.all_zones)
  subnet_id      = element(aws_subnet.private.*.id, count.index)
  route_table_id = aws_route_table.private.id
}

# ロードバランサーの作成
resource "aws_lb" "master" {
  name               = "${var.cluster_name}-api-lb"
  internal           = false
  load_balancer_type = "network"
  subnets            = aws_subnet.public.*.id
  tags               = map("Name", "${var.cluster_name}-master", )
}

# リスナーの作成
resource "aws_lb_listener" "master_api" {
  load_balancer_arn = aws_lb.master.arn
  port              = 6443
  protocol          = "TCP"

  default_action {
    target_group_arn = aws_lb_target_group.master_api.arn
    type             = "forward"
  }
}

resource "aws_lb_target_group" "master_api" {
  name     = "${var.cluster_name}-api"
  port     = 6443
  protocol = "TCP"
  vpc_id   = aws_vpc.vpc.id
}

# インスタンスのアタッチ
resource "aws_lb_target_group_attachment" "master_api" {
  count            = length(local.all_zones)
  target_group_arn = aws_lb_target_group.master_api.arn
  target_id        = element(aws_instance.master.*.id, count.index)
  port             = 6443
}

# Kubernetesマスターノードの作成(ゾーン数分)
resource "aws_instance" "master" {
  count                       = length(local.all_zones)
  tags                        = map("Name", "${var.cluster_name}-master-${count.index + 1}", )
  instance_type               = "t3.medium"
  ami                         = var.node_image
  key_name                    = aws_key_pair.deployer.key_name
  vpc_security_group_ids      = [aws_security_group.common.id, aws_security_group.master.id]
  availability_zone           = local.all_zones[count.index % length(local.all_zones)]
  subnet_id                   = element(aws_subnet.private.*.id, count.index % length(local.all_zones))
  associate_public_ip_address = false
  ebs_optimized               = true
  root_block_device {
    volume_type = "gp2"
    volume_size = 30
  }
}

# Kubernetesワーカーノードの作成(6つ)
resource "aws_instance" "workers" {
  count                       = 6
  tags                        = map("Name", "${var.cluster_name}-worker-${count.index + 1}", )
  instance_type               = "t3.medium"
  ami                         = var.node_image
  key_name                    = aws_key_pair.deployer.key_name
  vpc_security_group_ids      = [aws_security_group.common.id]
  availability_zone           = local.all_zones[count.index % length(local.all_zones)]
  subnet_id                   = element(aws_subnet.private.*.id, count.index % length(local.all_zones))
  associate_public_ip_address = false
  root_block_device {
    volume_type = "gp2"
    volume_size = 30
  }
}

# 踏み台サーバの作成
resource "aws_instance" "bastion" {
  tags                        = map("Name", "${var.cluster_name}-bastion", )
  instance_type               = "t3.nano"
  ami                         = var.bastion_image
  key_name                    = aws_key_pair.deployer.key_name
  vpc_security_group_ids      = [aws_security_group.common.id]
  availability_zone           = local.all_zones[0]
  subnet_id                   = aws_subnet.public.0.id
  associate_public_ip_address = true
  root_block_device {
    volume_type = "gp2"
    volume_size = 30
  }
}
```

```tf:variables.tf
# 共通設定
variable "cluster_name" {
  default = "fabric"
}

variable "ssh_public_key_file" {
  default = "~/.ssh/id_rsa.pub"
}

variable "ssh_port" {
  default = 22
}

variable "vpc_cidr" {
  default = "10.10.0.0/16"
}

variable "subnet_offset" {
  default = 0
}

variable "subnet_netmask_bits" {
  default = 8
}

# クラウドベンダー依存部分
variable "aws_region" {
  default = "ap-northeast-1"
}

variable "node_image" {
  default = "ami-05b96e7df14c3437a"  # kubernetes構築済みイメージ
}

variable "bastion_image" {
  default = "ami-07f4cb4629342979c"  # 素のUbuntu 18.04
}
```

```tf:output.tf
output "kubeadm_api" {
  value = {
    endpoint = aws_lb.master.dns_name
  }
}

output "kubernetes_bastion" {
  value = {
    bastion_ip = aws_instance.bastion.public_ip
  }
}

output "kubernetes_master" {
  value = {
    master_ip = aws_instance.master.*.private_ip
  }
}

output "kubernetes_worker" {
  value = {
    worker_ip = aws_instance.workers.*.private_ip
  }
}
```

Alicloudとの大きな違いは、LBのみだとendpointが作成できない点である。インターネットゲートウェイとNATゲートウェイを設定する必要がある。

#terraform applyを実行

```tf
Apply complete! Resources: 37 added, 0 changed, 0 destroyed.

Outputs:

kubeadm_api = {
  "endpoint" = "fabric-api-lb-9ae06cf14dc4c15d.elb.ap-northeast-1.amazonaws.com"
}
kubernetes_bastion = {
  "bastion_ip" = "54.178.4.137"
}
kubernetes_master = {
  "master_ip" = [
    "10.10.3.147",
    "10.10.4.161",
    "10.10.5.111",
  ]
}
kubernetes_worker = {
  "worker_ip" = [
    "10.10.3.170",
    "10.10.4.192",
    "10.10.5.245",
    "10.10.3.182",
    "10.10.4.246",
    "10.10.5.244",
  ]
}
```
Alicloudのコンソールでインスタンスを確認。ちゃんとマスターnodeがゾーンで分かれている。
![image.png](https://qiita-image-store.s3.ap-northeast-1.amazonaws.com/0/545831/c2e0927e-285e-8535-c83e-8b67d41b1940.png)

LB(NLB)にマスターnodeが3つぶら下がっているのがわかる。
![image.png](https://qiita-image-store.s3.ap-northeast-1.amazonaws.com/0/545831/b50aaa0e-e470-5d6d-386b-d41695f49686.png)

#Kubernetesの設定
これでインフラが構築できたので、ここからKubernetes環境を設定していく。基本的には[この][link2]ページのMaster nodes:以下を実行すれば良い。
[link2]:https://kruschecompany.com/kubernetes-1-15-whats-new/

あとすべてのnodeであらかじめ、

```sh
swapoff -a
export KUBECONFIG=/etc/kubernetes/admin.conf
```
してある。というかマシンイメージの.bashrcに書いてある。
1つのマスターnodeへ入って
/etc/kubernetes/kubeadm/kubeadm-config.yamlを次の通り記述。

```yaml:kubeadm-config.yaml
apiVersion: kubeadm.k8s.io/v1beta1
kind: ClusterConfiguration
kubernetesVersion: stable
controlPlaneEndpoint: "fabric-api-lb-9ae06cf14dc4c15d.elb.ap-northeast-1.amazonaws.com:6443"
```
controlPlaneEndpointには、Outputs:のkubeadm_api.endpointを設定する。

kubeadm initを実行。

```sh
root@fabric-master-2:~# kubeadm init --config=/etc/kubernetes/kubeadm/kubeadm-config.yaml --upload-certs
- 途中のkubeadmのメッセージは省略 -
Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

You can now join any number of the control-plane node running the following command on each as root:

  kubeadm join fabric-api-lb-9ae06cf14dc4c15d.elb.ap-northeast-1.amazonaws.com:6443 --token 6cwazk.a4669gp7f4uj4srh \
    --discovery-token-ca-cert-hash sha256:75bf6bfbeb01315f7b4ddcd49e0569bcd0f86bde0a35d40531cf4c221dc2f1e2 \
    --control-plane --certificate-key 76e2af2667fdaf453312df5be62ad1622a77d0e3b1d8a9130e37741cbfa72a32

Please note that the certificate-key gives access to cluster sensitive data, keep it secret!
As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use
"kubeadm init phase upload-certs --upload-certs" to reload certs afterward.

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join fabric-api-lb-9ae06cf14dc4c15d.elb.ap-northeast-1.amazonaws.com:6443 --token 6cwazk.a4669gp7f4uj4srh \
    --discovery-token-ca-cert-hash sha256:75bf6bfbeb01315f7b4ddcd49e0569bcd0f86bde0a35d40531cf4c221dc2f1e2
```

残り2つのマスターnodeでは、kubeadm initの出力にあるcontrol-plane node用のjoinコマンドを実行する。
6つのワーカーnodeでは、worker node用のjoinコマンドをひたすら実行する。

#できた環境を確認する
terraformを実行したLinux PCへ、Kubernetes環境を一式（たぶんkubectlだけで良いと思う）インストール。
最初にkubeadm initしたマスターnodeから/etc/kubernetes/admin.confをコピーする。
これでkubectlが利用できるようになる。

```sh
export KUBECONFIG=/etc/kubernetes/admin.conf
```
を忘れずに。

```sh
root@liva-z:fabric-dev# kubectl get node -o wide
NAME             STATUS   ROLES    AGE     VERSION   INTERNAL-IP   EXTERNAL-IP   OS-IMAGE             KERNEL-VERSION    CONTAINER-RUNTIME
ip-10-10-3-147   Ready    master   19m     v1.17.2   10.10.3.147   <none>        Ubuntu 18.04.3 LTS   4.15.0-1057-aws   docker://18.9.9
ip-10-10-3-170   Ready    <none>   13m     v1.17.2   10.10.3.170   <none>        Ubuntu 18.04.3 LTS   4.15.0-1057-aws   docker://18.9.9
ip-10-10-3-182   Ready    <none>   10m     v1.17.2   10.10.3.182   <none>        Ubuntu 18.04.3 LTS   4.15.0-1057-aws   docker://18.9.9
ip-10-10-4-161   Ready    master   16m     v1.17.2   10.10.4.161   <none>        Ubuntu 18.04.3 LTS   4.15.0-1057-aws   docker://18.9.9
ip-10-10-4-192   Ready    <none>   11m     v1.17.2   10.10.4.192   <none>        Ubuntu 18.04.3 LTS   4.15.0-1057-aws   docker://18.9.9
ip-10-10-4-246   Ready    <none>   9m46s   v1.17.2   10.10.4.246   <none>        Ubuntu 18.04.3 LTS   4.15.0-1057-aws   docker://18.9.9
ip-10-10-5-111   Ready    master   14m     v1.17.2   10.10.5.111   <none>        Ubuntu 18.04.3 LTS   4.15.0-1057-aws   docker://18.9.9
ip-10-10-5-244   Ready    <none>   8m58s   v1.17.2   10.10.5.244   <none>        Ubuntu 18.04.3 LTS   4.15.0-1057-aws   docker://18.9.9
ip-10-10-5-245   Ready    <none>   11m     v1.17.2   10.10.5.245   <none>        Ubuntu 18.04.3 LTS   4.15.0-1057-aws   docker://18.9.9
```
マスターnodeが3台、ワーカーnodeが6台確認でき、すべてReadyになっている。

```sh
root@liva-z:fabric-dev# kubectl get pod -A -o wide
NAMESPACE     NAME                                       READY   STATUS    RESTARTS   AGE     IP              NODE             NOMINATED NODE   READINESS GATES
kube-system   calico-kube-controllers-5c45f5bd9f-ljq95   1/1     Running   0          2m25s   192.168.101.3   ip-10-10-3-147   <none>           <none>
kube-system   calico-node-6hpbg                          1/1     Running   0          2m25s   10.10.3.170     ip-10-10-3-170   <none>           <none>
kube-system   calico-node-9xmjj                          1/1     Running   0          2m25s   10.10.3.182     ip-10-10-3-182   <none>           <none>
kube-system   calico-node-h5xrp                          1/1     Running   0          2m25s   10.10.5.244     ip-10-10-5-244   <none>           <none>
kube-system   calico-node-jkkmb                          1/1     Running   0          2m25s   10.10.4.192     ip-10-10-4-192   <none>           <none>
kube-system   calico-node-jmxl4                          1/1     Running   0          2m25s   10.10.4.161     ip-10-10-4-161   <none>           <none>
kube-system   calico-node-lc7kt                          1/1     Running   0          2m25s   10.10.5.111     ip-10-10-5-111   <none>           <none>
kube-system   calico-node-njh55                          1/1     Running   0          2m25s   10.10.3.147     ip-10-10-3-147   <none>           <none>
kube-system   calico-node-pzr6q                          1/1     Running   0          2m25s   10.10.4.246     ip-10-10-4-246   <none>           <none>
kube-system   calico-node-sqhc8                          1/1     Running   0          2m25s   10.10.5.245     ip-10-10-5-245   <none>           <none>
kube-system   coredns-6955765f44-bxmtc                   1/1     Running   0          19m     192.168.101.2   ip-10-10-3-147   <none>           <none>
kube-system   coredns-6955765f44-l8rjh                   1/1     Running   0          19m     192.168.101.1   ip-10-10-3-147   <none>           <none>
kube-system   etcd-ip-10-10-3-147                        1/1     Running   0          20m     10.10.3.147     ip-10-10-3-147   <none>           <none>
kube-system   etcd-ip-10-10-4-161                        1/1     Running   0          17m     10.10.4.161     ip-10-10-4-161   <none>           <none>
kube-system   etcd-ip-10-10-5-111                        1/1     Running   0          15m     10.10.5.111     ip-10-10-5-111   <none>           <none>
kube-system   kube-apiserver-ip-10-10-3-147              1/1     Running   0          20m     10.10.3.147     ip-10-10-3-147   <none>           <none>
kube-system   kube-apiserver-ip-10-10-4-161              1/1     Running   0          17m     10.10.4.161     ip-10-10-4-161   <none>           <none>
kube-system   kube-apiserver-ip-10-10-5-111              1/1     Running   0          14m     10.10.5.111     ip-10-10-5-111   <none>           <none>
kube-system   kube-controller-manager-ip-10-10-3-147     1/1     Running   1          20m     10.10.3.147     ip-10-10-3-147   <none>           <none>
kube-system   kube-controller-manager-ip-10-10-4-161     1/1     Running   0          17m     10.10.4.161     ip-10-10-4-161   <none>           <none>
kube-system   kube-controller-manager-ip-10-10-5-111     1/1     Running   0          14m     10.10.5.111     ip-10-10-5-111   <none>           <none>
kube-system   kube-proxy-62zdx                           1/1     Running   0          12m     10.10.5.245     ip-10-10-5-245   <none>           <none>
kube-system   kube-proxy-dqb4v                           1/1     Running   0          19m     10.10.3.147     ip-10-10-3-147   <none>           <none>
kube-system   kube-proxy-ktpt8                           1/1     Running   0          14m     10.10.3.170     ip-10-10-3-170   <none>           <none>
kube-system   kube-proxy-mmfcr                           1/1     Running   0          11m     10.10.3.182     ip-10-10-3-182   <none>           <none>
kube-system   kube-proxy-nqjbj                           1/1     Running   0          9m54s   10.10.5.244     ip-10-10-5-244   <none>           <none>
kube-system   kube-proxy-rjdmw                           1/1     Running   0          15m     10.10.5.111     ip-10-10-5-111   <none>           <none>
kube-system   kube-proxy-sw7pg                           1/1     Running   0          10m     10.10.4.246     ip-10-10-4-246   <none>           <none>
kube-system   kube-proxy-t6km4                           1/1     Running   0          17m     10.10.4.161     ip-10-10-4-161   <none>           <none>
kube-system   kube-proxy-xn2nl                           1/1     Running   0          12m     10.10.4.192     ip-10-10-4-192   <none>           <none>
kube-system   kube-scheduler-ip-10-10-3-147              1/1     Running   1          20m     10.10.3.147     ip-10-10-3-147   <none>           <none>
kube-system   kube-scheduler-ip-10-10-4-161              1/1     Running   0          17m     10.10.4.161     ip-10-10-4-161   <none>           <none>
kube-system   kube-scheduler-ip-10-10-5-111              1/1     Running   0          14m     10.10.5.111     ip-10-10-5-111   <none>           <none>
```
kube-system podも問題なく動作している。
eipを3つ取ってゾーン毎にNATゲートウェイへ登録すれば、もっと可用性が上がるだろう（たぶん）。
↓変更例。

```tf:main.tf
# eipの作成
resource "aws_eip" "nat" {
  count = length(local.all_zones)
  vpc   = true
}

# NATゲートウェイの作成
resource "aws_nat_gateway" "ngw" {
  count         = length(local.all_zones)
  allocation_id = element(aws_eip.nat.*.id, count.index)
  subnet_id     = element(aws_subnet.public.*.id, count.index)
}

resource "aws_route_table" "private" {
  vpc_id = aws_vpc.vpc.id
  tags   = map("Name", "${var.cluster_name}_private_route_table", )
  route {
    cidr_block     = "0.0.0.0/0"
    nat_gateway_id = aws_nat_gateway.ngw.*.id
  }
}
```

先に[Alicloudの記事][link3]を書いたのでAWSでも同等のことが出来ることを示してみた。デジャヴを感じたかもしれない。。
[link3]:https://qiita.com/settembre21/items/87b4285dc66cafb57cef
Azureでも出来ると思うがマルチゾーンの考え方が異なるので、その辺りのコードが違ってくるであろう。