[Podman]Fluentdで収集したログのElasticsearchのIndexをIndex Lifecycle Management(ILM)の管理対象にする

# 環境

Fedora CoreOS 31.20200113.3.1
Podman 1.7.0
Podman Compose 0.1.5
Elasticsearch 7.5.2
Kibana 7.5.2
Fluentd 1.9.3
MariaDB 10.4.12

```
/home/core/dkwork/es
|--docker-compose.yml
|--es
|  |--config
|  |  |--elasticsearch.yml
|  |--data
|--fluentd
|  |--dockerfiles
|  |  |--Dockerfile
|  |--etc
|  |  |--fluent.conf
|  |  |--mysql_template.json
|--kibana
|  |--config
|  |  |--kibana.yml
|--mariadb
|  |--etc
|  |  |--mymariadb.cnf
|  |--log
|  |  |--error.log
|  |  |--general.log
|  |  |--slow.log
```
# 準備

```
$ sudo sed -i --follow-symlinks 's/SELINUX=enforcing/SELINUX=permissive/g' /etc/sysconfig/selinux
$ sudo reboot
$ sudo rpm-ostree install python3 -r
$ sudo pip3 install podman-compose
$ mkdir -p ~/.local/bin
$ curl -o ~/.local/bin/podman-compose https://raw.githubusercontent.com/containers/podman-compose/devel/podman_compose.py
$ chmod +x ~/.local/bin/podman-compose

$ mkdir dkwork/es
$ cd dkwork/es
$ mkdir -p es/data
$ chmod 777 es/data
$ mkdir -p es/config 
$ mkdir -p mariadb/log
$ chmod 777 mariadb/log
$ mkdir -p fluentd/dockerfiles
$ mkdir -p fluentd/etc
$ mkdir -p kibana/config
$ mkdir -p mariadb/etc
```
# ファイル

healthcheckの記述は、Dockerの時のように状態が
表示されなかったのでコメントにしています。

```/home/core/dkwork/docker-compose.yml
version: "3.7"
services:
  elasticsearch:
    container_name: elasticsearch
    image: docker.elastic.co/elasticsearch/elasticsearch:7.5.2
    volumes:
      - ./es/data:/usr/share/elasticsearch/data
      - ./es/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
    ports:
      - 9200:9200
      - 9300:9300
    environment:
      - ES_JAVA_OPTS=-Xms128m -Xmx128m
      - discovery.type=single-node
    restart: always
    #healthcheck:
    #    test: ["CMD-SHELL", "curl --silent --fail localhost:9200/_cluster/health || exit 1"]
    #    interval: 30s
    #    timeout: 30s
    #    retries: 3

  kibana:
    container_name: kibana
    image: docker.elastic.co/kibana/kibana:7.5.2
    ports:
      - 5601:5601
    depends_on:
      - elasticsearch
    restart: always

  fluentd:
    container_name: fluentd
    build: ./fluentd/dockerfiles
    image: fluentd:1.9.3
    ports:
      - 24224:24224
      - 24224:24224/udp
    environment:
      - FLUENTD_CONF=fluent.conf
    volumes:
      - ./fluentd/etc/fluent.conf:/fluentd/etc/fluent.conf
      - ./fluentd/etc/mysql_template.json:/fluentd/etc/mysql_template.json
      - ./mariadb/log:/var/log/mysql
    user: root
    restart: always
    #healthcheck:
    #    test: ["CMD-SHELL", "curl --silent --fail localhost:5601 || exit 1"]
    #    interval: 30s
    #    timeout: 30s
    #    retries: 3

  mariadb:
    container_name: mariadb
    image: mariadb:10.4.12
    ports:
      - 3306:3306
    environment:
      - MYSQL_ROOT_PASSWORD=mariadb
    volumes:
      - ./mariadb/etc:/etc/mysql/conf.d
      - ./mariadb/log:/var/log/mysql
    restart: always
    #healthcheck:
    #    test: ["CMD-SHELL", "mysqladmin -h 'localhost' -u root -pmariadb ping --silent"]
    #    interval: 30s
    #    timeout: 30s
    #    retries: 3
```

```/home/core/dkwork/es/es/config/elasticsearch.yml
cluster.name: "docker-cluster"
network.host: 0.0.0.0
discovery.zen.minimum_master_nodes: 1
```

```:/home/core/dkwork/es/fluentd/dockerfiles/Dockerfile
FROM fluent/fluentd:v1.9.3-debian-1.0

# Use root account to use apt
USER root

# below RUN includes plugin as examples elasticsearch is not required
# you may customize including plugins as you wish
RUN buildDeps="sudo make gcc g++ libc-dev" \
 && apt-get update \
 && apt-get install -y --no-install-recommends $buildDeps \
 && sudo gem install fluent-plugin-elasticsearch -v 4.0.5 \
 && sudo gem install fluent-plugin-mysqlslowquery -v 0.0.9 \
 && sudo gem install elasticsearch-xpack -v 7.5.0 \
 && sudo gem sources --clear-all \
 && SUDO_FORCE_REMOVE=yes \
    apt-get purge -y --auto-remove \
                  -o APT::AutoRemove::RecommendsImportant=false \
                  $buildDeps \
 && rm -rf /var/lib/apt/lists/* \
 && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem

USER fluent
```

```/home/core/dkwork/es/fluentd/etc/fluent.conf
<source>
  @type mysql_slow_query
  path /var/log/mysql/slow.log
  pos_file /tmp/mysql/slow.pos
  tag mysqld.slow_query
   <parse>
     @type none
   </parse>
</source>
<source>
  @type tail
  format none
  path /var/log/mysql/general.log
  pos_file /tmp/mysql/general.pos
  tag mysqld.general
</source>
<source>
  @type tail
  format none
  path /var/log/mysql/error.log
  pos_file /tmp/mysql/error.pos
  tag mysqld.error
</source>

<match **.**>
  @type copy
  <store>
    @type stdout
  </store>
  <store>
    @type elasticsearch
    include_tag_key true
    include_timestamp true # defaults to false
    tag_key @log_name
    host elasticsearch
    port 9200
    # Rollover index configuration
    rollover_index true # defaults to false
    index_name mysql
    # fluent-plugin-elasticsearchの4.0.0から
    # index_prefixは廃止になったのでコメント化
    #index_prefix mysql # defaults to "logstash"
    application_name log # defaults to "default"
    deflector_alias mysql-log
    template_name mysql_template
    template_file /fluentd/etc/mysql_template.json
    # elasticsearch-xpack gem
    enable_ilm true # Default value is false 
    ilm_policy_id mysql-policy # Default value is logstash-policy
    ilm_policy { "policy": { "phases": { "hot": { "min_age": "0ms", "actions": { "rollover": { "max_age": "7d", "max_docs": 20, "max_size": "5gb" } } }, "delete": { "min_age": "1h", "actions": { "delete": {} } } } } }
    flush_interval 10s
  </store>
</match>
```

```/home/core/dkwork/es/fluentd/etc/mysql_template.json
{
  "index_patterns": ["mysql-log-*"], 
  "settings": {
    "number_of_shards": 1,
    "number_of_replicas": 1,
    "index.lifecycle.name": "myisql-policy", 
    "index.lifecycle.rollover_alias": "mysql-log"
  }
}
```

```/home/core/dkwork/es/kibana/config/kibana.yml
server.name: kibana
server.host: "0"
elasticsearch.url: http://elasticsearch:9200
```

```/home/core/dkwork/es/mariadb/etc/mymariadb.cnf 
[mysqld]
general_log
general_log_file=/var/log/mysql/general.log
slow_query_log
slow_query_log_file=/var/log/mysql/slow.log
long_query_time=5 # 5秒以上処理に時間がかかったクエリを記録
log-queries-not-using-indexes # インデックスが使用されていないクエリをログに出力
log-error=/var/log/mysql/error.log
```

# 実行

```
$ podman-compose up -d

$ podman ps

CONTAINER ID  IMAGE                                                COMMAND               CREATED        STATUS            PORTS                                             NAMES
cf13edb2e303  docker.elastic.co/kibana/kibana:7.5.2                /usr/local/bin/ki...  5 minutes ago  Up 5 minutes ago  0.0.0.0:3306->3306/tcp, 0.0.0.0:24224->24224/udp  kibana
c4f999f151a5  docker.io/library/mariadb:10.4.12                    mysqld                5 minutes ago  Up 5 minutes ago  0.0.0.0:3306->3306/tcp, 0.0.0.0:24224->24224/udp  mariadb
9f0a6d2ff83e  localhost/fluentd:1.9.0                              fluentd               5 minutes ago  Up 5 minutes ago  0.0.0.0:3306->3306/tcp, 0.0.0.0:24224->24224/udp  fluentd
1924e6a0e346  docker.elastic.co/elasticsearch/elasticsearch:7.5.2  eswrapper             5 minutes ago  Up 5 minutes ago  0.0.0.0:3306->3306/tcp, 0.0.0.0:24224->24224/udp  elasticsearch
```
![1.PNG](https://qiita-image-store.s3.ap-northeast-1.amazonaws.com/0/208807/ad82dbc9-546f-706f-2ee3-17118fe5944c.png)
![2.PNG](https://qiita-image-store.s3.ap-northeast-1.amazonaws.com/0/208807/5bc8ea22-6ae1-d7be-01b5-dca7c1abb34f.png)