Lustreインストールスクリプト（CentOS7.9.2009）

# 前提条件（MGS/MDS/OSS/CLIENT共通）
・CentOS 7.9.2009
・x86 64

# 構成

| Role   | PC |  IP         | NIC NAME |  DISK    | Comment |
|:-------|:---|:------------|:---------|:---------|:--------|
|MGS/MGT |PC1 |192.168.0.10 |enp2s0    |/dev/sdb1 |         |
|MDT/MDT |PC1 |192.168.0.10 |enp2s0    |/dev/sdb2 |MGSと兼用|
|OSS/OST |PC2 |192.168.0.11 |enp2s0    |/dev/sdb1 |         |
|Client  |PC3 |192.168.0.12 |-         |-         |         |

# インストール（サーバ側）

MGS、MDS、及びOSSをインストールするPC1、PC2では、以下を実行する。
以下はMGS/MDS枠割のPC1のIPとNICデバイス名なので、OSSのPC2の場合は、それぞれ修正してから実行する。

```
#!/bin/bash

  # http://www.comphys.las.shibaura-it.ac.jp/install_lustre2_1_3

NDEV="enp2s0"
IP_ADDR="192.168.0.10"

  echo "INFO : install wget  ------------------------------------------------------"; yum -y install wget

#--------------------------------------------------------------------------------------------------------------------------------
items=(
    "libcom_err-1.45.6.wc7-0.el7.x86_64.rpm"
    "libss-1.45.6.wc7-0.el7.x86_64.rpm"
    "e2fsprogs-libs-1.45.6.wc7-0.el7.x86_64.rpm"
    "e2fsprogs-1.45.6.wc7-0.el7.x86_64.rpm"
)

  for item in "${items[@]}" ; do
      echo "INFO : download ${item}  ----------------------------------------"; wget https://downloads.whamcloud.com/public/e2fsprogs/1.45.6.wc7/el7/RPMS/x86_64/${item}
  done


#--------------------------------------------------------------------------------------------------------------------------------
 #echo "INFO : uninstall libcom_err    ---------------------------------------"; yum -y remove libcom_err
  if [ -e /etc/yum/protected.d/systemd.conf ]; then
      if [ -e /etc/yum/protected.d/systemd.conf.bak ]; then
          echo "INFO : /etc/yum/protected.d/systemd.conf.bak exist  --> no action  -----------------";
      else
          echo "INFO : rename to prevent error -------------------------------"; mv /etc/yum/protected.d/systemd.conf /etc/yum/protected.d/systemd.conf.bak
      fi
  else
      echo "INFO : /etc/yum/protected.d/systemd.conf does not exist  --> no action  -----------------";
  fi

  echo "INFO : uninstall libcom_err    ---------------------------------------"; rpm -e --nodeps libcom_err
  echo "INFO : install   libcom_err    ---------------------------------------"; rpm -Uvh libcom_err-1.45.6.wc7-0.el7.x86_64.rpm

  echo "INFO : uninstall libss         ---------------------------------------"; yum -y remove libss
  echo "INFO : install   libss         ---------------------------------------"; rpm -Uvh libss-1.45.6.wc7-0.el7.x86_64.rpm

  echo "INFO : uninstall e2fsprogs-libs---------------------------------------"; yum -y remove e2fsprogs-libs
  echo "INFO : install   e2fsprogs-libs---------------------------------------"; rpm -Uvh e2fsprogs-libs-1.45.6.wc7-0.el7.x86_64.rpm

  echo "INFO : install   e2fsprogs     ---------------------------------------"; rpm -Uvh e2fsprogs-1.45.6.wc7-0.el7.x86_64.rpm


#--------------------------------------------------------------------------------------------------------------------------------
  # https://web.chaperone.jp/w/index.php?Lustre
  echo "INFO : create /etc/yum.repos.d/lustre.repo  --------------------------";
  echo "[lustre-server]"                                                                                       > /etc/yum.repos.d/lustre.repo
  echo "name=CentOS-\$releasever - Lustre"                                                                    >> /etc/yum.repos.d/lustre.repo
  echo "baseurl=https://downloads.whamcloud.com/public/lustre/latest-release/el7/patchless-ldiskfs-server/"   >> /etc/yum.repos.d/lustre.repo
  echo "gpgcheck=0"                                                                                           >> /etc/yum.repos.d/lustre.repo
  echo ""                                                                                                     >> /etc/yum.repos.d/lustre.repo
  echo "[e2fsprogs]"                                                                                          >> /etc/yum.repos.d/lustre.repo
  echo "name=CentOS-\$releasever - Ldiskfs"                                                                   >> /etc/yum.repos.d/lustre.repo
  echo "baseurl=https://downloads.whamcloud.com/public/e2fsprogs/latest/el7/"                                 >> /etc/yum.repos.d/lustre.repo
  echo "gpgcheck=0"                                                                                           >> /etc/yum.repos.d/lustre.repo
  echo ""                                                                                                     >> /etc/yum.repos.d/lustre.repo
  echo "[lustre-client]"                                                                                      >> /etc/yum.repos.d/lustre.repo
  echo "name=CentOS-\$releasever - Lustre"                                                                    >> /etc/yum.repos.d/lustre.repo
  echo "baseurl=https://downloads.whamcloud.com/public/lustre/latest-release/el7/client"                      >> /etc/yum.repos.d/lustre.repo
  echo "gpgcheck=0"                                                                                           >> /etc/yum.repos.d/lustre.repo

  echo "INFO : install  lustre-osd    ----------------------------------------"; yum -y install lustre-osd
  echo "INFO : install  libyaml       ----------------------------------------"; yum -y install libyaml

  echo "INFO : download lustre        ----------------------------------------"; wget https://downloads.whamcloud.com/public/lustre/lustre-2.12.8/el7.9.2009/server/RPMS/x86_64/lustre-2.12.8_6_g5457c37-1.el7.x86_64.rpm
  echo "INFO : install  lustre        ----------------------------------------"; rpm -ivh lustre-2.12.8_6_g5457c37-1.el7.x86_64.rpm


#--------------------------------------------------------------------------------------------------------------------------------
  grep -q $IP_ADDR  /etc/lnet.conf
  if [ $? = 0 ]; then
      echo "INFO : setting has already added to /etc/lnet.conf  --> skip  ---------";
  else
      echo "INFO : setting does not added to /etc/lnet.conf yet --> add setting  --";
      echo "net:"                             >> /etc/lnet.conf
      echo "  - net type: tcp"                >> /etc/lnet.conf
      echo "    local NI(s):"                 >> /etc/lnet.conf
      echo "      - nid: ${IP_ADDR}@tcp"      >> /etc/lnet.conf
      echo "        status: up"               >> /etc/lnet.conf
      echo "        interfaces:"              >> /etc/lnet.conf
      echo "            0: ${NDEV}"           >> /etc/lnet.conf
  fi

  echo "INFO : enable  lnet  ------------------------------------------------------"; systemctl enable  lnet
  echo "INFO : restart lnet  ------------------------------------------------------"; systemctl restart lnet
  echo "INFO : lnetctl net show  --------------------------------------------------"; lnetctl net show
  echo "INFO : restart  firewalld     ---------------------------------------------"; systemctl restart firewalld
  echo "INFO : enable   firewalld     ---------------------------------------------"; systemctl enable  firewalld
  echo "INFO : allow 988/tcp to firewalld  ----------------------------------------"; firewall-cmd --add-port=988/tcp --permanent
  echo "INFO : reload firewalld  --------------------------------------------------"; firewall-cmd --reload
```

# インストール（クライアント側）

クライアントのPC3では以下を実行する。

```
#!/bin/bash

  # http://www.comphys.las.shibaura-it.ac.jp/install_lustre2_1_3

NDEV="enp2s0"
IP_ADDR="192.168.0.12"

#--------------------------------------------------------------------------------------------------------------------------------
  # https://web.chaperone.jp/w/index.php?Lustre
  echo "INFO : create /etc/yum.repos.d/lustre.repo  --------------------------";
  echo "[lustre-server]"                                                                                       > /etc/yum.repos.d/lustre.repo
  echo "name=CentOS-\$releasever - Lustre"                                                                    >> /etc/yum.repos.d/lustre.repo
  echo "baseurl=https://downloads.whamcloud.com/public/lustre/latest-release/el7/patchless-ldiskfs-server/"   >> /etc/yum.repos.d/lustre.repo
  echo "gpgcheck=0"                                                                                           >> /etc/yum.repos.d/lustre.repo
  echo ""                                                                                                     >> /etc/yum.repos.d/lustre.repo
  echo "[e2fsprogs]"                                                                                          >> /etc/yum.repos.d/lustre.repo
  echo "name=CentOS-\$releasever - Ldiskfs"                                                                   >> /etc/yum.repos.d/lustre.repo
  echo "baseurl=https://downloads.whamcloud.com/public/e2fsprogs/latest/el7/"                                 >> /etc/yum.repos.d/lustre.repo
  echo "gpgcheck=0"                                                                                           >> /etc/yum.repos.d/lustre.repo
  echo ""                                                                                                     >> /etc/yum.repos.d/lustre.repo
  echo "[lustre-client]"                                                                                      >> /etc/yum.repos.d/lustre.repo
  echo "name=CentOS-\$releasever - Lustre"                                                                    >> /etc/yum.repos.d/lustre.repo
  echo "baseurl=https://downloads.whamcloud.com/public/lustre/latest-release/el7/client"                      >> /etc/yum.repos.d/lustre.repo
  echo "gpgcheck=0"                                                                                           >> /etc/yum.repos.d/lustre.repo


#--------------------------------------------------------------------------------------------------------------------------------
  grep -q $IP_ADDR  /etc/lnet.conf
  if [ $? = 0 ]; then
      echo "INFO : setting has already added to /etc/lnet.conf  --> skip  ---------";
  else
      echo "INFO : setting does not added to /etc/lnet.conf yet --> add setting  --";
      echo "net:"                             >> /etc/lnet.conf
      echo "  - net type: tcp"                >> /etc/lnet.conf
      echo "    local NI(s):"                 >> /etc/lnet.conf
      echo "      - nid: ${IP_ADDR}@tcp"      >> /etc/lnet.conf
      echo "        status: up"               >> /etc/lnet.conf
      echo "        interfaces:"              >> /etc/lnet.conf
      echo "            0: ${NDEV}"           >> /etc/lnet.conf
  fi

  echo "INFO : enable  lnet  ------------------------------------------------------"; systemctl enable  lnet
  echo "INFO : restart lnet  ------------------------------------------------------"; systemctl restart lnet
  echo "INFO : lnetctl net show  --------------------------------------------------"; lnetctl net show
  echo "INFO : restart  firewalld     ---------------------------------------------"; systemctl restart firewalld
  echo "INFO : enable   firewalld     ---------------------------------------------"; systemctl enable  firewalld
  echo "INFO : allow 988/tcp to firewalld  ----------------------------------------"; firewall-cmd --add-port=988/tcp --permanent
  echo "INFO : reload firewalld  --------------------------------------------------"; firewall-cmd --reload


#--------------------------------------------------------------------------------------------------------------------------------
  echo "INFO : install  wget     ---------------------------------------------"; yum -y install wget
  echo "INFO : install  attr     ---------------------------------------------"; yum -y install attr
  echo "INFO : install  lsof     ---------------------------------------------"; yum -y install lsof
  echo "INFO : install  libtool  ---------------------------------------------"; yum -y install libtool
  echo "INFO : install  openmpi to resolve error of libmpi.so.12(64bit)  -----"; yum -y install openmpi
  echo "INFO : install  libyaml  ---------------------------------------------"; yum -y install libyaml

items=(
    "lustre-client-2.12.8_6_g5457c37-1.el7.x86_64.rpm"
)
  for item in "${items[@]}" ; do
      echo "INFO : download ${item}  ----------------------------------------";
      if [ -e ${item} ]; then
          echo "INFO : ${item} is exist  --> skip download  -----------------";
      else
          wget https://downloads.whamcloud.com/public/lustre/lustre-2.12.8/el7.9.2009/client/RPMS/x86_64/${item}
      fi
      echo "INFO : install ${item}  -----------------------------------------"; rpm -ivh ${item}
  done


#--------------------------------------------------------------------------------------------------------------------------------
  echo "INFO : (CLIENT) yum upgrade e2fsprogs  ------------------------------------"; yum -y upgrade e2fsprogs
  echo "INFO : (CLIENT) yum install lustre-client  --------------------------------"; yum install lustre-client
  echo "INFO : (CLIENT) yum list installed | grep lustre-client  ------------------"; yum list installed | grep lustre-client
```

# インストール後は再起動

サーバ側とクライアント側で上記をそれぞれインストールしたら再起動しておく。


# ファイルシステム作成、マウント

再起動したら、後述のスクリプトを実行して、対象ディスクをLustreファイルシステムでフォーマットして、さらにクライアントからマウントする。
先にサーバ側の設定をすべきなので、実行順番は、MGS/MDS/OSSを先に実行し、その後にCLIENT側で実行する。

MGT/MDT役割のPC1の場合の設定は以下。
```
SW_MGT=YES
SW_MDT=YES
SW_OST=NO
SW_CLIENT=NO
NDEV="enp2s0"
IP_ADDR="192.168.0.10"
IP_MGT="192.168.0.10"
```

OST役割のPC2の場合の設定は以下。
```
SW_MGT=NO
SW_MDT=NO
SW_OST=YES
SW_CLIENT=NO
NDEV="enp2s0"
IP_ADDR="192.168.0.11"
IP_MGT="192.168.0.10"
```

CLIENT役割のPC3の場合の設定は以下。
```
SW_MGT=NO
SW_MDT=NO
SW_OST=NO
SW_CLIENT=YES
NDEV="enp2s0"
IP_ADDR="192.168.0.12"
IP_MGT="192.168.0.10"
```


```
#!/bin/bash

  # https://web.chaperone.jp/w/index.php?Lustre

SW_MGT=NO
SW_MDT=NO
SW_OST=NO
SW_CLIENT=YES


NDEV="enp2s0"
IP_ADDR="192.168.0.12"

IP_MGT="192.168.0.10"

DEV_MGT=sdb1
DEV_MDT=sdb2
DEV_OST=sdb3

DIR_MGT=/mnt/mgt
DIR_MDT=/mnt/mdt.0
DIR_OST=/mnt/ost.0
DIR_CLIENT=/mnt/lustre


if [ $SW_MGT == "YES" ]; then
  echo "INFO : (MGT) SW_MGT = $SW_MGT  --> mkfs.lustre  ---------------------------";
 #echo "INFO : (MGT) mkfs.lustre for MGT disk  ------------------------------------"; mkfs.lustre --reformat --fsname=fsname --mgs /dev/$DEV_MGT
  echo "INFO : (MGT) mkfs.lustre for MGT disk  ------------------------------------"; mkfs.lustre --fsname=fsname --mgs /dev/$DEV_MGT
  echo "INFO : (MGT) create dir for mount  ----------------------------------------"; mkdir -p $DIR_MGT
  echo "INFO : (MGT) mount as lustre file system  ---------------------------------"; mount -t lustre /dev/$DEV_MGT $DIR_MGT
else
  echo "INFO : (MGT) SW_MGT = $SW_MGT  --> no action  -----------------------------";
fi

if [ $SW_MDT == "YES" ]; then
  echo "INFO : (MDT) SW_MDT = $SW_MDT  --> mkfs.lustre  ---------------------------";
  echo "INFO : (MDT) mkfs.lustre for MDT disk  ------------------------------------"; mkfs.lustre --fsname=fsname --mgsnode=$IP_MGT@tcp0 --mdt --index=0 /dev/$DEV_MDT
  echo "INFO : (MDT) create dir for mount  ----------------------------------------"; mkdir -p $DIR_MDT
  echo "INFO : (MDT) mount as lustre file system  ---------------------------------"; mount -t lustre /dev/$DEV_MDT $DIR_MDT
else
  echo "INFO : (MDT) SW_MDT = $SW_MDT  --> no action  -----------------------------";
fi

if [ $SW_OST == "YES" ]; then
  echo "INFO : (OST) SW_OST = $SW_OST  --> mkfs.lustre  ---------------------------";
  echo "INFO : (OST) mkfs.lustre for OST disk  ------------------------------------"; mkfs.lustre --fsname=fsname --mgsnode=$IP_MGT@tcp0 --ost --index=0 /dev/$DEV_OST
  echo "INFO : (OST) create dir for mount  ----------------------------------------"; mkdir -p $DIR_OST
  echo "INFO : (OST) mount as lustre file system  ---------------------------------"; mount -t lustre /dev/$DEV_OST $DIR_OST
else
  echo "INFO : (OST) SW_OST = $SW_OST  --> no action  -----------------------------";
fi

if [ $SW_CLIENT == "YES" ]; then
  echo "INFO : (CLIENT) SW_CLIENT = $SW_CLIENT  --> mount lustre fs  --------------";
  echo "INFO : (CLIENT) create dir for mount  -------------------------------------"; mkdir -p $DIR_CLIENT
  echo "INFO : (CLIENT) mount lustre file system to $DIR_CLIENT  ------------------"; mount -t lustre $IP_MGT@tcp0:/fsname  $DIR_CLIENT
else
  echo "INFO : (CLIENT) SW_CLIENT = $SW_CLIENT  --> no action  --------------------";
fi
```