1. 安装 gpfs

2. 配置网卡和 /etc/hosts

PLan GPFS Config

192.168.5.193 plan01gpfs
192.168.5.194 plan02gpfs
bash-3.2# ifconfig -a
en0: flags=1e080863,c0
inet 10.1.16.193 netmask 0xffffff00 broadcast 10.1.16.255
tcp_sendspace 131072 tcp_recvspace 65536 rfc1323 0
en2: flags=5e080863,c0
inet 192.168.5.193 netmask 0xffffff00 broadcast 192.168.5.255
tcp_sendspace 131072 tcp_recvspace 65536 rfc1323 0
lo0: flags=e08084b,c0
inet 127.0.0.1 netmask 0xff000000 broadcast 127.255.255.255
inet6 ::1%1/0
tcp_sendspace 131072 tcp_recvspace 131072 rfc1323 1

3. 配置 root 允许远程登录和 /.rhosts

bash-3.2# cat /.rhosts
nimserver root
plan01gpfs root
plan02gpfs root

4. 编辑 /etc/profile 环境变量
   export PATH=$PATH:/usr/lpp/mmfs/bin
5. 创建主机 gpfsprofile

bash-3.2# cat /etc/gpfsprofile
plan01gpfs quorum-manager
plan02gpfs quorum-manager

6. 创建集群

mmcrcluster -C plangpfsclr -N /etc/gpfsprofile -p plan01gpfs -s plan02gpfs

[root@dstage01 gpfs]# mmchlicense server --accept -N plan01gpfs,plan02gpfs

7. 创建文件系统：

vi /etc/gpfsnsdprofile

hdisk2:plan01gpfs:plan02gpfs:dataAndMetadata::
hdisk3:plan02gpfs:plan01gpfs:dataAndMetadata::
hdisk4:plan01gpfs:plan02gpfs:dataAndMetadata::
hdisk5:plan02gpfs:plan01gpfs:dataAndMetadata::
hdisk6:plan01gpfs:plan02gpfs:dataAndMetadata::
hdisk7:plan02gpfs:plan01gpfs:dataAndMetadata::
hdisk8:plan01gpfs:plan02gpfs:dataAndMetadata::
hdisk9:plan02gpfs:plan01gpfs:dataAndMetadata::
hdisk10:plan01gpfs:plan02gpfs:dataAndMetadata::
hdisk11:plan02gpfs:plan01gpfs:dataAndMetadata::
hdisk12:plan01gpfs:plan02gpfs:dataAndMetadata::

bash-3.2# mmcrfs /planshare plangpfslv -F /etc/gpfsnsdprofile -A yes -B 1M -M 2 -R 2
mmcrfs: There are no available free disks.
Disks must be prepared prior to invoking mmcrfs.
Define the disks using the mmcrnsd command.
mmcrfs: Command failed. Examine previous error messages to determine cause.
bash-3.2# mmcrnsd -F /etc/gpfsnsdprofile
mmcrnsd: Processing disk hdisk2
mmcrnsd: Processing disk hdisk3
mmcrnsd: Processing disk hdisk4
mmcrnsd: Processing disk hdisk5
mmcrnsd: Processing disk hdisk6
mmcrnsd: Processing disk hdisk7
mmcrnsd: Processing disk hdisk8
mmcrnsd: Processing disk hdisk9
mmcrnsd: Processing disk hdisk10
mmcrnsd: Processing disk hdisk11
mmcrnsd: Propagating the cluster configuration data to all
affected nodes. This is an asynchronous process.

mmcrfs /planshare plangpfslv -F /etc/gpfsnsdprofile -A yes -B 1M -M 2 -R 2

8. 文件系统参数优化

[root@dstage01 gpfs]# mmchconfig pagepool=5G
[root@dstage01 gpfs]# mmchconfig tokenMemLimit=1G
[root@dstage01 gpfs]# mmchconfig maxFilesToCache=10000
[root@dstage01 gpfs]# mmchconfig work1Threads=100
[root@dstage01 gpfs]# mmchconfig worker3Threads=40
[root@dstage01 gpfs]# mmchconfig maxMBpS=1600
[root@dstage01 gpfs]# mmchconfig maxStatCache=40000
[root@dstage01 gpfs]# mmchconfig tiebreakerDisks=gpfs1nsd
[root@dstage01 gpfs]# mmchfs planshare --inode-limit 10m
bash-3.2# mmchfs /dev/plangpfslv --inode-limit 10M
Set maxInodes for inode space 0 to 10485760
Fileset 'root' changed.

[root@dstage01 gpfs]# mmchconfig failureDetectionTime=10

停止一个节点后，另一个节点 hang 30 秒 左右

failureDetectionTime 该值默认 35 修改为 10 秒后，效果不怎么明显

io 测试：

bash-3.2# time dd if=/dev/zero of=./test.out bs=1024k count=10000
10000+0 records in
10000+0 records out

real 0m20.323s
user 0m0.033s
sys 0m3.428s

bash-3.2# time dd if=/dev/zero of=./test.out bs=256k count=40000
40000+0 records in
40000+0 records out

real 0m18.673s
user 0m0.075s
sys 0m3.202s