1、安装第一个mon节点
cat > /etc/ceph/ceph.conf << EOF
[global]
fsid = f11240d4-86b1-49ba-aacc-6d3d37b24cc4
mon initial members = os-node1,os-node2,os-node3
mon host = 10.240.240.211,10.240.240.212,10.240.240.213
public network = 10.240.240.0/24
auth cluster required = cephx
auth service required = cephx
auth client required = cephx
filestore xattr use omap = true
[mon]
mon clock drift allowed = .50
mon osd down out interval = 900
[mds]
max mds = 5
mds max file size = 100000000
mds cache size = 100000
[osd]
osd journal size = 1024
osd pool default size = 3
osd pool default min size = 1
osd crush chooseleaf type = 1
osd recovery threads = 1
osd_mkfs_type = xfs
osd pool default pg num = 100
osd pool default pgp num = 100
EOF
ceph-authtool --create-keyring /etc/ceph/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'
ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --set-uid=0 --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *'
ceph-authtool /etc/ceph/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring
mkdir -p /var/lib/ceph/mon/ceph-os-node1
mkdir -p /var/lib/ceph/bootstrap-osd/
ceph-authtool -C /var/lib/ceph/bootstrap-osd/ceph.keyring
ceph-mon --mkfs -i os-node1 --keyring /etc/ceph/ceph.mon.keyring
touch /var/lib/ceph/mon/ceph-os-node1/done
touch /var/lib/ceph/mon/ceph-os-node1/sysvinit
/sbin/service ceph -c /etc/ceph/ceph.conf start mon.os-node1
2、建立第二个mon节点
1)、在os-node2节点
mkdir -p /var/lib/ceph/mon/ceph-os-node2
2)、在os-node1节点
scp /etc/ceph/* os-node2:/etc/ceph/
scp /var/lib/ceph/bootstrap-osd/ceph.keyring os-node2:/var/lib/ceph/bootstrap-osd/
scp /etc/ceph/ceph.mon.keyring os-node2:/etc/ceph/
3)、在os-node2节点
ceph-mon --mkfs -i os-node2 --keyring /etc/ceph/ceph.mon.keyring
touch /var/lib/ceph/mon/ceph-os-node2/done
touch /var/lib/ceph/mon/ceph-os-node2/sysvinit
/sbin/service ceph -c /etc/ceph/ceph.conf start mon.os-node2
2、建立第三个mon节点
1)、在os-node3节点
mkdir -p /var/lib/ceph/mon/ceph-os-node3
2)、在os-node1节点
scp /etc/ceph/* os-node3:/etc/ceph/
scp /var/lib/ceph/bootstrap-osd/ceph.keyring os-node3:/var/lib/ceph/bootstrap-osd/
scp /etc/ceph/ceph.mon.keyring os-node3:/etc/ceph/
3)、在os-node3节点
ceph-mon --mkfs -i os-node3 --keyring /etc/ceph/ceph.mon.keyring
touch /var/lib/ceph/mon/ceph-os-node3/done
touch /var/lib/ceph/mon/ceph-os-node3/sysvinit
/sbin/service ceph -c /etc/ceph/ceph.conf start mon.os-node3
添加osd节点
在os-node1节点复制ceph目录下的文件到osd节点
scp /etc/ceph/* admin:/etc/ceph
scp /etc/ceph/* os-node1:/etc/ceph
scp /etc/ceph/* os-node2:/etc/ceph
scp /etc/ceph/* os-node3:/etc/ceph
1、添加第一块osd节点
在os-node1节点
ceph osd create
mkdir -p /var/lib/ceph/osd/ceph-0
mkfs.xfs -f /dev/sdb
mount /dev/sdb /var/lib/ceph/osd/ceph-0
mount -o remount,user_xattr /var/lib/ceph/osd/ceph-0
echo "/dev/sdb /var/lib/ceph/osd/ceph-0 xfs defaults 0 0" >> /etc/fstab
echo "/dev/sdb /var/lib/ceph/osd/ceph-0 xfs remount,user_xattr 0 0" >> /etc/fstab
ceph-osd -i 0 --mkfs --mkkey
ceph auth add osd.0 osd 'allow *' mon 'allow profile osd' -i /var/lib/ceph/osd/ceph-0/keyring
ceph osd crush add-bucket rack1 rack
ceph osd crush add-bucket os-node1 host
ceph osd crush move os-node1 rack=rack1
ceph osd crush move rack1 root=default
ceph osd crush add osd.0 1.0 host=os-node1
touch /var/lib/ceph/osd/ceph-0/sysvinit
/etc/init.d/ceph start osd.0
2、添加第二个osd节点
在os-node2节点
ceph osd create
mkdir -p /var/lib/ceph/osd/ceph-1
mkfs.xfs -f /dev/sdb
mount /dev/sdb /var/lib/ceph/osd/ceph-1
mount -o remount,user_xattr /var/lib/ceph/osd/ceph-1
echo "/dev/sdb /var/lib/ceph/osd/ceph-1 xfs defaults 0 0" >> /etc/fstab
echo "/dev/sdb /var/lib/ceph/osd/ceph-1 xfs remount,user_xattr 0 0" >> /etc/fstab
ceph-osd -i 1 --mkfs --mkkey
ceph auth add osd.1 osd 'allow *' mon 'allow profile osd' -i /var/lib/ceph/osd/ceph-1/keyring
ceph osd crush add-bucket rack2 rack
ceph osd crush add-bucket os-node2 host
ceph osd crush move os-node2 rack=rack2
ceph osd crush move rack2 root=default
ceph osd crush add osd.1 1.0 host=os-node2
touch /var/lib/ceph/osd/ceph-1/sysvinit
/etc/init.d/ceph start osd.1
2、添加第三个osd节点
在os-node3节点
ceph osd create
mkdir -p /var/lib/ceph/osd/ceph-2
mkfs.xfs -f /dev/sdb
mount /dev/sdb /var/lib/ceph/osd/ceph-2
mount -o remount,user_xattr /var/lib/ceph/osd/ceph-2
echo "/dev/sdb /var/lib/ceph/osd/ceph-2 xfs defaults 0 0" >> /etc/fstab
echo "/dev/sdb /var/lib/ceph/osd/ceph-2 xfs remount,user_xattr 0 0" >> /etc/fstab
ceph-osd -i 2 --mkfs --mkkey
ceph auth add osd.2 osd 'allow *' mon 'allow profile osd' -i /var/lib/ceph/osd/ceph-2/keyring
ceph osd crush add-bucket rack3 rack
ceph osd crush add-bucket os-node3 host
ceph osd crush move os-node3 rack=rack3
ceph osd crush move rack3 root=default
ceph osd crush add osd.2 1.0 host=os-node3
touch /var/lib/ceph/osd/ceph-2/sysvinit
/etc/init.d/ceph start osd.2
添加元数据服务器
添加第一个元数据服务器
1、在os-node1节点
mkdir -p /var/lib/ceph/mds/ceph-os-node1
touch /root/ceph.bootstrap-mds.keyring
ceph-authtool --import-keyring /var/lib/ceph/bootstrap-mds/ceph.keyring ceph.bootstrap-mds.keyring
ceph --cluster ceph --name client.bootstrap-mds --keyring /var/lib/ceph/bootstrap-mds/ceph.keyring auth get-or-create mds.os-node1 osd 'allow rwx' mds 'allow' mon 'allow profile mds' -o /var/lib/ceph/mds/ceph-os-node1/keyring
touch /var/lib/ceph/mds/ceph-os-node1/sysvinit
touch /var/lib/ceph/mds/ceph-os-node1/done
service ceph start mds.os-node1
添加第二个元数据服务器
1、在os-node2节点
mkdir -p /var/lib/ceph/mds/ceph-os-node2
mkdir -p /var/lib/ceph/bootstrap-mds/
2、在os-node1节点
scp /var/lib/ceph/bootstrap-mds/ceph.keyring os-node2:/var/lib/ceph/bootstrap-mds/
scp /root/ceph.bootstrap-mds.keyring os-node2:/root/
scp /var/lib/ceph/mds/ceph-os-node1/sysvinit os-node2:/var/lib/ceph/mds/ceph-os-node2/
3、在os-node2节点
ceph --cluster ceph --name client.bootstrap-mds --keyring /var/lib/ceph/bootstrap-mds/ceph.keyring auth get-or-create mds.os-node2 osd 'allow rwx' mds 'allow' mon 'allow profile mds' -o /var/lib/ceph/mds/ceph-os-node2/keyring
touch /var/lib/ceph/mds/ceph-os-node2/done
service ceph start mds.os-node2
添加第三个元数据服务器
1、在os-node3节点
mkdir -p /var/lib/ceph/mds/ceph-os-node3
mkdir -p /var/lib/ceph/bootstrap-mds/
2、在os-node1节点
scp /var/lib/ceph/bootstrap-mds/ceph.keyring os-node3:/var/lib/ceph/bootstrap-mds/
scp /root/ceph.bootstrap-mds.keyring os-node3:/root/
scp /var/lib/ceph/mds/ceph-os-node1/sysvinit os-node3:/var/lib/ceph/mds/ceph-os-node3/
3、在os-node3节点
ceph --cluster ceph --name client.bootstrap-mds --keyring /var/lib/ceph/bootstrap-mds/ceph.keyring auth get-or-create mds.os-node3 osd 'allow rwx' mds 'allow' mon 'allow profile mds' -o /var/lib/ceph/mds/ceph-os-node3/keyring
touch /var/lib/ceph/mds/ceph-os-node3/done
service ceph start mds.os-node3
