cat >> ceph-config.yml <<EOF
# for all nodes
all: &all
- mon0
- osd0
- osd1
- osd2
# for mon and mgr nodes
mons: &mons
- mon0
- osd0
- osd1
# for osd node
osds:
- osd0
- osd1
- osd2
# for cephfs node
mds:
*mons
# for admin node
admin:
*mons
# for client node
clients:
- k8s1
disks:
osd0: &osd0
- /dev/sdb
- /dev/sdc
osd1:
*osd0
osd2:
*osd0
global_config: |
#public network = 192.168.42.0/24
mgr initial modules = prometheus
mon allow pool delete = true
max_open_files = 131072
osd_journal_size = 10000
osd_pool_default_size = 3
osd_pool_default_min_size = 1
EOF
cat >> ceph-auto-deploy.py <<EOF
#!/usr/bin/env python
# coding=utf-8
from executor import execute
import sys, os
import argparse
import json
import yaml
from whichcraft import which
cluster_dir = './myceph'
ceph_config_yml = os.getcwd() + '/ceph-config.yml'
os.environ['CEPH_DEPLOY_REPO_URL'] = 'https://mirrors.aliyun.com/ceph/rpm-mimic/el7/'
os.environ['CEPH_DEPLOY_GPG_URL'] = 'https://mirrors.aliyun.com/ceph/keys/release.asc'
def get_config():
with open(ceph_config_yml, 'r') as f:
config = yaml.load(f, Loader=yaml.Loader)
return config
# 参数hosts 是all mons osds mds admin 中的一个
def get_nodes(hosts):
config = get_config()
return ' '.join(config[hosts])
def get_global_config():
config = get_config()
return config['global_config']
def get_disks():
config = get_config()
return config['disks']
# 安装epel
def install_epel(hosts):
config = get_config()
# 先安装epel
for node in config[hosts]:
#execute('ssh {node} sudo yum -y install https://mirrors.aliyun.com/centos/7/extras/x86_64/Packages/epel-release-7-11.noarch.rpm'.format(node=node), check=False)
execute('ssh {node} sudo wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo'.format(node=node), check=False)
execute('ssh {node} sudo yum repolist'.format(node=node), check=False)
execute('ssh {node} sudo yum yum-plugin-priorities'.format(node=node), check=False)
# 对节点进行ceph的安装
def install_ceph(hosts):
config = get_config()
nodes = ' '.join(config[hosts])
# 安装ceph 包
if not execute('ceph-deploy --overwrite-conf install {nodes}'.format(nodes=nodes), check=False):
sys.exit(1)
# 对clients 节点进行ceph的安装
def install_clients():
config = get_config()
nodes = ' '.join(config['clients'])
# 安装ceph 包
if not execute('ceph-deploy --overwrite-conf install --cli {nodes}'.format(nodes=nodes), check=False):
sys.exit(1)
# 安装 ceph-fuse
for node in config['clients']:
execute('ssh {node} sudo yum -y install ceph-fuse'.format(node=node), check=False)
# 将节点配置成管理节点
def admin_ceph():
nodes = get_nodes('admin')
if not execute('ceph-deploy --overwrite-conf admin {nodes}'.format(nodes=nodes), check=False):
sys.exit(1)
# 添加mgr
def add_mgr():
nodes = get_nodes('mons')
if not execute('ceph-deploy --overwrite-conf mgr create {nodes}'.format(nodes=nodes), check=False):
sys.exit(1)
# 添加osd
def add_osd():
disks = get_disks()
for node, devs in disks.items():
for dev in devs:
if not execute('ceph-deploy --overwrite-conf osd create {node} --data {dev}'.format(node=node, dev=dev), check=False):
sys.exit(1)
# 创建pool
# app:有三种类型 cephfs, rbd, rgw
def add_pool(poolName, app, pgNum):
config = get_config()
node = config['admin'][0]
if which('ceph'):
execute('ceph osd pool create {poolName} {pgNum}'.format(poolName=poolName, pgNum=pgNum), check=False, sudo=True)
execute('ceph osd pool application enable {poolName} {app}'.format(poolName=poolName, app=app), check=False, sudo=True)
execute('ceph auth get-or-create client.{poolName} mon "allow r" osd "allow class-read object_prefix rbd_children, allow rwx pool={poolName}" -o ceph.client.{poolName}.keyring'.format(poolName=poolName), check=False, sudo=True)
else:
execute('ssh {node} sudo ceph osd pool create {poolName} {pgNum}'.format(node=node, poolName=poolName, pgNum=pgNum), check=False)
execute('ssh {node} sudo ceph osd pool application enable {poolName} {app}'.format(node=node, poolName=poolName, app=app), check=False)
if app == 'rbd':
execute('ssh {node} sudo ceph auth get-or-create client.{poolName} mon "allow r" osd "allow class-read object_prefix rbd_children, allow rwx pool={poolName}" -o ceph.client.{poolName}.keyring'.format(node=node, poolName=poolName), check=False)
# 生成cephfs 文件系统
def gen_cephfs_filesystem():
add_pool('cephfs_data', 'cephfs', '64')
add_pool('cephfs_metadata', 'cephfs', '64')
config = get_config()
node = config['admin'][0]
if which('ceph'):
execute('ceph fs new data cephfs_metadata cephfs_data', check=False, sudo=True)
else:
execute('ssh {node} sudo ceph fs new data cephfs_metadata cephfs_data'.format(node=node), check=False)
# 添加mds
def add_mds():
nodes = get_nodes('mds')
if not execute('ceph-deploy --overwrite-conf mds create {nodes}'.format(nodes=nodes), check=False):
sys.exit(1)
# 删除集群
def purge_cluster():
nodes = get_nodes('all')
if not execute('ceph-deploy purge {nodes}'.format(nodes=nodes), check=False):
sys.exit(1)
# 创建集群
def create_cluster():
# 对所有的节点安装epel 软件包
print('=='*10 + 'Install epel' + '=='*10)
install_epel('all')
print('=='*10 + 'Init ceph cluster....' + '=='*10)
mons_node = get_nodes('mons')
if not os.path.exists(cluster_dir):
os.makedirs(cluster_dir)
os.chdir(cluster_dir)
global_config = get_global_config()
if execute('ceph-deploy --overwrite-conf new {mons_node}'.format(mons_node=mons_node), check=False):
with open('ceph.conf', 'a') as f:
f.write(global_config)
# 对所有的节点安装ceph 软件包
print('=='*10 + 'Install ceph package....' + '=='*10)
install_ceph('all')
# 初始化monitor
print('=='*10 + 'Init ceph monitor....' + '=='*10)
execute('ceph-deploy --overwrite-conf mon create-initial', check=False)
# 配置管理节点
print('=='*10 + 'Config admin....' + '=='*10)
admin_ceph()
# 部署mgr服务
print('=='*10 + 'Deploy mgr....' + '=='*10)
add_mgr()
# 部署osd服务
print('=='*10 + 'Deploy osd....' + '=='*10)
add_osd()
# 创建k8s pool
print('=='*10 + 'Create pool....' + '=='*10)
add_pool('kube','rbd','128')
# 部署mds服务
print('=='*10 + 'Deploy mds....' + '=='*10)
add_mds()
# 生成cephfs 文件系统
print('=='*10 + 'Create cephfs filesystem....' + '=='*10)
gen_cephfs_filesystem()
def parse_arguments(args=None):
parser = argparse.ArgumentParser(description='Used to deploy ceph cluster')
parser.add_argument('-d', '--deploy', dest='deploy', action="store_true", help='Deploy ceph cluster')
parser.add_argument('-p', '--purge', dest='purge', action="store_true", help='Purge ceph cluster')
parser.add_argument('-c', '--client', dest='client', action="store_true", help='Install ceph package for client node')
try:
args = parser.parse_args(args=args)
except IOError as msg:
parser.error(str(msg))
if len(sys.argv) == 1:
parser.print_help(sys.stderr)
sys.exit(1)
return args
def main(arg):
args = parse_arguments(arg)
if args.deploy:
create_cluster()
elif args.purge:
purge_cluster()
elif args.client:
install_clients()
if __name__ == '__main__':
main(sys.argv[1:])
EOF