centos7 + greenplum6.2.1集群快速部署

原创

feko大魔王 2021-04-28 00:59:08 博主文章分类：Linux基础教程 ©著作权

文章标签 greenplum6.2.1 greenplum greenplum6 文章分类 集群服务器

©著作权归作者所有：来自51CTO博客作者feko大魔王的原创作品，请联系作者获取转载授权，否则将追究法律责任

集群部署

版本说明

版本：centos7.2 + greenplum-db-6.2.1-rhel7-x86_64.rpm

架构说明：一个主Master和2个节点

环境准备前奏

面向所有服务器

安装依赖

yum install apr apr-util bash bzip2 curl krb5 libcurl libevent libxml2 libyaml zlib openldap openssh openssl openssl-libs perl readline rsync R sed  tar zip  -y

关闭iptables 和 selinux

systemctl stop iptables
systemctl  disable iptables
systemctl stop firewalld
systemctl disable firewalld

sed -i 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config
setenforce 0

添加hosts解析

cat >> /etc/hosts << EOF
192.168.110.81      gp-master1
192.168.110.82      gp-node1
192.168.110.83      gp-node2
EOF

修改内核参数

cat > /etc/sysctl.conf << EOF
# kernel.shmall = _PHYS_PAGES / 2 # See Shared Memory Pages
kernel.shmall = 4000000000
# kernel.shmmax = kernel.shmall * PAGE_SIZE 
kernel.shmmax = 500000000
kernel.shmmni = 4096
vm.overcommit_memory = 2 # See Segment Host Memory
vm.overcommit_ratio = 95 # See Segment Host Memory

net.ipv4.ip_local_port_range = 10000 65535 # See Port Settings
kernel.sem = 500 2048000 200 40960
kernel.sysrq = 1
kernel.core_uses_pid = 1
kernel.msgmnb = 65536
kernel.msgmax = 65536
kernel.msgmni = 2048
net.ipv4.tcp_syncookies = 1
net.ipv4.conf.default.accept_source_route = 0
net.ipv4.tcp_max_syn_backlog = 4096
net.ipv4.conf.all.arp_filter = 1
net.core.netdev_max_backlog = 10000
net.core.rmem_max = 2097152
net.core.wmem_max = 2097152
vm.swappiness = 10
vm.zone_reclaim_mode = 0
vm.dirty_expire_centisecs = 500
vm.dirty_writeback_centisecs = 100
#  64GB of memory or less, remove vm.dirty_background_bytes and vm.dirty_bytes and set the two ratio parameters to these values
vm.dirty_background_ratio = 3 # See System Memory
vm.dirty_ratio = 10
# vm.dirty_background_bytes = 1610612736
# vm.dirty_bytes = 4294967296
EOF
awk 'BEGIN {OFMT = "%.0f";} /MemTotal/ {print "vm.min_free_kbytes =", $2 * .03;}' /proc/meminfo >> /etc/sysctl.conf
sysctl -p

配置文件连接数和进程数

cat >> /etc/security/limits.conf << EOF
* soft nofile 524288
* hard nofile 524288
* soft nproc 131072
* hard nproc 131072
EOF

# centos7
cat > /etc/security/limits.d/20-nproc.conf <<EOF
*          soft    nproc     131072
root       soft    nproc     unlimited
EOF

挂载XFS文件系统（没有挂载磁盘跳过）

vim >> /etc/fstab << EOF
/dev/data /data xfs nodev,noatime,nobarrier,inode64 0 0
EOF
# 挂载必须是xfs格式的

设置disk I/O

/sbin/blockdev --setra 16384  /dev/sda
chmod +x /etc/rc.d/rc.local
echo deadline > /sys/block/sda/queue/scheduler
# centos7
grubby --update-kernel=ALL --args="elevator=deadline"
grubby --info=ALL

设置THP

# centos7
grubby --update-kernel=ALL --args="transparent_hugepage=never"
# 检查，有输出表示正常
cat /sys/kernel/mm/*transparent_hugepage/enabled

禁用IPC 和创建系统用户gpadmin

cat > /etc/systemd/logind.conf << EOF
RemoveIPC=no
EOF
/bin/systemctl restart  systemd-logind.service

groupadd -r -g 420 gpadmin
useradd -r -u  420 -g gpadmin -m -s /bin/bash gpadmin
echo gpadmin | passwd  gpadmin --stdin
usermod -aG wheel gpadmin
# groupadd gpadmin
# useradd gpadmin -r -m -g gpadmin
# echo gpadmin | passwd  gpadmin --stdin
# usermod -aG wheel gpadmin
visudo
%wheel        ALL=(ALL)       NOPASSWD: ALL

配置动态库

cat >> /etc/ld.so.conf << EOF
/usr/local/lib
EOF

Master端搭建

安装greenplum的rpm包

su - gpadmin
sudo yum install greenplum-db-6.2.1-rhel7-x86_64.rpm  -y
sudo chown -R gpadmin:gpadmin /usr/local/greenplum*
source /usr/local/greenplum-db-6.2.1/greenplum_path.sh

在Master创建密钥对以及分发公钥(Master)

su - gpadmin
source /usr/local/greenplum-db-6.2.1/greenplum_path.sh
cat >ssh-keygen.exp << EOF
#!/usr/bin/expect
spawn ssh-keygen -t rsa
expect {
   "(/home/gpadmin/.ssh/id_rsa):" {send "\r";exp_continue}
   "empty for no passphrase):" {send "\r";exp_continue}
   "Enter same passphrase again:"  {send "\r"}
}
expect eof
EOF

cat > fenfa_sshkey.exp  <<EOF
#!/usr/bin/expect
if { \$argc != 2 } {
send_user  "You have to have two parameters \n"
exit
}
set file [lindex \$argv 0]
set host [lindex \$argv 1]
set password "gpadmin"
spawn ssh-copy-id -i \$file gpadmin@\$host  
expect {
"yes/no"    {send "yes\r";exp_continue}
"password:" {send "\$password\r"}
}
expect eof
EOF

cat > fenfa_sshkey.sh << EOF
#!/bin/sh
. /etc/init.d/functions
[ ! -d /home/gpadmin/.ssh ] && /usr/bin/expect  ssh-keygen.exp 
for ip in \`cat /etc/hosts|grep "000"|awk '{print \$3}'\`
do 
/usr/bin/expect fenfa_sshkey.exp /home/gpadmin/.ssh/id_rsa.pub \$ip &>/dev/null
if [ \$? -eq 0 ]
then
action "\$ip is ok" /bin/true
else
action "\$ip is fail" /bin/false
fi
done
EOF

sh  fenfa_sshkey.sh

创建主机hostlist 和 seg_hosts

mkdir /home/gpadmin/conf
cd /home/gpadmin/conf
cat > hostlist <<EOF
gp-master1
gp-node1
gp-node1
EOF
cat >  seg_hosts  <<EOF
gp-node1
gp-node1
EOF

打通所有服务器服务器之间的ssh连接

cd /home/gpadmin/conf  
gpssh-exkeys -f hostlist

创建数据库目录

### 创建主master的数据目录
sudo mkdir -p /data/master
sudo chown gpadmin:gpadmin /data/master

##创建备用master的数据目录
# source /usr/local/greenplum-db/greenplum_path.sh 
# gpssh -h master-0002 -e 'sudo mkdir -p /data/master'
# gpssh -h master-0002 -e 'sudo chown gpadmin:gpadmin /data/master'
##创建备用master的数据目录

### 创建segment的数据目录
source /usr/local/greenplum-db/greenplum_path.sh 
gpssh -f /home/gpadmin/conf/seg_hosts -e 'sudo mkdir -p /data/primary'
gpssh -f /home/gpadmin/conf/seg_hosts -e 'sudo mkdir -p /data/mirror'
gpssh -f /home/gpadmin/conf/seg_hosts -e 'sudo chown -R gpadmin /data/primary'
gpssh -f /home/gpadmin/conf/seg_hosts -e 'sudo chown -R gpadmin /data/mirror'

测试内存和磁盘

gpcheckperf  -f /home/gpadmin/conf/seg_hosts -r ds -D -d /data/primary -d /data/mirror
# 该实用程序可能需要一段时间来执行测试，因为它正在主机之间复制非常大的文件。完成后，您将看到磁盘写、磁盘读和流测试的汇总结果。

分发软件包

cd /usr/local/
tar -cf ~/gp6.tar greenplum-db-6.2.1/
chown -R gpadmin:gpadmin  ~/gp6.tar
gpscp -f /home/gpadmin/conf/seg_hosts  ~/gp6.tar =:/home/gpadmin/
gpssh -f /home/gpadmin/conf/seg_hosts
=> cd /home/gpadmin
=> sudo cp ~/gp6.tar /usr/local/
=> cd /usr/local
=> sudo tar -xf gp6.tar
=> sudo ln -s greenplum-db-6.2.1 greenplum-db
=> sudo chown -R gpadmin:gpadmin  /usr/local/greenplum*
=> exit

初始化数据库

su - gpadmin
cat >>  /home/gpadmin/.bash_profile << EOF
source /usr/local/greenplum-db/greenplum_path.sh
export MASTER_DATA_DIRECTORY=/data/master/gpseg-1
export PGPORT=5432
export PGUSER=gpadmin 
export PGDATABASE=testdb
export LD_PRELOAD=/lib64/libz.so.1 ps
EOF
source ~/.bash_profile

mkdir /home/gpadmin/gpconfigs
cd /home/gpadmin/gpconfigs
cp $GPHOME/docs/cli_help/gpconfigs/gpinitsystem_config      /home/gpadmin/gpconfigs/gpinitsystem_config
cat > /home/gpadmin/gpconfigs/gpinitsystem_config << EOF
ARRAY_NAME="Greenplum Data Platform"
SEG_PREFIX=gpseg
PORT_BASE=6000    # primary端口起始
# 三个相同的目录表示在一个Segment机器上创建三个Segment实例
declare -a DATA_DIRECTORY=(/data/primary /data/primary /data/primary)
MASTER_HOSTNAME=gp-master1
MASTER_DIRECTORY=/data/master
MASTER_PORT=5432    # 监听端口
TRUSTED_SHELL=ssh
CHECK_POINT_SEGMENTS=8
ENCODING=UNICODE
MIRROR_PORT_BASE=7000   # mirror端口起始
declare -a MIRROR_DATA_DIRECTORY=(/data/mirror /data/mirror /data/mirror)
EOF

cat > hostfile_gpinitsystem << EOF
gp-node1
gp-node2
EOF

cd ~
gpinitsystem -c gpconfigs/gpinitsystem_config -h gpconfigs/hostfile_gpinitsystem
###主备master
###gpinitsystem -c gpconfigs/gpinitsystem_config -h gpconfigs/hostfile_gpinitsystem -s sgp-master1
###主备master

搞完收工，亲测集群正常跑，严格按官网的环境初始化，要不可能不成功，踩过好多坑，按照官网的环境初始化是完全没问题的，其他的就呵呵了。

Master测试

greenplum命令

gpstop 关闭

$ gpstop # 交互式关闭，等待所有数据库连接断开才停库 $ gpstop -M fast #交互式关闭，强制断开所有数据库连接，直接停库

gpstart 启动

$ gpstart # 交互式启动 $ gpstart -a # 快速启动，非交互式启动

gpstate 查看状态

$ gpstate -s # 显示所有Master和Segment信息 $ gpstate -c # 正常情况，显示primary 和 mirror 的映射关系 $ gpstate -e # 异常情况，显示primary 和 mirror 的映射关系

reload配置，刷新配置，两条都可以

$ pg_ctl reload -D /data/master/gpseg-1 $ gpstop -u

配置gpadmin数据库用户密码

$ psql -d postgres
$ postgres=# alter role gpadmin with password 'gpadmin';

允许远程用户登录和进制本地空密码登录

$ vim  /data/master/gpseg-1/pg_hba.conf
host     all         gpadmin        192.168.0.0/16        trust
local    all         gpadmin         password
$ pg_ctl reload -D  /data/master/gpseg-1

PostgreSQL 基本语法使用

psql -d testdb -U gpadmin  # 密码gpadmin 登录     
\quit     # 退出        
\l    # 查询库   
\d        # 查看当前数据库的表和视图
\c 数据库 # 切换数据库
\d tablename  # 查看表的结构
select * from uid_name; # 查看表内容
\password # 设置密码
select gp_segment_id,count(*) from name_table group by gp_segment_id; # 查看某个表在各个节点的情况
select datname,pid,application_name,state from pg_stat_activity;  # 列出当前数据库连接的信息
select * from gp_segment_configuration ; # 节点维护列表
create role testdb password 'testdb' create login; #  创建用户