在上一篇文章中,我们了解了saltstack搭建Web集群的基础环境以及haproxy的部署,这里我们将想继续了解一下其他的web服务如何通过saltstack部署起来。

参考资料:https://github.com/unixhot/saltbook-code


SaltStack部署keepalived


编写安装配置sls文件

在keepalived目标编写install.sls:

[root@node1 /srv/salt/prod/modules/keepalived]# cat install.sls 
{% set keepalived_tar = 'keepalived-1.2.17.tar.gz' %} #利用pillar自定义变量和路径
{% set keepalived_source = 'salt://modules/keepalived/files/keepalived-1.2.17.tar.gz' %}
keepalived-install:
  file.managed:
    - name: /usr/local/src/{{ keepalived_tar }} #通过pillar变量定义,可以方便修改软件版本
    - source: {{ keepalived_source }}
    - mode: 755
    - user: root
    - group: root
  cmd.run:
    - name: cd /usr/local/src && tar zxf {{ keepalived_tar }} && \
    cd keepalived-1.2.17 && ./configure --prefix=/usr/local/keepalived \
    --disable-fwmark && make && make install
    - unless: test -d /usr/local/keepalived #存在此目录则不执行name中的命令
    - require:
      - file: keepalived-install
/etc/sysconfig/keepalived:
  file.managed:
    - source: salt://modules/keepalived/files/keepalived.sysconfig
    - mode: 644
    - user: root
    - group: root
/etc/init.d/keepalived:
  file.managed:
    - source: salt://modules/keepalived/files/keepalived.init
    - mode: 755
    - user: root
    - group: root
keepalived-init:
  cmd.run:
    - name: chkconfig --add keepalived
    - unless: chkconfig --list | grep keepalived
    - require:
      - file: /etc/init.d/keepalived
/etc/keepalived:
  file.directory:   # 目录管理,如果使用这个模块,指定的ID为一个目录
    - user: root
    - group: root

拷贝安装包和文件

将需要的安装包和配置文件拷贝到files目录:

[root@node1 /srv/salt/prod/modules/keepalived/files]# ll
total 372
-rw-r--r--. 1 root root 368827 Nov 15 16:42 keepalived-1.2.17.tar.gz
-rw-r--r--. 1 root root   1380 Nov 15 16:46 keepalived.init #启动脚本
-rw-r--r--. 1 root root    668 Nov 15 18:43 keepalived.sysconfig # sysconfig目录下的配置文件
[root@node1 /srv/salt/prod/modules/keepalived/files]# cat keepalived.sysconfig 
# Options for keepalived. See `keepalived --help' output and keepalived(8) and
# keepalived.conf(5) man pages for a list of all options. Here are the most
# common ones :
#
# --vrrp               -P    Only run with VRRP subsystem.
# --check              -C    Only run with Health-checker subsystem.
# --dont-release-vrrp  -V    Dont remove VRRP VIPs & VROUTEs on daemon stop.
# --dont-release-ipvs  -I    Dont remove IPVS topology on daemon stop.
# --dump-conf          -d    Dump the configuration data.
# --log-detail         -D    Detailed log messages.
# --log-facility       -S    0-7 Set local syslog facility (default=LOG_DAEMON)
#
KEEPALIVED_OPTIONS="-D"


定义业务参数

对keepalived的配置文件使用pillar来定义修改的参数,其中参数引用了jinja模板:

[root@node1 /srv/salt/prod/cluster/files]# cat haproxy-outside-keepalived.conf 
! Configuration File for keepalived
global_defs {
   notification_email {
     saltstack@example.com
   }
   notification_email_from keepalived@example.com
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id `ROUTEID`      #引入参数,在jinja中配置
}
vrrp_instance haproxy_ha {
state `STATEID`
interface eth0
    virtual_router_id 36
priority `PRIORITYID`
    advert_int 1
authentication {
auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
       172.16.10.62
    }
}

定义这些参数的sls文件为:

[root@node1 /srv/salt/prod/cluster]# cat haproxy-outside-keepalived.sls 
include:
  - modules.keepalived.install    # 执行keepalived的install文件
keepalived-server:
  file.managed:
    - name: /etc/keepalived/keepalived.conf
    - source: salt://cluster/files/haproxy-outside-keepalived.conf
    - mode: 644
    - user: root
    - group: root
    - template: jinja
    {% if grains['fqdn'] == 'node1' %} # 定义node1 为master
    - ROUTEID: haproxy_ha
    - STATEID: MASTER
    - PRIORITYID: 150                  # 优先级为150
    {% elif grains['fqdn'] == 'node2' %}
    - ROUTEID: haproxy_ha
    - STATEID: BACKUP                   # 定义node2为backup
    - PRIORITYID: 100                   # 优先级100
    {% endif %}
  service.running:
    - name: keepalived
    - enable: True
    - watch:
      - file: keepalived-server

修改top file,加载对应的执行文件:

[root@node1 /srv/salt/base]# cat top.sls 
base:
  '*':
    - init.init
prod:
  'node*':
    - cluster.haproxy-outside
    - cluster.haproxy-outside-keepalived

执行salt命令:

salt '*' state.highstate test=True
salt '*' state.highstate

提示:在配置中要一层层指定运行和调用的文件,对于业务上需要变动修改的文件,单独存放在/srv/salt/prod/cluster/files目录中,在它的上一级目录(也就是cluster目录)中存放sls文件,修改参数模板(jinja).

这里针对keepalived,使用salt高级命令的执行顺序是:

  1. 执行高级状态,默认去执行base环境的top.sls文件。 

  2. salt 会按照top.sls顺序执行base模块和prod模块定义的内容,此处,会根据指定的主机加载对应的文件,如果是base模块就执行 /srv/salt/base目录中指定的文件,并以这个目录为base环境的起始目录,如果是prod模块就去执行/srv/salt/prod目录中的对应文件,并以/srv/salt/prod目录为prod环境的起始目录。这里的base环境和prod环境路径都是在salt自身的配置文件中定义的。

  3. 执行prod环境中cluster目录中的haproxy-outside-keepalived.sls文件,此文件又include了一个modules/keepalived/install.sls的文件,所以会先执行prod环境中modules/keepalived/install.sls文件。

  4. install.sls会完成编译安装初始化keepalived的任务,完成之后,再执行haproxy-outside-keepalived.sls文件。

  5. haproxy-outside-keepalived.sls文件中主要负责了对不同业务上服务配置的修改。modules目录中存放的则是服务通用的基本安装配置。



SaltStack部署Memcached


部署软件包

下载部署软件包和初始配置文件,由于memcachd不需要使用配置文件来管理,所有只需要将对于的软件包放入files目录即可:

[root@node1 /srv/salt/prod/modules/memcached/files]# ls
memcached-1.4.24.tar.gz


定义安装配置sls文件

[root@node1 /srv/salt/prod/modules/memcached]# cat install.sls 
include:
  - modules.libevent.install
memcached-source-install:
  file.managed:
    - name: /usr/local/src/memcached-1.4.24.tar.gz
    - source: salt://modules/memcached/files/memcached-1.4.24.tar.gz
    - user: root
    - group: root
    - mode: 644
  cmd.run:
    - name: cd /usr/local/src && tar zxf memcached-1.4.24.tar.gz && cd memcached-1.4.24&& ./configure --prefix=/usr/local/memcached --enable-64bit --with-libevent=/usr/local/libevent && make && make install
    - unless: test -d /usr/local/memcached # 存在此目录则不执行
    - require:
      - cmd: libevent-source-install
      - file: memcached-source-install

由于需要安装libevent包,所以在安装之前还需先确认安装好libevent:

[root@node1 /srv/salt/prod/modules/libevent]# cat install.sls 
libevent-source-install:
  file.managed:
    - name: /usr/local/src/libevent-2.0.22-stable.tar
    - source: salt://modules/libevent/files/libevent-2.0.22-stable.tar
    - user: root
    - group: root
    - mode: 644
  cmd.run:
    - name: cd /usr/local/src && tar xf libevent-2.0.22-stable.tar && cd libevent-2.0.22-stable &&  ./configure --prefix=/usr/local/libevent && make && make install
    - unless: test -d /usr/local/libevent
    - require:
      - file: libevent-source-install

libevent对应的安装包:

[root@node1 /srv/salt/prod/modules/libevent/files]# ls
libevent-2.0.22-stable.tar


配置启动服务

将服务启动文件放到一个单独的bbs目录,作为一个与业务相关的配置模块:

[root@node1 /srv/salt/prod/bbs]# cat memcached.sls 
include:
  - moudles.memcached.install
  - moudles.user.www
memcached-service:
  cmd.run:
    - name: /usr/local/memcached/bin/memcached -d -m 128 -p 11211 -c 8096 -u www
    - unless: netstat -ntlp | grep 11211 # 支持多次执行salt,已经启动就不再启动
    - require:
      - cmd: memcached-source-install
      - user: www-user-group


这里定义了一个统一的用户,用于系统的标准化管理:

[root@node1 /srv/salt/prod/modules/user]# cat www.sls 
www-user-group:
  group.present:  #用户组状态模块,添加用户,并授予id 1000,防止不同服务器id不同
    - name: www
    - gid: 1000
  user.present:   #用户状态模块,指定用户信息
    - name: www
    - fullname: www
    - shell: /sbin/nologin
    - uid: 1000
    - gid: 1000


修改top file 运行高级状态

修改top file,指定memcached的安装:

# cat /srv/salt/base/top.sls
base:
  '*':
    - init.init
prod:
  'node*':
    - cluster.haproxy-outside
    - cluster.haproxy-outside-keepalived
  'node2':
    - bbs.memcached

运行高级状态:

# salt '*' state.highstate test=true
# salt '*' state.highstate

确认memcached是否启动:

# telnet 172.16.10.61 11211
Trying 172.16.10.61...
Connected to 172.16.10.61.
Escape character is '^]'.


SaltStack部署NGINX-PHP


在php中配置memcached模块只需要在php.ini加上两行参数:

session.save_handler = memcached

session.save_path = "localhost:11211"


定义安装配置PHP sls文件

在/srv/salt/prod/modules/php目录中定义install.sls文件:

[root@node1 /srv/salt/prod/modules/php]# cat install.sls 
include:
  - modules.pkg.make
  - modules.user.www

pkg-php:
  pkg.installed:
    - names:
      - mariadb-devel
      - swig
      - libjpeg-turbo
      - libjpeg-turbo-devel
      - libpng
      - libpng-devel
      - freetype
      - freetype-devel
      - libxml2
      - libxml2-devel
      - zlib
      - zlib-devel
      - libcurl
      - libcurl-devel

php-source-install:
  file.managed:
    - name: /usr/local/src/php-5.6.9.tar.gz
    - source: salt://modules/php/files/php-5.6.9.tar.gz
    - user: root
    - group: root
    - mode: 755

  cmd.run:
    - name: cd /usr/local/src && tar zxf php-5.6.9.tar.gz && cd php-5.6.9&&  ./configure --prefix=/usr/local/php-fastcgi --with-pdo-mysql=mysqlnd --with-mysqli=mysqlnd --with-mysql=mysqlnd --with-jpeg-dir --with-png-dir --with-zlib --enable-xml  --with-libxml-dir --with-curl --enable-bcmath --enable-shmop --enable-sysvsem  --enable-inline-optimization --enable-mbregex --with-openssl --enable-mbstring --with-gd --enable-gd-native-ttf --with-freetype-dir=/usr/lib64 --with-gettext=/usr/lib64 --enable-sockets --with-xmlrpc --enable-zip --enable-soap --disable-debug --enable-opcache --enable-zip --with-config-file-path=/usr/local/php-fastcgi/etc --enable-fpm --with-fpm-user=www --with-fpm-group=www && make && make install
    - require:
      - file: php-source-install
      - user: www-user-group
    - unless: test -d /usr/local/php-fastcgi

pdo-plugin:
  cmd.run:
    - name: cd /usr/local/src/php-5.6.9/ext/pdo_mysql/ && /usr/local/php-fastcgi/bin/phpize && ./configure --with-php-config=/usr/local/php-fastcgi/bin/php-config &&  make&& make install
    - unless: test -f /usr/local/php-fastcgi/lib/php/extensions/*/pdo_mysql.so
    - require:
      - cmd: php-source-install

php-ini:
  file.managed:
    - name: /usr/local/php-fastcgi/etc/php.ini
    - source: salt://modules/php/files/php.ini-production
    - user: root
    - group: root
    - mode: 644


php-fpm:
  file.managed:
    - name: /usr/local/php-fastcgi/etc/php-fpm.conf
    - source: salt://modules/php/files/php-fpm.conf.default
    - user: root
    - group: root
    - mode: 644

php-fastcgi-service:
  file.managed:
    - name: /etc/init.d/php-fpm
    - source: salt://modules/php/files/init.d.php-fpm
    - user: root
    - group: root
    - mode: 755
  cmd.run:
    - name: chkconfig --add php-fpm
    - unless: chkconfig --list | grep php-fpm
    - require:
      - file: php-fastcgi-service
  service.running:
    - name: php-fpm
    - enable: True
    - require:
      - cmd: php-fastcgi-service
    - watch:
      - file: php-ini
      - file: php-fpm

在php/files目录中,存放php的源码包和配置文件:

[root@node1 /srv/salt/prod/modules/php/files]# ll
total 116216
-rw-r--r--. 1 root root      2362 Nov 17 12:33 init.d.php-fpm
-rw-r--r--. 1 root root 118906880 Nov 17 11:06 php-5.6.28.tar
-rw-r--r--. 1 root root     22252 Nov 17 12:34 php-fpm.conf.default
-rw-r--r--. 1 root root     69599 Nov 17 12:33 php.ini-production

执行salt命令进行安装:

salt "*" state.sls modules.php.install saltenv=prod

提示:如果安装失败,可以查看错误是在那一步报错,对应的目录看文件是否分发,解压命令或编译安装命令是否正确等。


定义安装nginx sls文件

在/srv/salt/prod/modules/nginx目录中编写安装sls文件:

[root@node1 /srv/salt/prod/modules/nginx]# cat install.sls 
include:
  - modules.user.www
nginx-source-install:
  file.managed:
    - name: /usr/local/src/nginx-1.10.2.tar.gz
    - source: salt://modules/nginx/files/nginx-1.10.2.tar.gz
    - user: root
    - group: root
    - mode: 755
  cmd.run:
    - name: cd /usr/local/src && tar zxf nginx-1.10.2.tar.gz && cd nginx-1.10.2&& ./configure --prefix=/usr/local/nginx --user=www --group=www --with-http_ssl_module --with-http_stub_status_module --with-file-aio --with-http_dav_module --with-pcre && make && make install && chown -R www:www /usr/local/nginx
    - unless: test -d /usr/local/nginx
    - require:
      - user: www-user-group
      - file: nginx-source-install
      - pkg: make-pkg

nginx服务启动文件配置sls:

[root@node1 /srv/salt/prod/modules/nginx]# cat service.sls 
include:
  - modules.nginx.install

nginx-init:
  file.managed:
    - name: /etc/init.d/nginx
    - source: salt://modules/nginx/files/nginx-init
    - mode: 755
    - user: root
    - group: root
  cmd.run:
    - name: chkconfig --add nginx
    - unless: chkconfig --list | grep nginx
    - require:
      - file: nginx-init

/usr/local/nginx/conf/nginx.conf:
  file.managed:
    - source: salt://modules/nginx/files/nginx.conf
    - user: www
    - group: www
    - mode: 644 

nginx-service:
  service.running:
    - name: nginx
    - enable: True
    - reload: True
    - require:
      - cmd: nginx-init
    - watch:
      - file: /usr/local/nginx/conf/nginx.conf
      - file: nginx-online            #目录内容改变则重新reload配置

nginx-offline:
  file.directory:
    - name: /usr/local/nginx/conf/vhost_offline #下线机器的配置文件存放处

nginx-online:
  file.directory:
    - name: /usr/local/nginx/conf/vhost_online # 创建vhost_online目录


拷贝安装的软件包和配置文件

在nginx/files目录下:

[root@node1 /srv/salt/prod/modules/nginx/files]# ll
total 900
-rw-r--r-- 1 root root 910812 Nov 17 15:34 nginx-1.10.2.tar.gz
-rw-r--r-- 1 root root    621 Nov 17 15:33 nginx.conf
-rw-r--r-- 1 root root   2630 Nov 17 15:29 nginx-init

nginx默认配置文件,此文件全网统一,不同的业务配置在vhost_online目录:

[root@node1 /srv/salt/prod/modules/nginx/files]# cat nginx.conf 
user  www;
worker_processes  16;         #生产实践
error_log  logs/error.log  error;
worker_rlimit_nofile 30000;
pid        logs/nginx.pid;
events {
    use epoll;
    worker_connections  65535;
}
http {
    include       mime.types;
    default_type  application/octet-stream;
    sendfile    on;
    tcp_nopush  on;
    underscores_in_headers on;
    keepalive_timeout  10;
       send_timeout 60;
include /usr/local/nginx/conf/vhost_online/*.conf;  #此处对不同的业务定义不同的配置文件
server {
                listen         8080;
                server_name 127.0.0.1;
        location /nginx_status {
                stub_status on;
                access_log off;
                allow 127.0.0.1;
                deny all;
                }
        }
}

如果出现nginx 启动状态报错,或者启动失败的情况,先查看PID是否是在正确的位置,在启动脚本中修改pid file的文件路径。


不同业务配置

在/prod/bbs目录定义了对不同的业务所用的nginx配置文件:

[root@node1 /srv/salt/prod/bbs]# cat bbs.sls 
include:
  - modules.php.install
  - medules.php.php-memcached  # 添加memchached模块
  - medules.php.php-redis      # 添加redis模块
  - modules.nginx.service

web-bbs:
  file.managed:
    - name: /usr/local/nginx/conf/vhost_online/bbs.conf
    - source: salt://bbs/files/nginx-bbs.conf
    - user: root
    - group: root
    - mode: 644
    - require:
      - service: php-fastcgi-service
    - watch_in:
      - service: nginx-service

在bbs目录中:

[root@node1 /srv/salt/prod/bbs]# tree
.
├── bbs.sls
├── files
│   └── nginx-bbs.conf
└── memcached.sls


添加memcache和redis缓存模块

将对应的模块文件拷贝到files目录下:

[root@node1 /srv/salt/prod/modules/php/files]# ll
total 116384
-rw-r--r--. 1 root root      2362 Nov 17 12:33 init.d.php-fpm
-rw-r--r--  1 root root     36459 Nov 17 19:27 memcache-2.2.7.tgz
-rw-r--r--. 1 root root 118906880 Nov 17 11:06 php-5.6.28.tar
-rw-r--r--  1 root root     22255 Nov 17 15:16 php-fpm.conf.default
-rw-r--r--. 1 root root     69599 Nov 17 12:33 php.ini-production
-rw-r--r--  1 root root    134340 Nov 17 19:27 redis-2.2.7.tgz

对应的memcahed和redis安装sls:

[root@node1 /srv/salt/prod/modules/php]# cat php-memcache.sls 
memcache-plugin:
  file.managed:
    - name: /usr/local/src/memcache-2.2.7.tgz
    - source: salt://modules/php/files/memcache-2.2.7.tgz
    - user: root
    - group: root
    - mode: 755
  cmd.run:
    - name: cd /usr/local/src && tar zxf memcache-2.2.7.tgz && cd memcache-2.2.7&& /usr/local/php-fastcgi/bin/phpize && ./configure --enable-memcache --with-php-config=/usr/local/php-fastcgi/bin/php-config &&  make&& make install
    - unless: test -f /usr/local/php-fastcgi/lib/php/extensions/*/memcache.so
  require:
    - file: memcache-plugin
    - cmd: php-install
memcache-php-config:
  file.append:  
    - name: /usr/local/php-fastcgi/etc/php.ini
    - text:
      - extension=memcache.so


[root@node1 /srv/salt/prod/modules/php]# cat php-redis.sls
redis-plugin:
  file.managed:
    - name: /usr/local/src/redis-2.2.7.tgz
    - source: salt://modules/php/files/redis-2.2.7.tgz
    - user: root
    - group: root
    - mode: 755
  cmd.run:
    - name: cd /usr/local/src && tar zxf redis-2.2.7.tgz && cd redis-2.2.7&& /usr/local/php-fastcgi/bin/phpize && ./configure --with-php-config=/usr/local/php-fastcgi/bin/php-config &&  make&& make install
    - unless: test -f /usr/local/php-fastcgi/lib/php/extensions/*/redis.so
  require:
    - file: redis-plugin
    - cmd: php-install
redis-php-config:
  file.append:
    - name: /usr/local/php-fastcgi/etc/php.ini
    - text:
      - extension=redis.so


定义高级状态

修改top file,添加nginx和php执行模块:

[root@node1 /srv/salt/base]# cat top.sls    
base:
  '*':
    - init.init

prod:
  'node*':
    - cluster.haproxy-outside
    - cluster.haproxy-outside-keepalived
    - bbs.bbs     # 执行php nginx 模块

  'node2':
    - bbs.memcached

执行高级状态:

# salt '*' state.highstate test=true

# salt '*' state.highstate 


当执行salt 命令中,出现中断或者使用 ctrl+c 中断,可以通过jid查看进程的执行的状态:    

#salt-run jobs.lookup_jid  20161118141146222666      查看salt ID执行的状态结果,即时执行结束,仍然可以查看到

# salt '*' saltutil.running                                                  列出当前所有正在执行的job 

[root@node1 /srv/salt/prod/modules/php]# salt '*' saltutil.running     
node1:
    |_
      ----------
      arg:
      fun:
          state.highstate
      jid:
          20161118143331422864     #JID
      pid:
          21329
      ret:
      tgt:
          *
      tgt_type:
          glob
      user:
          root
node2:
    |_
      ----------
      arg:
      fun:
          state.highstate
      jid:
          20161118143331422864     #JID
      pid:
          1665
      ret:
      tgt:
          *
      tgt_type:
          glob
      user:
          root

# salt '*' saltutil.kill_job    20161118141146222666        结束进程