创建命名空间

vi 0zentao-namespace.yaml
apiVersion: v1
kind: Namespace
metadata:
  name: zentao

 kubectl apply -f 0zentao-namespace.yaml

创建时区及lxcfs 自动挂在docker

vi 1allow-lxcfs-tz-env.yaml
apiVersion: settings.k8s.io/v1alpha1
kind: PodPreset
metadata:
  name: allow-lxcfs-tz-env
  namespace: zentao 
  labels:
    k8s-app: zentao
spec:
  selector:
    matchLabels:
  volumeMounts:
    - mountPath: /proc/cpuinfo
      name: proc-cpuinfo
      readOnly: false
    - mountPath: /proc/diskstats
      name: proc-diskstats
      readOnly: false
    - mountPath: /proc/meminfo
      name: proc-meminfo
      readOnly: false
    - mountPath: /proc/stat
      name: proc-stat
      readOnly: false
    - mountPath: /proc/swaps
      name: proc-swaps
      readOnly: false
    - mountPath: /proc/uptime
      name: proc-uptime
      readOnly: false
    - mountPath: /etc/localtime
      name: allow-tz-env

  volumes:
    - name: proc-cpuinfo
      hostPath:
        path: /var/lib/lxcfs/proc/cpuinfo
    - name: proc-diskstats
      hostPath:
        path: /var/lib/lxcfs/proc/diskstats
    - name: proc-meminfo
      hostPath:
        path: /var/lib/lxcfs/proc/meminfo
    - name: proc-stat
      hostPath:
        path: /var/lib/lxcfs/proc/stat
    - name: proc-swaps
      hostPath:
        path: /var/lib/lxcfs/proc/swaps
    - name: proc-uptime
      hostPath:
        path: /var/lib/lxcfs/proc/uptime
    - name: allow-tz-env
      hostPath:
        path: /usr/share/zoneinfo/Asia/Shanghai

kubectl apply -f 1allow-lxcfs-tz-env.yaml

创建mysql 使用 helm v3.0.0-alpha.2


### 查找MySQL 包
root@Qist:~# helm search mysql
NAME                                    CHART VERSION   APP VERSION     DESCRIPTION
charts/mysql                            1.3.1           5.7.14          Fast, reliable, scalable, and easy to use open-...
charts/mysqldump                        2.6.0           2.4.1           A Helm chart to help backup MySQL databases usi...
charts/prometheus-mysql-exporter        0.5.1           v0.11.0         A Helm chart for prometheus mysql exporter with...
### 下载mysql 包
helm pull charts/mysql
### 解压 下载包
tar -xvf mysql-1.3.1.tgz
### 修改values.yaml
cd mysql
vi values.yaml

## mysql image version
## ref: https://hub.docker.com/r/library/mysql/tags/
##
image: "xxx.com/mysql"
# 使用MySQL8.0.17 版本
imageTag: "8.0.17" 

busybox:
  image: "xxxx/library/busybox"
  tag: "1.29.3"

testFramework:
  image: "xxxx/bats"
  tag: "0.4.0"

## Specify password for root user
##
## Default: random 10 character string
# mysqlRootPassword: testing

## Create a database user
##
# mysqlUser:
## Default: random 10 character string
# mysqlPassword:

## Allow unauthenticated access, uncomment to enable
##
# mysqlAllowEmptyPassword: true

## Create a database
##
# mysqlDatabase:

## Specify an imagePullPolicy (Required)
## It's recommended to change this to 'Always' if the image tag is 'latest'
## ref: http://kubernetes.io/docs/user-guide/images/#updating-images
##
imagePullPolicy: IfNotPresent

## Additionnal arguments that are passed to the MySQL container.
## For example use --default-authentication-plugin=mysql_native_password if older clients need to
## connect to a MySQL 8 instance.
args: []

extraVolumes: |
  # - name: extras
  #   emptyDir: {}

extraVolumeMounts: |
  # - name: extras
  #   mountPath: /usr/share/extras
  #   readOnly: true

extraInitContainers: |
  # - name: do-something
  #   image: busybox
  #   command: ['do', 'something']

# Optionally specify an array of imagePullSecrets.
# Secrets must be manually created in the namespace.
# ref: https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod
# imagePullSecrets:
  # - name: myRegistryKeySecretName

## Node selector
## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
nodeSelector: {}

## Tolerations for pod assignment
## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
##
tolerations: []

livenessProbe:
  initialDelaySeconds: 30
  periodSeconds: 10
  timeoutSeconds: 5
  successThreshold: 1
  failureThreshold: 3

readinessProbe:
  initialDelaySeconds: 5
  periodSeconds: 10
  timeoutSeconds: 1
  successThreshold: 1
  failureThreshold: 3

## Persist data to a persistent volume
persistence:
  enabled: true
  ## database data Persistent Volume Storage Class
  ## If defined, storageClassName: <storageClass>
  ## If set to "-", storageClassName: "", which disables dynamic provisioning
  ## If undefined (the default) or set to null, no storageClassName spec is
  ##   set, choosing the default provisioner.  (gp2 on AWS, standard on
  ##   GKE, AWS & OpenStack)
  ##
    # 存储大小一定要设置默认存储不然会一直创建不成功当然你可以指定
  # storageClass: "-"
  accessMode: ReadWriteOnce
  size: 20Gi
  annotations: {}

## Use an alternate scheduler, e.g. "stork".
## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/
##
# schedulerName:

## Security context
securityContext:
  enabled: false
  runAsUser: 999
  fsGroup: 999

## Configure resource requests and limits
## ref: http://kubernetes.io/docs/user-guide/compute-resources/
##
resources:
  requests:
    memory: 4096Mi
    cpu: 400m

# Custom mysql configuration files path
configurationFilesPath: /etc/mysql/conf.d/
# 添加MySQL 配置 由于8.0 默认加密放与5.7版本不一样同时zentao 还不支持8.0的加密方式所以修改默认加密方式default_authentication_plugin=mysql_native_password
# Custom mysql configuration files used to override default mysql settings
configurationFiles: {"docker.cnf": "[mysqld]\nuser  = mysql\nport  = 3306\ncharacter-set-server = utf8mb4\nskip_name_resolve = 1\ndefault_time_zone = \"+8:00\"   \nopen_files_limit    = 65535\nback_log = 1024\nmax_connections = 512\nmax_connect_errors = 1000000\ntable_open_cache = 1024\ntable_definition_cache = 1024\ntable_open_cache_instances = 64\nthread_stack = 512K\nexternal-locking = FALSE\nmax_allowed_packet = 32M\nsort_buffer_size = 4M\njoin_buffer_size = 4M\nthread_cache_size = 768\ninteractive_timeout = 600\nwait_timeout = 600\ntmp_table_size = 32M\nmax_heap_table_size = 32M\nslow_query_log = 1\nlog_timestamps = SYSTEM\ndefault_authentication_plugin=mysql_native_password"} 
#  mysql.cnf: |-
#    [mysqld]
#    skip-name-resolve
#    ssl-ca=/ssl/ca.pem
#    ssl-cert=/ssl/server-cert.pem
#    ssl-key=/ssl/server-key.pem

# Custom mysql init SQL files used to initialize the database
initializationFiles: {}
#  first-db.sql: |-
#    CREATE DATABASE IF NOT EXISTS first DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
#  second-db.sql: |-
#    CREATE DATABASE IF NOT EXISTS second DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
# 开启mysql 监控
metrics:
  enabled: true
  image: xxx.com/mysqld-exporter
  imageTag: v0.12.1
  imagePullPolicy: IfNotPresent
  resources: {}
  annotations: {}
    # prometheus.io/scrape: "true"
    # prometheus.io/port: "9104"
  livenessProbe:
    initialDelaySeconds: 15
    timeoutSeconds: 5
  readinessProbe:
    initialDelaySeconds: 5
    timeoutSeconds: 1
  flags: []
  serviceMonitor:
    enabled: true
    additionalLabels: {}

## Configure the service
## ref: http://kubernetes.io/docs/user-guide/services/
service:
  annotations: {}
  ## Specify a service type
  ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services---service-types
  type: ClusterIP
  port: 3306
  # nodePort: 32000
  # loadBalancerIP:

ssl:
  enabled: false
  secret: mysql-ssl-certs
  certificates:
#  - name: mysql-ssl-certs
#    ca: |-
#      -----BEGIN CERTIFICATE-----
#      ...
#      -----END CERTIFICATE-----
#    cert: |-
#      -----BEGIN CERTIFICATE-----
#      ...
#      -----END CERTIFICATE-----
#    key: |-
#      -----BEGIN RSA PRIVATE KEY-----
#      ...
#      -----END RSA PRIVATE KEY-----

## Populates the 'TZ' system timezone environment variable
## ref: https://dev.mysql.com/doc/refman/5.7/en/time-zone-support.html
##
## Default: nil (mysql will use image's default timezone, normally UTC)
## Example: 'Australia/Sydney'
# timezone:

# To be added to the database server pod(s)
podAnnotations: {}

podLabels: {}

## Set pod priorityClassName
# priorityClassName: {}

## Init container resources defaults
initContainer:
  resources:
    requests:
      memory: 10Mi
      cpu: 10m

创建MySQL服务

 helm install  -n zentao --name-template mysql -f values.yaml charts/mysql

kubectl get secret --namespace zentao mysql -o jsonpath="{.data.mysql-root-password}" | base64 --decode; echo
查看root 密码

创建业务使用pvc

vi zentao-pvc.yaml
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: zentao-pvc
  namespace: zentao 
  labels:
    k8s-app: zentao  
spec:
  accessModes:
    - ReadWriteMany
  storageClassName: nfs-storage
  resources:
    requests:
      storage: 50Gi

创建configmap

vi zentao-nginx-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: nginx-etc
  namespace: zentao 
  labels:
    k8s-app: zentao  
data:
  default.conf: |
    server {

            listen                  80;
            listen [::]:80 ipv6only=on;
            server_name  zentao.xxxx.com;
            client_max_body_size 1024m;
            chunked_transfer_encoding on;
            location / {
                root   /www/zentaopms/www;
                index  index.php;
                client_max_body_size 1024m;
                chunked_transfer_encoding on;

            }

            error_page  404              /404.html;
            error_page   500 502 503 504  /50x.html;
            location = /50x.html {
                root   /var/lib/nginx/html/;
            }
            location ~ \.php$ {
                root           /www/zentaopms/www;
                fastcgi_split_path_info ^(.+\.php)(/.+)$;
                fastcgi_pass   unix:/var/run/svr/php-fpm.sock;
                fastcgi_index  index.php;
                fastcgi_param  SCRIPT_FILENAME  $document_root$fastcgi_script_name;
                include        fastcgi_params;
            }
            location ~* ^.+\.(jpg|jpeg|gif|png|bmp)$ {
                access_log  off;
                root        /www/zentaopms/www;
                expires     30d;
                            break;
         }
        location /ping {
                 access_log off;
                 fastcgi_pass unix:/var/run/svr/php-fpm.sock;
                 include fastcgi_params;
                 fastcgi_param SCRIPT_FILENAME $fastcgi_script_name;
      }
    }

vi zentao-www-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: www-etc
  namespace: zentao 
  labels:
    k8s-app: zentao 
data:
  www.conf: |
    ; Start a new pool named 'www'.
    ; the variable $pool can be used in any directive and will be replaced by the
    ; pool name ('www' here)
    [www]

    ; Per pool prefix
    ; It only applies on the following directives:
    ; - 'access.log'
    ; - 'slowlog'
    ; - 'listen' (unixsocket)
    ; - 'chroot'
    ; - 'chdir'
    ; - 'php_values'
    ; - 'php_admin_values'
    ; When not set, the global prefix (or /usr/local/php7) applies instead.
    ; Note: This directive can also be relative to the global prefix.
    ; Default Value: none
    ;prefix = /path/to/pools/$pool

    ; Unix user/group of processes
    ; Note: The user is mandatory. If the group is not set, the default user's group
    ;       will be used.
    user = nginx
    group = nginx

    ; The address on which to accept FastCGI requests.
    ; Valid syntaxes are:
    ;   'ip.add.re.ss:port'    - to listen on a TCP socket to a specific IPv4 address on
    ;                            a specific port;
    ;   '[ip:6:addr:ess]:port' - to listen on a TCP socket to a specific IPv6 address on
    ;                            a specific port;
    ;   'port'                 - to listen on a TCP socket to all addresses
    ;                            (IPv6 and IPv4-mapped) on a specific port;
    ;   '/path/to/unix/socket' - to listen on a unix socket.
    ; Note: This value is mandatory.
    listen = 127.0.0.1:9000
    listen = /var/run/svr/php-fpm.sock
    ; Set listen(2) backlog.
    ; Default Value: 511 (-1 on FreeBSD and OpenBSD)
    listen.backlog = -1

    ; Set permissions for unix socket, if one is used. In Linux, read/write
    ; permissions must be set in order to allow connections from a web server. Many
    ; BSD-derived systems allow connections regardless of permissions.
    ; Default Values: user and group are set as the running user
    ;                 mode is set to 0660
    ;listen.owner = nobody
    ;listen.group = nobody
    ;listen.mode = 0660
    listen.owner = nginx
    listen.group = nginx
    listen.mode = 0666
    ; When POSIX Access Control Lists are supported you can set them using
    ; these options, value is a comma separated list of user/group names.
    ; When set, listen.owner and listen.group are ignored
    ;listen.acl_users =
    ;listen.acl_groups =

    ; List of addresses (IPv4/IPv6) of FastCGI clients which are allowed to connect.
    ; Equivalent to the FCGI_WEB_SERVER_ADDRS environment variable in the original
    ; PHP FCGI (5.2.2+). Makes sense only with a tcp listening socket. Each address
    ; must be separated by a comma. If this value is left blank, connections will be
    ; accepted from any ip address.
    ; Default Value: any
    listen.allowed_clients = 127.0.0.1

    ; Specify the nice(2) priority to apply to the pool processes (only if set)
    ; The value can vary from -19 (highest priority) to 20 (lower priority)
    ; Note: - It will only work if the FPM master process is launched as root
    ;       - The pool processes will inherit the master process priority
    ;         unless it specified otherwise
    ; Default Value: no set
    ; process.priority = -19

    ; Set the process dumpable flag (PR_SET_DUMPABLE prctl) even if the process user
    ; or group is differrent than the master process user. It allows to create process
    ; core dump and ptrace the process for the pool user.
    ; Default Value: no
    ; process.dumpable = yes

    ; Choose how the process manager will control the number of child processes.
    ; Possible Values:
    ;   static  - a fixed number (pm.max_children) of child processes;
    ;   dynamic - the number of child processes are set dynamically based on the
    ;             following directives. With this process management, there will be
    ;             always at least 1 children.
    ;             pm.max_children      - the maximum number of children that can
    ;                                    be alive at the same time.
    ;             pm.start_servers     - the number of children created on startup.
    ;             pm.min_spare_servers - the minimum number of children in 'idle'
    ;                                    state (waiting to process). If the number
    ;                                    of 'idle' processes is less than this
    ;                                    number then some children will be created.
    ;             pm.max_spare_servers - the maximum number of children in 'idle'
    ;                                    state (waiting to process). If the number
    ;                                    of 'idle' processes is greater than this
    ;                                    number then some children will be killed.
    ;  ondemand - no children are created at startup. Children will be forked when
    ;             new requests will connect. The following parameter are used:
    ;             pm.max_children           - the maximum number of children that
    ;                                         can be alive at the same time.
    ;             pm.process_idle_timeout   - The number of seconds after which
    ;                                         an idle process will be killed.
    ; Note: This value is mandatory.
    pm = static

    ; The number of child processes to be created when pm is set to 'static' and the
    ; maximum number of child processes when pm is set to 'dynamic' or 'ondemand'.
    ; This value sets the limit on the number of simultaneous requests that will be
    ; served. Equivalent to the ApacheMaxClients directive with mpm_prefork.
    ; Equivalent to the PHP_FCGI_CHILDREN environment variable in the original PHP
    ; CGI. The below defaults are based on a server without much resources. Don't
    ; forget to tweak pm.* to fit your needs.
    ; Note: Used when pm is set to 'static', 'dynamic' or 'ondemand'
    ; Note: This value is mandatory.
    pm.max_children = 30

    ; The number of child processes created on startup.
    ; Note: Used only when pm is set to 'dynamic'
    ; Default Value: min_spare_servers + (max_spare_servers - min_spare_servers) / 2
    pm.start_servers = 20

    ; The desired minimum number of idle server processes.
    ; Note: Used only when pm is set to 'dynamic'
    ; Note: Mandatory when pm is set to 'dynamic'
    pm.min_spare_servers = 5

    ; The desired maximum number of idle server processes.
    ; Note: Used only when pm is set to 'dynamic'
    ; Note: Mandatory when pm is set to 'dynamic'
    pm.max_spare_servers = 55

    ; The number of seconds after which an idle process will be killed.
    ; Note: Used only when pm is set to 'ondemand'
    ; Default Value: 10s
    ;pm.process_idle_timeout = 10s;

    ; The number of requests each child process should execute before respawning.
    ; This can be useful to work around memory leaks in 3rd party libraries. For
    ; endless request processing specify '0'. Equivalent to PHP_FCGI_MAX_REQUESTS.
    ; Default Value: 0
    pm.max_requests = 65535

    ; The URI to view the FPM status page. If this value is not set, no URI will be
    ; recognized as a status page. It shows the following informations:
    ;   pool                 - the name of the pool;
    ;   process manager      - static, dynamic or ondemand;
    ;   start time           - the date and time FPM has started;
    ;   start since          - number of seconds since FPM has started;
    ;   accepted conn        - the number of request accepted by the pool;
    ;   listen queue         - the number of request in the queue of pending
    ;                          connections (see backlog in listen(2));
    ;   max listen queue     - the maximum number of requests in the queue
    ;                          of pending connections since FPM has started;
    ;   listen queue len     - the size of the socket queue of pending connections;
    ;   idle processes       - the number of idle processes;
    ;   active processes     - the number of active processes;
    ;   total processes      - the number of idle + active processes;
    ;   max active processes - the maximum number of active processes since FPM
    ;                          has started;
    ;   max children reached - number of times, the process limit has been reached,
    ;                          when pm tries to start more children (works only for
    ;                          pm 'dynamic' and 'ondemand');
    ; Value are updated in real time.
    ; Example output:
    ;   pool:                 www
    ;   process manager:      static
    ;   start time:           01/Jul/2011:17:53:49 +0200
    ;   start since:          62636
    ;   accepted conn:        190460
    ;   listen queue:         0
    ;   max listen queue:     1
    ;   listen queue len:     42
    ;   idle processes:       4
    ;   active processes:     11
    ;   total processes:      15
    ;   max active processes: 12
    ;   max children reached: 0
    ;
    ; By default the status page output is formatted as text/plain. Passing either
    ; 'html', 'xml' or 'json' in the query string will return the corresponding
    ; output syntax. Example:
    ;   http://www.foo.bar/status
    ;   http://www.foo.bar/status?json
    ;   http://www.foo.bar/status?html
    ;   http://www.foo.bar/status?xml
    ;
    ; By default the status page only outputs short status. Passing 'full' in the
    ; query string will also return status for each pool process.
    ; Example:
    ;   http://www.foo.bar/status?full
    ;   http://www.foo.bar/status?json&full
    ;   http://www.foo.bar/status?html&full
    ;   http://www.foo.bar/status?xml&full
    ; The Full status returns for each process:
    ;   pid                  - the PID of the process;
    ;   state                - the state of the process (Idle, Running, ...);
    ;   start time           - the date and time the process has started;
    ;   start since          - the number of seconds since the process has started;
    ;   requests             - the number of requests the process has served;
    ;   request duration     - the duration in µs of the requests;
    ;   request method       - the request method (GET, POST, ...);
    ;   request URI          - the request URI with the query string;
    ;   content length       - the content length of the request (only with POST);
    ;   user                 - the user (PHP_AUTH_USER) (or '-' if not set);
    ;   script               - the main script called (or '-' if not set);
    ;   last request cpu     - the %cpu the last request consumed
    ;                          it's always 0 if the process is not in Idle state
    ;                          because CPU calculation is done when the request
    ;                          processing has terminated;
    ;   last request memory  - the max amount of memory the last request consumed
    ;                          it's always 0 if the process is not in Idle state
    ;                          because memory calculation is done when the request
    ;                          processing has terminated;
    ; If the process is in Idle state, then informations are related to the
    ; last request the process has served. Otherwise informations are related to
    ; the current request being served.
    ; Example output:
    ;   ************************
    ;   pid:                  31330
    ;   state:                Running
    ;   start time:           01/Jul/2011:17:53:49 +0200
    ;   start since:          63087
    ;   requests:             12808
    ;   request duration:     1250261
    ;   request method:       GET
    ;   request URI:          /test_mem.php?N=10000
    ;   content length:       0
    ;   user:                 -
    ;   script:               /home/fat/web/docs/php/test_mem.php
    ;   last request cpu:     0.00
    ;   last request memory:  0
    ;
    ; Note: There is a real-time FPM status monitoring sample web page available
    ;       It's available in: /usr/local/php7/share/php/fpm/status.html
    ;
    ; Note: The value must start with a leading slash (/). The value can be
    ;       anything, but it may not be a good idea to use the .php extension or it
    ;       may conflict with a real PHP file.
    ; Default Value: not set
    pm.status_path = /status

    ; The ping URI to call the monitoring page of FPM. If this value is not set, no
    ; URI will be recognized as a ping page. This could be used to test from outside
    ; that FPM is alive and responding, or to
    ; - create a graph of FPM availability (rrd or such);
    ; - remove a server from a group if it is not responding (load balancing);
    ; - trigger alerts for the operating team (24/7).
    ; Note: The value must start with a leading slash (/). The value can be
    ;       anything, but it may not be a good idea to use the .php extension or it
    ;       may conflict with a real PHP file.
    ; Default Value: not set
    ping.path = /ping

    ; This directive may be used to customize the response of a ping request. The
    ; response is formatted as text/plain with a 200 response code.
    ; Default Value: pong
    ping.response = pong

    ; The access log file
    ; Default: not set
    ;access.log = log/$pool.access.log

    ; The access log format.
    ; The following syntax is allowed
    ;  %%: the '%' character
    ;  %C: %CPU used by the request
    ;      it can accept the following format:
    ;      - %{user}C for user CPU only
    ;      - %{system}C for system CPU only
    ;      - %{total}C  for user + system CPU (default)
    ;  %d: time taken to serve the request
    ;      it can accept the following format:
    ;      - %{seconds}d (default)
    ;      - %{miliseconds}d
    ;      - %{mili}d
    ;      - %{microseconds}d
    ;      - %{micro}d
    ;  %e: an environment variable (same as $_ENV or $_SERVER)
    ;      it must be associated with embraces to specify the name of the env
    ;      variable. Some exemples:
    ;      - server specifics like: %{REQUEST_METHOD}e or %{SERVER_PROTOCOL}e
    ;      - HTTP headers like: %{HTTP_HOST}e or %{HTTP_USER_AGENT}e
    ;  %f: script filename
    ;  %l: content-length of the request (for POST request only)
    ;  %m: request method
    ;  %M: peak of memory allocated by PHP
    ;      it can accept the following format:
    ;      - %{bytes}M (default)
    ;      - %{kilobytes}M
    ;      - %{kilo}M
    ;      - %{megabytes}M
    ;      - %{mega}M
    ;  %n: pool name
    ;  %o: output header
    ;      it must be associated with embraces to specify the name of the header:
    ;      - %{Content-Type}o
    ;      - %{X-Powered-By}o
    ;      - %{Transfert-Encoding}o
    ;      - ....
    ;  %p: PID of the child that serviced the request
    ;  %P: PID of the parent of the child that serviced the request
    ;  %q: the query string
    ;  %Q: the '?' character if query string exists
    ;  %r: the request URI (without the query string, see %q and %Q)
    ;  %R: remote IP address
    ;  %s: status (response code)
    ;  %t: server time the request was received
    ;      it can accept a strftime(3) format:
    ;      %d/%b/%Y:%H:%M:%S %z (default)
    ;      The strftime(3) format must be encapsuled in a %{<strftime_format>}t tag
    ;      e.g. for a ISO8601 formatted timestring, use: %{%Y-%m-%dT%H:%M:%S%z}t
    ;  %T: time the log has been written (the request has finished)
    ;      it can accept a strftime(3) format:
    ;      %d/%b/%Y:%H:%M:%S %z (default)
    ;      The strftime(3) format must be encapsuled in a %{<strftime_format>}t tag
    ;      e.g. for a ISO8601 formatted timestring, use: %{%Y-%m-%dT%H:%M:%S%z}t
    ;  %u: remote user
    ;
    ; Default: "%R - %u %t \"%m %r\" %s"
    ;access.format = "%R - %u %t \"%m %r%Q%q\" %s %f %{mili}d %{kilo}M %C%%"

    ; The log file for slow requests
    ; Default Value: not set
    ; Note: slowlog is mandatory if request_slowlog_timeout is set
    slowlog = /dev/null

    ; The timeout for serving a single request after which a PHP backtrace will be
    ; dumped to the 'slowlog' file. A value of '0s' means 'off'.
    ; Available units: s(econds)(default), m(inutes), h(ours), or d(ays)
    ; Default Value: 0
    request_slowlog_timeout = 10s

    ; Depth of slow log stack trace.
    ; Default Value: 20
    ;request_slowlog_trace_depth = 20

    ; The timeout for serving a single request after which the worker process will
    ; be killed. This option should be used when the 'max_execution_time' ini option
    ; does not stop script execution for some reason. A value of '0' means 'off'.
    ; Available units: s(econds)(default), m(inutes), h(ours), or d(ays)
    ; Default Value: 0
    ;request_terminate_timeout = 0

    ; Set open file descriptor rlimit.
    ; Default Value: system defined value
    rlimit_files = 65535

    ; Set max core size rlimit.
    ; Possible Values: 'unlimited' or an integer greater or equal to 0
    ; Default Value: system defined value
    ;rlimit_core = 0

    ; Chroot to this directory at the start. This value must be defined as an
    ; absolute path. When this value is not set, chroot is not used.
    ; Note: you can prefix with '$prefix' to chroot to the pool prefix or one
    ; of its subdirectories. If the pool prefix is not set, the global prefix
    ; will be used instead.
    ; Note: chrooting is a great security feature and should be used whenever
    ;       possible. However, all PHP paths will be relative to the chroot
    ;       (error_log, sessions.save_path, ...).
    ; Default Value: not set
    ;chroot =

    ; Chdir to this directory at the start.
    ; Note: relative path can be used.
    ; Default Value: current directory or / when chroot
    ;chdir = /var/www

    ; Redirect worker stdout and stderr into main error log. If not set, stdout and
    ; stderr will be redirected to /dev/null according to FastCGI specs.
    ; Note: on highloaded environement, this can cause some delay in the page
    ; process time (several ms).
    ; Default Value: no
    ;catch_workers_output = yes

    ; Decorate worker output with prefix and suffix containing information about
    ; the child that writes to the log and if stdout or stderr is used as well as
    ; log level and time. This options is used only if catch_workers_output is yes.
    ; Settings to "no" will output data as written to the stdout or stderr.
    ; Default value: yes
    ;decorate_workers_output = no

    ; Clear environment in FPM workers
    ; Prevents arbitrary environment variables from reaching FPM worker processes
    ; by clearing the environment in workers before env vars specified in this
    ; pool configuration are added.
    ; Setting to "no" will make all environment variables available to PHP code
    ; via getenv(), $_ENV and $_SERVER.
    ; Default Value: yes
    ;clear_env = no

    ; Limits the extensions of the main script FPM will allow to parse. This can
    ; prevent configuration mistakes on the web server side. You should only limit
    ; FPM to .php extensions to prevent malicious users to use other extensions to
    ; execute php code.
    ; Note: set an empty value to allow all extensions.
    ; Default Value: .php
    ;security.limit_extensions = .php .php3 .php4 .php5 .php7

    ; Pass environment variables like LD_LIBRARY_PATH. All $VARIABLEs are taken from
    ; the current environment.
    ; Default Value: clean env
    ;env[HOSTNAME] = $HOSTNAME
    ;env[PATH] = /usr/local/bin:/usr/bin:/bin
    ;env[TMP] = /tmp
    ;env[TMPDIR] = /tmp
    ;env[TEMP] = /tmp

    ; Additional php.ini defines, specific to this pool of workers. These settings
    ; overwrite the values previously defined in the php.ini. The directives are the
    ; same as the PHP SAPI:
    ;   php_value/php_flag             - you can set classic ini defines which can
    ;                                    be overwritten from PHP call 'ini_set'.
    ;   php_admin_value/php_admin_flag - these directives won't be overwritten by
    ;                                     PHP call 'ini_set'
    ; For php_*flag, valid values are on, off, 1, 0, true, false, yes or no.

    ; Defining 'extension' will load the corresponding shared extension from
    ; extension_dir. Defining 'disable_functions' or 'disable_classes' will not
    ; overwrite previously defined php.ini values, but will append the new value
    ; instead.

    ; Note: path INI options can be relative and will be expanded with the prefix
    ; (pool, global or /usr/local/php7)

    ; Default Value: nothing is defined by default except the values in php.ini and
    ;                specified at startup with the -d argument
    ;php_admin_value[sendmail_path] = /usr/sbin/sendmail -t -i -f www@my.domain.com
    ;php_flag[display_errors] = off
    ;php_admin_value[error_log] = /var/log/fpm-php.www.log
    ;php_admin_flag[log_errors] = on
    ;php_admin_value[memory_limit] = 32M

创建 service

vi zentao-service.yaml
---
kind: Service
apiVersion: v1
metadata:
  labels:
    k8s-app: zentao
  name: zentao
  namespace: zentao
  annotations:
    kubernetes.io/ingress.class: traefik
    traefik.ingress.kubernetes.io/affinity: "true"
    traefik.ingress.kubernetes.io/load-balancer-method: drr
spec:
  sessionAffinity: ClientIP
  sessionAffinityConfig:
    clientIP:
      timeoutSeconds: 10800
  selector:
    k8s-app: zentao
  ports:
    - protocol: TCP
      port: 80
      name: web
    - protocol: TCP
      port: 9913
      name: http-metrics
    - protocol: TCP
      port: 9253
      name: php-metrics
  type: ClusterIP

创建Monitor

vi zentao-Monitor.yaml
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  labels:
    k8s-app: zentao
  name: zentao
  namespace: zentao
spec:
  endpoints:
  - honorLabels: true
    interval: 15s
    port: http-metrics
  - honorLabels: true
    interval: 15s
    port: php-metrics
  jobLabel: k8s-app
  selector:
    matchLabels:
      k8s-app: zentao

创建ingress

vi zentao-ingress.yaml 
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: zentao
  namespace: zentao
  annotations:
    kubernetes.io/ingress.class: traefik
    traefik.frontend.rule.type: PathPrefixStrip
spec:
  rules:
  - host: zentao.xxxx.com
    http:
      paths:
        - path: /
          backend:
            serviceName: zentao
            servicePort: 80

创建hpa

vi zentao-hpa.yaml
apiVersion: autoscaling/v2beta1
kind: HorizontalPodAutoscaler
metadata:
  name: zentao
  namespace: zentao
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: zentao
  minReplicas: 2
  maxReplicas: 10
  metrics:
  - type: Resource
    resource:
      name: memory
      targetAverageUtilization: 80
  - type: Resource
    resource:
      name: cpu
      targetAverageUtilization: 70
  - type: Pods
    pods:
      metricName: phpfpm_listen_queue
      targetAverageValue: 100

创建deployment

### 使用initContainers 去禅道官网下载zip包并解压 initContainers shell 可以改成部署及以后版本升级
同时对容器内核参数有修改请修改kubelet 配置添加如果为了安全可以删除securityContext 段不然会一直部署不成功

--allowed-unsafe-sysctls 'kernel.msg*,kernel.shm*,kernel.sem,fs.mqueue.*,net.*' \

vi zentao-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: zentao
  namespace: zentao 
  labels:
    k8s-app: zentao
spec:
  replicas: 1
  selector:
    matchLabels:
      k8s-app: zentao
  template:
    metadata:
      labels:
        k8s-app: zentao
    spec:
      securityContext:
        sysctls:
        - name: kernel.shm_rmid_forced
          value: "0"
        - name: net.core.somaxconn
          value: "65535"
        - name: kernel.msgmax
          value: "65536" 
        - name: kernel.msgmnb
          value: "65536"
        - name: kernel.shmmax
          value: "68719476736"
        - name: kernel.shmall
          value: "4294967296"
        - name: net.ipv4.tcp_mem
          value: "94500000 915000000 927000000"
        - name: net.ipv4.ip_local_port_range
          value: "1024 65535"
        - name: net.unix.max_dgram_qlen
          value: "65535"
      containers:
        - name: zentao-php
          image: docker.xxxx.com/library/alpine-php
          imagePullPolicy: IfNotPresent
          readinessProbe:
            httpGet:
              path: /ping
              port: 80
              scheme: HTTP
            initialDelaySeconds: 30
            periodSeconds: 10
          livenessProbe:
            httpGet:
              path: /ping
              port: 80
              scheme: HTTP
            initialDelaySeconds: 30
            periodSeconds: 10
          resources:
            requests:
              cpu: 200m
              memory: 200Mi
          volumeMounts:
          - name: www
            mountPath: /www
          - name: php-fpm-volume
            mountPath: /var/run/svr/
          - name: www-etc-volume
            mountPath: /etc/php-fpm.d/www.conf
            subPath: www.conf
        - name: zentao-php-fpm-exporter
          image: docker.xxxx.com/library/php-fpm_exporter
          imagePullPolicy: IfNotPresent
          env:
          - name: PHP_FPM_SCRAPE_URI
            value: "unix:///var/run/svr/php-fpm.sock;/status"
          ports:
            - containerPort: 9253
              name: php-metrics
              protocol: TCP
          resources:
            requests:
              memory: 30Mi
              cpu: 102m
            limits:
              memory: 50Mi
              cpu: 250m
          volumeMounts:
          - name: php-fpm-volume
            mountPath: /var/run/svr
        - name: zentao-nginx
          image: docker.xxxx.com/library/nginx
          imagePullPolicy: Always
          ports:
            - containerPort: 80
              name: web
              protocol: TCP
            - containerPort: 8080
              name: vts
              protocol: TCP
          readinessProbe:
            failureThreshold: 10
            httpGet:
              path: /ping
              port: 80
              scheme: HTTP
            initialDelaySeconds: 10
            periodSeconds: 3
          livenessProbe:
            httpGet:
              path: /ping            
              port: 80
              scheme: HTTP
            initialDelaySeconds: 15
            periodSeconds: 20
          resources:
            requests:
              cpu: 200m
              memory: 4096Mi
            limits:
              memory: 4096Mi
              cpu: 200m
          volumeMounts:
          - name: www
            mountPath: /www/
          - name: nginx-etc-volume
            mountPath: /etc/nginx/conf.d/default.conf
            subPath: default.conf
          - name: php-fpm-volume
            mountPath: /var/run/svr
        - name: zentao-nginx-vts-exporter
          image: docker.xxxx.com/library/nginx-vts-exporter
          imagePullPolicy: IfNotPresent
          args:
          - "-nginx.scrape_uri=http://localhost:8080/format/json"
          ports:
            - containerPort: 9913
              name: http-metrics
              protocol: TCP
          resources:
            requests:
              memory: 30Mi
              cpu: 102m
            limits:
              memory: 50Mi
              cpu: 250m                     
      initContainers:
        - name: zentao-http
          image: docker.xxxx.com/library/busybox
          imagePullPolicy: IfNotPresent
          env:
          - name: ZENTZO_VERSION
            value: "11.6"          
          command:
          - /bin/sh
          - -c
          - set -e -x;
             if [ ! -d /www/zentaopms/www ]; then
                cd /www;
                wget http://dl.cnezsoft.com/zentao/${ZENTZO_VERSION}/ZenTaoPMS.${ZENTZO_VERSION}.stable.zip;
                unzip -o ZenTaoPMS.${ZENTZO_VERSION}.stable.zip;
                rm -f ZenTaoPMS.${ZENTZO_VERSION}.stable.zip;
                echo "<?php phpinfo(); ?>" >/www/zentaopms/www/phpinfo.php;
             fi             
          volumeMounts:
           - name: www
             mountPath: /www/
      volumes:
        - name: www
          persistentVolumeClaim:
            claimName: zentao-pvc
        - name: nginx-etc-volume
          configMap:
            name: nginx-etc
            defaultMode: 0644
            items:
            - key: default.conf
              path: default.conf
        - name: www-etc-volume
          configMap:
            name: www-etc
            defaultMode: 0644
            items:
            - key: www.conf
              path: www.conf
        - name: php-fpm-volume
          emptyDir: {}

部署服务

 kubectl apply -f .
 ## 验证服务器是否启动成功
 kubectl get all -n zentao
 root@Qist:/mnt/e/work/k8s-php# kubectl get all -n zentao
NAME                          READY   STATUS    RESTARTS   AGE
pod/mysql-6889d75464-2d28n    2/2     Running   1          20h
pod/zentao-585c889f7b-c972q   4/4     Running   6          5h5m
pod/zentao-585c889f7b-vr4b2   4/4     Running   0          5h43m

NAME             TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)                    AGE
service/mysql    ClusterIP   10.64.162.181   <none>        3306/TCP,9104/TCP          20h
service/zentao   ClusterIP   10.64.137.208   <none>        80/TCP,9913/TCP,9253/TCP   18h

NAME                     READY   UP-TO-DATE   AVAILABLE   AGE
deployment.apps/mysql    1/1     1            1           20h
deployment.apps/zentao   2/2     2            2           19h

NAME                                DESIRED   CURRENT   READY   AGE
replicaset.apps/mysql-6889d75464    1         1         1       20h
replicaset.apps/zentao-585c889f7b   2         2         2       19h

NAME                                         REFERENCE           TARGETS                      MINPODS   MAXPODS   REPLICAS   AGE
horizontalpodautoscaler.autoscaling/zentao   Deployment/zentao   49%/80%, 0/100 + 1 more...   2         10        2          3h22m
### 修改hosts
我这里的ingress 服务ip 192.168.30.36
192.168.30.36 zentao.xxxx.com
访问
http://zentao.xxxx.com 
XXXX 请修改成自己的使用的域名
检查php 配置及配置mysql 这里mysql 连接地址直接添加 mysql 服务都在同一个命名空间

k8s 部署php+nginx+ mysql以zentao为例

查看prometheus 是否生成监控

k8s 部署php+nginx+ mysql以zentao为例