kubernetes nginx ingress controller日志efk收集分析

nginx ingress controller日志默认输出到stdout，stderror。删除链接文件，使之输出到文件。

FROM registry.cn-hangzhou.aliyuncs.com/hxpdocker/nginx-ingress-controller:0.11.0
RUN rm -rf /var/log/nginx/access.log && rm -rf /var/log/nginx/error.log

修改nginx log format：

log-format-upstream:这个参数

kind: ConfigMap
apiVersion: v1
metadata:
  name: nginx-configuration
  labels:
    app: ingress-nginx
data:
  proxy-body-size: "100m"
  log-format-upstream: '{ "@timestamp": "$time_iso8601", "@fields": {"remote_addr": "$remote_addr","remote_user": "$remote_user","body_bytes_sent": "$body_bytes_sent", "request_time": "$request_time", "status": "$status", "request": "$request", "request_method": "$request_method", "http_referrer": "$http_referer", "http_x_forwarded_for": "$http_x_forwarded_for", "http_user_agent": "$http_user_agent" } }'

efk配置文件：

kind: ConfigMap
apiVersion: v1
metadata:
  name: fluentd-es-config-nginx
  labels:
    addonmanager.kubernetes.io/mode: Reconcile
data:
  system.conf: |-
    <system>
      root_dir /tmp/fluentd-buffers/
    </system>

  input.conf: |- 
    <source>
      @id nginx.access.log
      @type tail
      path /var/log/nginx/access.log
      pos_file /var/log/nginx.access.log.pos
      time_format %Y-%m-%dT%H:%M:%S.%NZ
      tag nginx.access
      format json
      read_from_head true
    </source>
    <source>
      @type tail
      @id nginx.error.log
      path /var/log/nginx/error.log
      pos_file /var/log/nginx.error.log.pos
      tag nginx.error
      read_from_head true
      format /^(?<time>\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}) \[(?<log_level>\w+)\] (?<pid>\d+).(?<tid>\d+): (?<message>.*)$/
    </source>
    
  output.conf: |-
    <match nginx.access>
      @id elasticsearch.access
      @type elasticsearch
      @log_level info
      include_tag_key true
      host elasticsearch-logging.kube-system.svc.cluster.local
      port 9200
      logstash_format true
      logstash_prefix docker.nginx.access
      logstash_dateformat %Y-%m-%d
      type_name docker_container_nginx_access_log
      <buffer>
        @type file
        path /var/log/fluentd-buffers/nginx.access.buffer
        flush_mode interval
        retry_type exponential_backoff
        flush_thread_count 2
        flush_interval 5s
        retry_forever
        retry_max_interval 30
        chunk_limit_size 2M
        queue_limit_length 8
        overflow_action block
      </buffer>
    </match>
    
    <match nginx.error>
      @id elasticsearch.error
      @type elasticsearch
      @log_level info
      include_tag_key true
      host elasticsearch-logging.kube-system.svc.cluster.local
      port 9200
      logstash_format true
      logstash_prefix docker.nginx.error
      logstash_dateformat %Y-%m-%d
      type_name docker_container_nginx_error_log
      <buffer>
        @type file
        path /var/log/fluentd-buffers/nginx.error.buffer
        flush_mode interval
        retry_type exponential_backoff
        flush_thread_count 2
        flush_interval 5s
        retry_forever
        retry_max_interval 30
        chunk_limit_size 2M
        queue_limit_length 8
        overflow_action block
      </buffer>
    </match>

nginx ingress controller已sidecar的方式部署fluentd，发送日志到elasticsearch

apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: nginx-ingress-controller
spec:
  replicas: 1
  selector:
    matchLabels:
      app: ingress-nginx
  template:
    metadata:
      labels:
        app: ingress-nginx
      annotations:
        prometheus.io/port: '10254'
        prometheus.io/scrape: 'true'
    spec:
      serviceAccountName: nginx-ingress-serviceaccount
      hostNetwork: true
      nodeSelector:
        nginx: app
      initContainers:
      - name: init-dns
        image: busybox
        command: ['sh', '-c', 'echo -e "nameserver 10.96.0.10\nsearch nginx-app.svc.cluster.local svc.cluster.local cluster.local\noptions ndots:5" > /etc/resolv.conf']
      containers:
        - name: nginx-ingress-controller
          image: registry.cn-hangzhou.aliyuncs.com/hxpdocker/nginx-ingress-controller:0.11.0_youben
          args:
            - /nginx-ingress-controller
            - --default-backend-service=$(POD_NAMESPACE)/default-http-backend
            - --configmap=$(POD_NAMESPACE)/nginx-configuration
            - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services
            - --udp-services-configmap=$(POD_NAMESPACE)/udp-services
            - --annotations-prefix=nginx.ingress.kubernetes.io
            - --ingress-class=app
          env:
            - name: POD_NAME
              valueFrom:
                fieldRef:
                  fieldPath: metadata.name
            - name: POD_NAMESPACE
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
          ports:
          - name: http
            containerPort: 80
          - name: https
            containerPort: 443
          livenessProbe:
            failureThreshold: 3
            httpGet:
              path: /healthz
              port: 10254
              scheme: HTTP
            initialDelaySeconds: 10
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          readinessProbe:
            failureThreshold: 3
            httpGet:
              path: /healthz
              port: 10254
              scheme: HTTP
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          volumeMounts:
          - name: shared-data
            mountPath: /var/log/nginx
        - name: fluentd-es
          image: registry.cn-hangzhou.aliyuncs.com/hxpdocker/fluentd-elasticsearch:v2.0.4
          env:
          - name: FLUENTD_ARGS
            value: --no-supervisor -q
          volumeMounts:
          - name: shared-data
            mountPath: /var/log/nginx
          - name: config-volume
            mountPath: /etc/fluent/config.d
      volumes:
      - name: config-volume
        configMap:
          name: fluentd-es-config-nginx
      - name: shared-data
        emptyDir: {}