16-[信创]-使用Prometheus监控二进制部署的etcd

使用Prometheus监控二进制部署的etcd

一、配置监控etcd

etcd和其他的不太一样，他是https的 需要证书 略微麻烦一些

我的etcd集群没有单独部署在三台服务器上 部署在master01-master03上了

10.1.19.51
10.1.19.52
10.1.19.53

我们需要使用自签证书来访问etcd的metrics

1.找到证书位置

####

[root@master01 manifests]# ll /etc/kubernetes/ssl/
总用量 40
-rw-r--r-- 1 root root 1675  4月 16 12:35 aggregator-proxy-key.pem
-rw-r--r-- 1 root root 1383  4月 16 12:35 aggregator-proxy.pem
-rw-r--r-- 1 root root 1679  4月 16 12:26 ca-key.pem
-rw-r--r-- 1 root root 1350  4月 16 12:24 ca.pem
-rw-r--r-- 1 root root 1679  4月 16 12:34 etcd-key.pem
-rw-r--r-- 1 root root 1428  4月 16 12:34 etcd.pem
-rw-r--r-- 1 root root 1679  4月 16 12:35 kubelet-key.pem
-rw-r--r-- 1 root root 1448  4月 16 12:35 kubelet.pem
-rw-r--r-- 1 root root 1675  4月 16 12:35 kubernetes-key.pem
-rw-r--r-- 1 root root 1655  4月 16 12:35 kubernetes.pem

测试证书能否使用

[root@master01 manifests]# curl --cacert /etc/kubernetes/ssl/ca.pem --cert /etc/kubernetes/ssl/etcd.pem --key /etc/kubernetes/ssl/etcd-key.pem https://10.1.19.51:2379/metrics | tail -5
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100  168k    0  168k    0     0  15.0M      0 --:--:-- --:--:-- --:--:-- 15.0M
# HELP promhttp_metric_handler_requests_total Total number of scrapes by HTTP status code.
# TYPE promhttp_metric_handler_requests_total counter
promhttp_metric_handler_requests_total{code="200"} 1
promhttp_metric_handler_requests_total{code="500"} 0
promhttp_metric_handler_requests_total{code="503"} 0
[root@master01 manifests]#

2.创建secrets

[root@master01 manifests]# cd /data/aqsc_k8s_yaml/kube-prometheus/kube-prometheus-0.8.0/manifests/
[root@master01 manifests]# mkdir etcd



[root@master01 etcd]# kubectl create secret -n monitoring generic etcd-secret --from-file=/etc/kubernetes/ssl/ca.pem --from-file=/etc/kubernetes/ssl/etcd.pem --from-file=/etc/kubernetes/ssl/etcd-key.pem --dry-run -o yaml > ./etcd/secret-etcd.yaml

#转换成yaml 然后创建


root@guoguo-M5-Pro:/# kubectl apply -f ./etcd/secret-etcd.yaml
secret/etcd-secret created
#创建

root@guoguo-M5-Pro:/# kubectl edit -n monitoring prometheus k8s
......
    app.kubernetes.io/component: prometheus
    app.kubernetes.io/name: prometheus
    app.kubernetes.io/part-of: kube-prometheus
    app.kubernetes.io/version: 2.26.0
    prometheus: k8s
  name: k8s
  namespace: monitoring
  resourceVersion: "10516499"
  uid: 167328a1-df5a-4249-a107-fc7189f9f20c
spec:
  secrets:  #先搜索一下 看有没有 secrets  没有就在spec.下面新建一个  然后加上上面创建的etcd-secret
  - etcd-secret  #添加的
  alerting:
    alertmanagers:
    - apiVersion: v2
      name: alertmanager-main
      namespace: monitoring
      port: web
......
......
#然后也加到配置文件里面去
[root@master01 manifests]# vim prometheus-prometheus.yaml
spec:
  secrets:  #先搜索一下 看有没有 secrets  没有就在spec.下面新建一个  然后加上上面创建的etcd-secret
  - etcd-secret  #添加的
  alerting:
    alertmanagers:
    - apiVersion: v2
      name: alertmanager-main
      namespace: monitoring
      port: web

上面是引用这个secret 我们需要进到prometheus 的pod里面查看是否挂载到了

[root@master01 manifests]# kubectl exec -it -n monitoring prometheus-k8s-0 -c prometheus -- sh
/prometheus $ ls /etc/prometheus/secrets/etcd-secret/
ca.pem        etcd-key.pem  etcd.pem
#这个目录是挂载的目录

[root@master01 manifests]# cat servicemonitor/etcd-svc-endpoints-servicemonitor.yaml
---
apiVersion: v1
kind: Service
metadata:
  name: etcd-k8s   #名字
  namespace: monitoring
  labels:
    app: etcd   #标签 这个无头svc 和endpoints 和ServiceMointor 三个要一致
spec:
  ports:
  - name: api-etcd    #定义名字  endpoints 和 ServiceMonitor 要引用
    port: 2379        #端口号
  type: ClusterIP
  clusterIP: None  #无头service
---
apiVersion: v1
kind: Endpoints
metadata:
  name: etcd-k8s   #名字要和无头svc一致
  namespace: monitoring
  labels:
    app: etcd   #上面标签
subsets:
- addresses:
  - ip: 10.1.19.51  #ETCD的IP
  - ip: 10.1.19.52
  - ip: 10.1.19.53
  ports:
  - name: api-etcd   #上面无头svc的 spec.ports.name的名字
    port: 2379       #etcd端口号
    protocol: TCP    #协议
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: etcd-monitor
  namespace: monitoring
  labels:
    app: etcd
spec:
  selector:
    matchLabels:
      app: etcd
  endpoints:
  - port: api-etcd   #名字
    scheme: HTTPS    #协议
    interval: 30s    #获取监控数据的频率
    tlsConfig:   #证书的目录 这三个证书 目录都是 上面prometheus pod挂载的目录  并不是宿主机的目录！！！切记！！！
      caFile: /etc/prometheus/secrets/etcd-secret/ca.pem
      certFile: /etc/prometheus/secrets/etcd-secret/etcd.pem
      keyFile: /etc/prometheus/secrets/etcd-secret/etcd-key.pem
      insecureSkipVerify: true
  namespaceSelector:
    matchNames:
      - monitoring

3.创建完后登录prometheus ui 界面就能看到被监控的etcd了

serviceMonitor/monitoring/etcd-monitor/0 (3/3 up)

二、配置grafana出图展示

[root@master01 manifests]# kubectl get svc -n monitoring grafana
NAME      TYPE       CLUSTER-IP      EXTERNAL-IP   PORT(S)          AGE
grafana   NodePort   10.101.232.40   <none>        3000:27477/TCP   16h

master的ip加端口号登录grafana

默认用户名admin 密码admin  首次登录需要修改密码


在grafana官网模板中心搜索etcd，下载这个json格式的模板文件
https://grafana.com/dashboards/3070
https://grafana.com/api/dashboards/3070/revisions/3/download

上传刚才下载的

有数据了