Kubernetes K8s Prometheus添加监控二进制安装的etcd集群

使用Prometheus监控二进制部署的etcd

一、配置监控etcd

etcd和其他的不太一样，他是https的 需要证书 略微麻烦一些

我的etcd集群是单独部署在三台服务器上的

172.17.20.104   etcd1.guoguo.com
172.17.20.105   etcd2.guoguo.com
172.17.20.106   etcd3.guoguo.com

我们需要使用自签证书来访问etcd的metrics

1.下载etcd自签的证书

因为我是单独部署的，我需要去登录到etcd的服务器去把证书下载下来

[root@etcd2 ssl]# cd /etc/kubernetes/ssl/
[root@etcd2 ssl]# ll
total 12
-rw-r--r-- 1 root root 1350 Jun  4 00:34 ca.pem
-rw-r--r-- 1 root root 1679 Jun  4 00:34 etcd-key.pem
-rw-r--r-- 1 root root 1428 Jun  4 00:34 etcd.pem
#我是用kubeasz安装的 证书是放在这个位置的
[root@etcd2 ssl]# scp /etc/kubernetes/ssl/* 172.17.0.1:/apps/k8s/prometheus/etcd/secret/
#传到创建监控的主机

测试证书能否使用

root@guoguo-M5-Pro:/apps/k8s/prometheus/etcd/secret# curl --cacert ca.pem --cert etcd.pem --key etcd-key.pem https://172.17.20.106:2379/metrics | tail -10
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100  166k    0  166k    0     0   9.9M      0 --:--:-- --:--:-- --:--:-- 10.1M
# TYPE process_virtual_memory_max_bytes gauge
process_virtual_memory_max_bytes 1.8446744073709552e+19
# HELP promhttp_metric_handler_requests_in_flight Current number of scrapes being served.
# TYPE promhttp_metric_handler_requests_in_flight gauge
promhttp_metric_handler_requests_in_flight 1
# HELP promhttp_metric_handler_requests_total Total number of scrapes by HTTP status code.
# TYPE promhttp_metric_handler_requests_total counter
promhttp_metric_handler_requests_total{code="200"} 3
promhttp_metric_handler_requests_total{code="500"} 0
promhttp_metric_handler_requests_total{code="503"} 0

2.创建secrets

root@guoguo-M5-Pro:/# kubectl create secret -n monitoring generic etcd-secret --from-file=/apps/k8s/prometheus/etcd/secret/ca.pem --from-file=/apps/k8s/prometheus/etcd/secret/etcd.pem --from-file=/apps/k8s/prometheus/etcd/secret/etcd-key.pem --dry-run -o yaml > secret-etcd.yaml
W0825 19:10:57.388357  532305 helpers.go:557] --dry-run is deprecated and can be replaced with --dry-run=client.
#转换成yaml 然后创建


root@guoguo-M5-Pro:/# kubectl apply -f secret-etcd.yaml
secret/etcd-secret created
#创建

root@guoguo-M5-Pro:/# kubectl edit -n monitoring prometheus k8s
......
    app.kubernetes.io/component: prometheus
    app.kubernetes.io/name: prometheus
    app.kubernetes.io/part-of: kube-prometheus
    app.kubernetes.io/version: 2.26.0
    prometheus: k8s
  name: k8s
  namespace: monitoring
  resourceVersion: "10516499"
  uid: 167328a1-df5a-4249-a107-fc7189f9f20c
spec:
  secrets:  #先搜索一下 看有没有 secrets  没有就在spec.下面新建一个  然后加上上面创建的etcd-secret
  - etcd-secret  #添加的
  alerting:
    alertmanagers:
    - apiVersion: v2
      name: alertmanager-main
      namespace: monitoring
      port: web
......
......

上面是引用这个secret 我们需要进到prometheus 的pod里面查看是否挂载到了

root@guoguo-M5-Pro:/# kubectl exec -it -n monitoring prometheus-k8s-0 -c prometheus -- sh
/prometheus $ ls /etc/prometheus/secrets/etcd-secret/
ca.pem        etcd-key.pem  etcd.pem
#这个目录是挂载的目录

root@guoguo-M5-Pro:/apps/k8s/prometheus/etcd# cat etcd-svc-endpoints-servicemonitor.yaml
---
apiVersion: v1
kind: Service
metadata:
  name: etcd-k8s   #名字
  namespace: monitoring
  labels:
    app: etcd   #标签 这个无头svc 和endpoints 和ServiceMointor 三个要一致
spec:
  ports:
  - name: api-etcd    #定义名字  endpoints 和 ServiceMonitor 要引用
    port: 2379        #端口号
  type: ClusterIP
  clusterIP: None  #无头service
---
apiVersion: v1
kind: Endpoints
metadata:
  name: etcd-k8s   #名字要和无头svc一致
  namespace: monitoring
  labels:
    app: etcd   #上面标签
subsets:
- addresses:
  - ip: 172.17.20.104  #ETCD的IP
  - ip: 172.17.20.105
  - ip: 172.17.20.106
  ports:
  - name: api-etcd   #上面无头svc的 spec.ports.name的名字
    port: 2379       #etcd端口号
    protocol: TCP    #协议
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: etcd-monitor
  namespace: monitoring
  labels:
    app: etcd
spec:
  selector:
    matchLabels:
      app: etcd
  endpoints:
  - port: api-etcd   #名字
    scheme: HTTPS    #协议
    interval: 30s    #获取监控数据的频率
    tlsConfig:   #证书的目录 这三个证书 目录都是 上面prometheus pod挂载的目录  并不是宿主机的目录！！！切记！！！
      caFile: /etc/prometheus/secrets/etcd-secret/ca.pem
      certFile: /etc/prometheus/secrets/etcd-secret/etcd.pem
      keyFile: /etc/prometheus/secrets/etcd-secret/etcd-key.pem
      insecureSkipVerify: true
  namespaceSelector:
    matchNames:
      - monitoring

3.创建完后登录prometheus ui 界面就能看到被监控的etcd了

serviceMonitor/monitoring/etcd-monitor/0 (3/3 up)

二、配置grafana出图展示

root@guoguo-M5-Pro:/apps/k8s# kubectl get svc -n monitoring grafana
NAME      TYPE       CLUSTER-IP      EXTERNAL-IP   PORT(S)          AGE
grafana   NodePort   10.99.178.106   <none>        3000:23204/TCP   25h

master的ip加端口号登录grafana

默认用户名admin 密码admin  首次登录需要修改密码


在grafana官网模板中心搜索etcd，下载这个json格式的模板文件
https://grafana.com/dashboards/3070
https://grafana.com/api/dashboards/3070/revisions/3/download

上传刚才下载的

有数据了