k8s 部署prometheus 监控

k8s版本	prometheus
1.14.x	v0.2

1、下载、安装

下载protheus
[root@master protheus]# wget https://github.com/coreos/kube-prometheus/archive/v0.2.0.tar.gz

mkdir prometheus
cd prometheus
[root@master protheus]# tar xf v0.2.0.tar.gz 

[root@master kube-prometheus-0.2.0]# ls
build.sh            examples      jsonnet                Makefile   sync-to-internal-registry.jsonnet
code-of-conduct.md  experimental  jsonnetfile.json       manifests  tests
DCO                 go.mod        jsonnetfile.lock.json  NOTICE     test.sh
docs                go.sum        kustomization.yaml     OWNERS
example.jsonnet     hack          LICENSE                README.md

2、修改配置

#修改grafana配置，添加nodePort
[root@master manifests]# vim grafana-service.yaml
apiVersion: v1
kind: Service
metadata:
  labels:
    app: grafana
  name: grafana
  namespace: monitoring
spec:
  type: NodePort
  ports:
  - name: http
    port: 3000
    targetPort: http
    nodePort: 30100
  selector:
    app: grafana

#修改prometheus配置，添加nodePort
[root@master manifests]# vim prometheus-service.yaml
apiVersion: v1
kind: Service
metadata:
  labels:
    prometheus: k8s
  name: prometheus-k8s
  namespace: monitoring
spec:
  type: nodePort
  ports:
  - name: web
    port: 9090
    targetPort: web
    nodePort: 30200
  selector:
    app: prometheus
    prometheus: k8s

#修改grafana deployment
[root@master manifests]# vim grafana-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: grafana
  name: grafana

3、使用国内镜像

[root@master manifests]# pwd
/root/protheus/kube-prometheus-0.2.0/manifests
[root@master manifests]# grep -riE 'quay.io|k8s.gcr|grafana/' *
0prometheus-operator-deployment.yaml:        - --config-reloader-image=quay.io/coreos/configmap-reload:v0.0.1
0prometheus-operator-deployment.yaml:        - --prometheus-config-reloader=quay.io/coreos/prometheus-config-reloader:v0.33.0
0prometheus-operator-deployment.yaml:        image: quay.io/coreos/prometheus-operator:v0.33.0
alertmanager-alertmanager.yaml:  baseImage: quay.io/prometheus/alertmanager
grafana-deployment.yaml:      - image: grafana/grafana:6.2.2
grafana-deployment.yaml:        - mountPath: /etc/grafana/provisioning/datasources
grafana-deployment.yaml:        - mountPath: /etc/grafana/provisioning/dashboards
kube-state-metrics-deployment.yaml:        image: quay.io/coreos/kube-rbac-proxy:v0.4.1
kube-state-metrics-deployment.yaml:        image: quay.io/coreos/kube-rbac-proxy:v0.4.1
kube-state-metrics-deployment.yaml:        image: quay.io/coreos/kube-state-metrics:v1.7.2
kube-state-metrics-deployment.yaml:        image: k8s.gcr.io/addon-resizer:1.8.4
node-exporter-daemonset.yaml:        image: quay.io/prometheus/node-exporter:v0.18.1
node-exporter-daemonset.yaml:        image: quay.io/coreos/kube-rbac-proxy:v0.4.1
prometheus-adapter-deployment.yaml:        image: quay.io/coreos/k8s-prometheus-adapter-amd64:v0.4.1
prometheus-prometheus.yaml:  baseImage: quay.io/prometheus/prometheus

#镜像如下
quay.io/coreos/configmap-reload:v0.0.1
quay.io/coreos/prometheus-config-reloader:v0.33.0
quay.io/coreos/prometheus-operator:v0.33.0
quay.io/prometheus/alertmanager
quay.io/coreos/kube-rbac-proxy:v0.4.1
quay.io/coreos/kube-rbac-proxy:v0.4.1
quay.io/coreos/kube-state-metrics:v1.7.2
quay.io/prometheus/node-exporter:v0.18.1
quay.io/coreos/kube-rbac-proxy:v0.4.1
quay.io/coreos/k8s-prometheus-adapter-amd64:v0.4.1
quay.io/prometheus/prometheus
grafana/grafana:6.2.2
k8s.gcr.io/addon-resizer:1.8.4
alertmanager:v0.18.0
node-exporter:v0.18.1
prometheus:v2.11.0
grafana:6.2.2
addon-resizer:1.8.4

#使用脚本拉取镜像、改名

vim download.sh
#! /bin/bash
image=( kube-rbac-proxy:v0.4.1 kube-state-metrics:v1.7.2 k8s-prometheus-adapter-amd64:v0.4.1 configmap-reload:v0.0.1 prometheus-config-reloader:v0.33.0 prometheus-operator:v0.33.0 addon-resizer:1.8.4 node-exporter:v0.18.1 k8s-prometheus-adapter-amd64:v0.4.1 grafana:6.2.2 addon-resizer:1.8.4)

cn_regis="registry.cn-beijing.aliyuncs.com/cloud_registry"

for i in ${image[@]} 
do
docker pull $cn_regis/$i
docker tag $cn_regis/$i quay.io/coreos/$i
docker rmi $cn_regis/$i
done

4、安装组件及插件

#安装所有组件及插件
[root@master kube-prometheus-0.2.0]# kubectl apply -f manifests/

[root@master kube-prometheus-0.2.0]# kubectl get svc -n monitoring
NAME                    TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)                      AGE
alertmanager-main       ClusterIP   10.96.59.97      <none>        9093/TCP                     5m2s
alertmanager-operated   ClusterIP   None             <none>        9093/TCP,9094/TCP,9094/UDP   3m42s
grafana                 NodePort    10.106.80.28     <none>        3000:30100/TCP               5m1s
kube-state-metrics      ClusterIP   None             <none>        8443/TCP,9443/TCP            5m1s
node-exporter           ClusterIP   None             <none>        9100/TCP                     5m1s
prometheus-adapter      ClusterIP   10.99.233.37     <none>        443/TCP                      5m
prometheus-k8s          NodePort    10.109.153.215   <none>        9090:30200/TCP               7s
prometheus-operated     ClusterIP   None             <none>        9090/TCP                     3m41s
prometheus-operator     ClusterIP   None             <none>        8080/TCP                     5m2s
[root@master kube-prometheus-0.2.0]# kubectl top node
NAME     CPU(cores)   CPU%   MEMORY(bytes)   MEMORY%   
master   266m         13%    1248Mi          66%       
node01   187m         9%     1400Mi          74%       
node02   224m         11%    1151Mi          61%       
[root@master kube-prometheus-0.2.0]# kubectl get pods -n monitoring
NAME                                   READY   STATUS             RESTARTS   AGE
alertmanager-main-0                    2/2     Running            0          5m51s
alertmanager-main-1                    2/2     Running            0          5m51s
alertmanager-main-2                    2/2     Running            0          5m51s
grafana-598f68c777-bl5r7               1/1     Running            0          7m9s
kube-state-metrics-74894c6cf9-zjv8z    3/4     ImagePullBackOff   0          7m10s //注意镜像
node-exporter-659v6                    2/2     Running            0          7m9s
node-exporter-87n9p                    2/2     Running            0          7m9s
node-exporter-v982s                    2/2     Running            0          7m9s
prometheus-adapter-74fc6495d7-mgmr4    1/1     Running            0          7m9s
prometheus-k8s-0                       3/3     Running            1          5m50s
prometheus-k8s-1                       3/3     Running            1          5m50s
prometheus-operator-579b9fdc44-7qkpm   1/1     Running            0          7m10s

#安装完成后可以访问 prometheus 以及 grafana

http://192.168.100.50:30100/login      //grafana 登陆 admin/admin
http://192.168.100.50:30200/targets    //prometheus targets 登陆  status有很多选项可使用

#prometheus、grafana

#grafana添加prometheus源

五、使用node_exporter 监控node

#使用ID 导入模板

https://grafana.com/grafana/dashboards/1860-node-exporter-full/  使用ID号

#node_exporter 使用

https://github.com/prometheus/node_exporter#enabled-by-default
https://prometheus.io/docs/guides/node-exporter/

#node节点监控

#集群监控

#通过node-exporter 查看可调用监控内容

[root@node01 ~]# curl http://127.0.0.1:9100/metrics | grep -v "#"