jenkins连接k8s执行k8s命令 jenkins k8s 持续部署

书接上文jenkins自动化部署go【docker+jenkins+go+gitlab+harbor+k8s】  我原计划是想把jenkins安装到docker，后来搞了一些时间也没有搞定所以才安装在ubuntu虚拟机上，这次尝试安装到k8s上，关于nfs的安装大家可以参考 ubuntu kubernetes中使用NFS创建pv_pvc

这里 jenkins 使用的存储为 NFS

安装 nfs 工具

#1安装nfs服务端
sudo apt install nfs-kernel-server -y
 
#2. 创建目录
sudo mkdir -p /nfs/jenkins
 
#3. 使任何客户端均可访问
sudo chown nobody:nogroup /nfs/jenkins　
#sudo chmod 755 /nfs/jenkins
sudo chmod 777 /nfs/jenkins
 
#4. 配置/etc/exports文件, 使任何ip均可访问(加入以下语句)
vi /etc/exports
/nfs/jenkins *(rw,sync,no_subtree_check)
　　
#5. 检查nfs服务的目录
sudo exportfs -ra (重新加载配置)
sudo showmount -e (查看共享的目录和允许访问的ip段)
 
#6. 重启nfs服务使以上配置生效
sudo systemctl restart nfs-kernel-server
#sudo /etc/init.d/nfs-kernel-server restart
 
#查看nfs服务的状态是否为active状态:active(exited)或active(runing)
systemctl status nfs-kernel-server
 
#7. 测试nfs服务是否成功启动
 
#安装nfs 客户端
sudo apt-get install nfs-common
#创建挂载目录
 sudo mkdir /nfs/jenkins/
 
#7.4 在主机上的Linux中测试是否正常
sudo mount -t nfs -o nolock -o tcp 192.168.100.11:/nfs/jenkins/ /nfs/jenkins/(挂载成功，说明nfs服务正常)

创建 nfs-client-provisioner deployment 

kubectl apply -f nfs-client-provisioner.yaml

kind: Deployment
apiVersion: apps/v1
metadata:
  name: nfs-client-provisioner
  namespace: kube-system
spec:
  replicas: 1
  strategy:
    type: Recreate
  selector:
    matchLabels:
      app: nfs-client-provisioner
  template:
    metadata:
      labels:
        app: nfs-client-provisioner
    spec:
      serviceAccountName: nfs-client-provisioner
      containers:
        - name: nfs-client-provisioner
          image: quay.io/external_storage/nfs-client-provisioner:latest
          volumeMounts:
            - name: nfs-client-root
              mountPath: /persistentvolumes
          env:
            - name: PROVISIONER_NAME
              value: jenkinsnfs                  # 注意这里的值不能有下划线 _
            - name: NFS_SERVER
              value: 192.168.100.11
            - name: NFS_PATH
              value: /nfs/jenkins
      volumes:
        - name: nfs-client-root
          nfs:
            server: 192.168.100.11
            path: /nfs/jenkins

## 创建 RBAC 授权
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: nfs-client-provisioner
  # replace with namespace where provisioner is deployed
  namespace: kube-system
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: nfs-client-provisioner-runner
rules:
  - apiGroups: [""]
    resources: ["persistentvolumes"]
    verbs: ["get", "list", "watch", "create", "delete"]
  - apiGroups: [""]
    resources: ["persistentvolumeclaims"]
    verbs: ["get", "list", "watch", "update"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["storageclasses"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["create", "update", "patch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: run-nfs-client-provisioner
subjects:
  - kind: ServiceAccount
    name: nfs-client-provisioner
    # replace with namespace where provisioner is deployed
    namespace: kube-system
roleRef:
  kind: ClusterRole
  name: nfs-client-provisioner-runner
  apiGroup: rbac.authorization.k8s.io
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: leader-locking-nfs-client-provisioner
  # replace with namespace where provisioner is deployed
  namespace: kube-system
rules:
  - apiGroups: [""]
    resources: ["endpoints"]
    verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: leader-locking-nfs-client-provisioner
  # replace with namespace where provisioner is deployed
  namespace: kube-system
subjects:
  - kind: ServiceAccount
    name: nfs-client-provisioner
    # replace with namespace where provisioner is deployed
    namespace: kube-system
roleRef:
  kind: Role
  name: leader-locking-nfs-client-provisioner
  apiGroup: rbac.authorization.k8s.io

 创建storageclass

名称为 jenkinsnfs，并且 provisioner 需要与 deployment 中的 PROVISIONER_NAME对应,注意这个变量不能有下划线 _

kubectl apply -f storageclass.yaml

apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: nfs
  namespace: kube-ops
provisioner: jenkinsnfs
parameters:
  archiveOnDelete: "true" # "false" 删除PVC时不会保留数据，"true"将保留PVC数据

创建 jenkins-deployment.yaml

kubectl apply -f jenkins-deployment.yaml

# 创建一个新的 namespace kube-ops 
apiVersion: v1
kind: Namespace
metadata:
  name: kube-ops
--- 
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: jenkins-claim
  namespace: kube-ops
  annotations:
    volume.beta.kubernetes.io/storage-class: "nfs"
spec:
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 2Gi

# jenkins 对应的 RBAC
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: jenkins-admin
  namespace: kube-ops       
  labels:
    name: jenkins
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: jenkins-admin
  labels:
    name: jenkins
subjects:
  - kind: ServiceAccount
    name: jenkins-admin
    namespace: kube-ops
roleRef:
  kind: ClusterRole
  name: cluster-admin
  apiGroup: rbac.authorization.k8s.io

# jenkins 对应的 svc
---
apiVersion: v1
kind: Service
metadata:
  name: jenkins
  namespace: kube-ops
  labels:
    app: jenkins
spec:
  type: NodePort
  ports:
  - name: http
    port: 8080                      #服务端口
    targetPort: 8080
    nodePort: 32001                 #NodePort方式暴露 Jenkins 端口
  - name: jnlp
    port: 50000                     #代理端口
    targetPort: 50000
    nodePort: 32002
  selector:
    app: jenkins

# jenkins Deployment
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: jenkins
  namespace: kube-ops
  labels:
    app: jenkins
spec:
  selector:
    matchLabels:
      app: jenkins
  replicas: 1
  template:
    metadata:
      labels:
        app: jenkins
    spec:
      serviceAccountName: jenkins-admin
      containers:
      - name: jenkins
        image: jenkins/jenkins:lts-alpine
        securityContext:                     
          runAsUser: 0                      #设置以ROOT用户运行容器
          privileged: true                  #拥有特权
        ports:
        - name: http
          containerPort: 8080
        - name: jnlp
          containerPort: 50000
        resources:
          limits:
            memory: 2Gi
            cpu: "2000m"
          requests:
            memory: 2Gi
            cpu: "1000m"
        env:
        - name: LIMITS_MEMORY
          valueFrom:
            resourceFieldRef:
              resource: limits.memory
              divisor: 1Mi
        - name: "JAVA_OPTS"                 #设置变量，指定时区和 jenkins slave 执行者设置
          value: " 
                   -Xmx$(LIMITS_MEMORY)m 
                   -XshowSettings:vm 
                   -Dhudson.slaves.NodeProvisioner.initialDelay=0
                   -Dhudson.slaves.NodeProvisioner.MARGIN=50
                   -Dhudson.slaves.NodeProvisioner.MARGIN0=0.85
                   -Duser.timezone=Asia/Shanghai
                 "    
        # - name: "JENKINS_OPTS"
        #  value: "--prefix=/jenkins"         #设置路径前缀加上 Jenkins，设置该选项会影响 jenkins-slave 的启动
        volumeMounts:                        #设置要挂在的目录
        - name: data
          mountPath: /var/jenkins_home
      volumes:
      - name: data
        persistentVolumeClaim:
          claimName: jenkins-claim           #设置PVC

如果已经有nfs 以上可以省略【比如我后面用kubora创建nfs， k8s在2.0.5的版本修改以上yaml】

 

# jenkins 对应的 RBAC
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: jenkins-admin
  namespace: kube-system       
  labels:
    name: jenkins
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: jenkins-admin
  labels:
    name: jenkins
subjects:
  - kind: ServiceAccount
    name: jenkins-admin
    namespace: kube-system
roleRef:
  kind: ClusterRole
  name: cluster-admin
  apiGroup: rbac.authorization.k8s.io
---
apiVersion: v1
kind: Service
metadata:
  name: jenkins
  namespace: kube-system
  labels:
    app: jenkins
spec:
  type: NodePort
  ports:
  - name: http
    port: 8080                      #服务端口
    targetPort: 8080
    nodePort: 32001                 #NodePort方式暴露 Jenkins 端口
  - name: jnlp
    port: 50000                     #代理端口
    targetPort: 50000
    nodePort: 32002
  selector:
    app: jenkins
 
# jenkins Deployment
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: jenkins
  namespace: kube-system
  labels:
    app: jenkins
spec:
  serviceName: jenkins
  selector:
    matchLabels:
      app: jenkins
  replicas: 1
  template:
    metadata:
      labels:
        app: jenkins
    spec:
      serviceAccountName: jenkins-admin
      containers:
      - name: jenkins
        image: jenkins/jenkins:lts-alpine
        securityContext:                     
          runAsUser: 0                      #设置以ROOT用户运行容器
          privileged: true                  #拥有特权
        ports:
        - name: http
          containerPort: 8080
        - name: jnlp
          containerPort: 50000
        resources:
          limits:
            memory: 2Gi
            cpu: "200m"
          requests:
            memory: 1Gi
            cpu: "100m"
        env:
        - name: LIMITS_MEMORY
          valueFrom:
            resourceFieldRef:
              resource: limits.memory
              divisor: 1Mi
        - name: "JAVA_OPTS"                 #设置变量，指定时区和 jenkins slave 执行者设置
          value: " 
                   -Xmx$(LIMITS_MEMORY)m 
                   -XshowSettings:vm 
                   -Dhudson.slaves.NodeProvisioner.initialDelay=0
                   -Dhudson.slaves.NodeProvisioner.MARGIN=50
                   -Dhudson.slaves.NodeProvisioner.MARGIN0=0.85
                   -Duser.timezone=Asia/Shanghai
                 "    
        # - name: "JENKINS_OPTS"
        #  value: "--prefix=/jenkins"         #设置路径前缀加上 Jenkins，设置该选项会影响 jenkins-slave 的启动
        volumeMounts:                        #设置要挂在的目录
        - name: data
          mountPath: /var/jenkins_home
  volumeClaimTemplates:
    - metadata:
        name: data
      spec:
         storageClassName: "nfs"
         accessModes: [ "ReadWriteMany" ]
         resources:
             requests:
                storage: 1Gi

 

获取 svc NodePort 的端口

root@k8s-master:/nfs# kubectl get pod -n kube-ops
NAME                       READY   STATUS    RESTARTS   AGE
jenkins-6ccc9676d4-khpct   1/1     Running   0          6m57s
root@k8s-master:/nfs# kubectl get svc -n kube-ops
NAME      TYPE       CLUSTER-IP      EXTERNAL-IP   PORT(S)                          AGE
jenkins   NodePort   10.99.157.233   <none>        8080:32001/TCP,50000:32002/TCP   23m
root@k8s-master:/nfs#

浏览器访问 http://192.168.100.11:32001

进入 nfs server 查看密码

动态创建 jenkins-slave 构建任务

把 Jenkins 插件源更改为国内

1. 进入 Manage Jenkins -》 Manage Plugin -> Advanced 最下面有 Update Site 设置为：https://mirrors.tuna.tsinghua.edu.cn/jenkins/updates/update-center.json
2. 修改服务器配置，进入 jenkins安装目录 ， /updates/default.json ，将其中的 updates.jenkins-ci.org/download 替换为 mirrors.tuna.tsinghua.edu.cn/jenkins ，然后把www.google.com 修改为 www.baidu.com

cd /nfs/jenkins/kube-ops-jenkins-claim-pvc-f6ba2f57-6dd3-4309-889c-2cb383e09dfa/updates
cp default.json default.json.ori
sed -i "s#www.google.com#www.baidu.com#g" default.json 
sed -i "s#updates.jenkins-ci.org/download#mirrors.tuna.tsinghua.edu.cn/jenkins#g" default.json

1. 重启Jenkins服务，，如果jenkins遇到 “There were errors checking the update sites: UnknownHostException: updates.jenkins.io” 错误， 可以考虑修改 jenkins 容器的dns 修改 vi /etc/resolv.conf文件中添加：

#nameserver 8.8.8.8
#nameserver 8.8.4.4

root@k8s-master:/nfs# kubectl get pod -n kube-ops
NAME                       READY   STATUS    RESTARTS   AGE
jenkins-8447bcb8bc-c6kxk   1/1     Running   0          25m
root@k8s-master:/nfs# kubectl exec -it jenkins-8447bcb8bc-c6kxk  -n kube-ops bash
bash-5.0#                                                                                                                                                                                                                                                                          bash-5.0# cat /etc/resolv.conf
nameserver 10.96.0.10
search kube-ops.svc.cluster.local svc.cluster.local cluster.local
options ndots:5
nameserver 8.8.8.8
nameserver 8.8.4.4
bash-5.0#

安装插件

Localization:Chinese
Git
Pipeline
Extended Choice Parameter
Kubernetes

实现Jenkins与Kubernetes整合

系统管理 -> 系统配置 -> 云 -> 新建云 -> Kubernetes

Kubernetes 地址: https://kubernetes.default.svc.cluster.localKubernetes 命名空间: kube-ops
然后点击"连接测试"，如果出现 Connection test successful 的提示信息证明 Jenkins 已经可以和 Kubernetes 系统正常通信
Jenkins 地址: http://jenkins.kube-ops.svc.cluster.local:8080

测试

创建一个 流水线 任务,流水脚本如下

//创建一个Pod的模板，label为jenkins-slave
podTemplate(label: 'jenkins-slave', cloud: 'kubernetes', containers: [
    containerTemplate(
        name: 'jnlp', 
        image: "jenkins/jnlp-slave:latest"
    )
  ]
) 
{
//引用jenkins-slave的pod模块来构建Jenkins-Slave的pod
node("jenkins-slave"){
      // 第一步
      stage('测试'){
    sh '''
            echo "hello world" 
        '''
      }
  }
}

点击执行后，你可以在 Jenkins 的日志中看到有一个 jenkins-slave 节点生成来执行任务，任务执行完成后便自动销毁

配置 Jenkins Slave 运行的 Pod 模板

配置 Jenkins Slave 运行的 Pod 模板

以便在自由风格的软件项目中使用

Pod Templates 下面添加

名称 ：jnlp

命名空间 ：kube-ops

标签列表 ：jenkins-slave

容器列表 添加
名称 ： jnlp
Docker 镜像 ：cnych/jenkins:jnlp6
运行的命令 和 命令参数 留空

挂载 /var/run/docker.sock 和 /root/.kube 和/usr/bin/docker                                  以便 cnych/jenkins:jnlp6 使用 kubectl 和 docker 命令

Service Account : jenkins-admin

测试

创建一个 自由风格的软件项目 mytest

标签表达式 ：jenkins-slave # 以上面的模板一致

构建 -> 执行 shell

echo "测试 Kubernetes 动态生成 jenkins slave"
echo "==============docker in docker==========="
docker info

echo "=============kubectl============="
kubectl get pods

点击 构建执行 后，成功进行构

GO的持续集成

由于我们的容器是没有go环境的 所以必须要有go plugin的支持【当然如有一个镜像有go，docker这些东西是更好的】【这里我尝试过很多方式都没有得到好的解决方案，就是能不能不从远程拉go的环境，比如从本地】

修改后的build.sh文件如下：

#!/bin/bash
#cd $WORKSPACE
 
export GOPROXY=https://goproxy.io
 
 #根据 go.mod 文件来处理依赖关系。
go mod tidy
 
# linux环境编译
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o main
 
# 构建docker镜像，项目中需要在当前目录下有dockerfile，否则构建失败

docker build -t gavintest .
docker tag  gavintest 192.168.100.30:8080/go/gavintest:${BUILD_NUMBER}

docker login -u admin -p '123456' 192.168.100.30:8080
docker push 192.168.100.30:8080/go/gavintest
 
docker rmi  gavintest
docker rmi 192.168.100.30:8080/go/gavintest:${BUILD_NUMBER}

kubectl set image deploy gavintest -n go  gavintest=192.168.100.30:8080/go/gavintest:${BUILD_NUMBER}
kubectl rollout status deploy gavintest -n go

创建自由风项目

 

 

 -------------------

发现go 插件不好用，于是自己把 go的文件先下载下来， 然后放到内网环境【我这里是win10 下面】，脚本 内容如下：有些时候发现 

go1.15.6.linux-amd64.tar.gz文件不需要重复下载，有时候提示 “wget: can't open 'go1.15.6.linux-amd64.tar.gz': File exists 实际就是文件已经存在

wget http://192.168.100.2/go1.15.6.linux-amd64.tar.gz
tar xfz go1.15.6.linux-amd64.tar.gz -C /usr/local
export GOROOT=/usr/local/go
export PATH=$PATH:$GOROOT/bin
go version

可以放到go 插件里面，也可以放到项目里面

-----------------------2021-04-15---------------------

我的jankin-slave 没有go，以前习惯于用docke commit 构建新的镜像， 后来发现比较困难， 还是用Dockerfile 内容如下：

FROM cnych/jenkins:jnlp6

ADD go /usr/local/
RUN export GOROOT=/usr/local/go
RUN export PATH=$PATH:$GOROOT/bin

WORKDIR /home/jenkins

ENTRYPOINT ["/usr/local/bin/jenkins-slave"]

制作镜像

docker build -t 192.168.100.30:8080/go/jenkins-agent:v1 .
docker push 192.168.100.30:8080/go/jenkins-agent

 

create secret docker-registry regsecret --docker-server=192.168.100.30:8080 --docker-username=admin --docker-password=123456 -n=go

apiVersion: apps/v1
kind: Deployment
metadata:
  name: gavintest
  namespace: go
spec:
  replicas: 3
  minReadySeconds: 10 
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxSurge: 25%
      maxUnavailable: 25%
  selector:
    matchLabels:
      name: gavintest
  template:
    metadata:
      labels:
        name: gavintest
    spec:
      imagePullSecrets:
      - name: regsecret
      containers:
      - name: gavintest
        image: 192.168.100.30:8080/go/gavintest:20210301
        ports:
        - containerPort: 80
        imagePullPolicy: Always
        volumeMounts:
         - mountPath: "/app/conf"
           name: httpd-volume
      volumes:
      - name: httpd-volume
        nfs:                        
           server: 192.168.100.11        
           path: "/nfs/data"                   
---
apiVersion: v1 
kind: Service 
metadata:
  name: gavintest
  namespace: go 
spec:
  type: ClusterIP
  ports:
    - port: 80 
      targetPort: 80 
      protocol: TCP 
  selector:
    name: gavintest
---
apiVersion: extensions/v1beta1 
kind: Ingress 
metadata:
  name: gavintest
  namespace: go
spec:
  rules:
    - host: go.k8s.com
      http:
        paths: 
          - path: /  
            backend:
              serviceName: gavintest
              servicePort: 80