基于nerdctl + buildkitd+containerd构建容器镜像
- 处理逻辑
软件部署
- 部署nerdctl
#官方源码
https://github.com/containerd/nerdctl
#下载并安装nerdctl
[root@K8s-ansible containerd-bin]#wget https://github.com/containerd/nerdctl/releases/download/v1.3.0/nerdctl-1.3.0-linux-amd64.tar.gz
[root@K8s-ansible containerd-bin]#tar xvf nerdctl-1.3.0-linux-amd64.tar.gz
nerdctl
containerd-rootless-setuptool.sh
containerd-rootless.sh
[root@K8s-ansible containerd-bin]#ls nerdctl
nerdctl
#将nerdctl复制到Master节点和Node节点的/usr/local/bin下
[root@K8s-ansible containerd-bin]#scp nerdctl 192.168.11.211:/usr/local/bin/
nerdctl 100% 24MB 70.2MB/s 00:00
#重启kubelet kube-proxy containerd 如果是已经有业务的节点，需要驱逐业务后再重启，慎用
[root@K8s-master01 ~]#systemctl restart kubelet kube-proxy containerd
[root@K8s-master01 ~]#nerdctl --version
nerdctl version 1.3.0
#添加命令补全
[root@K8s-master01 ~] vim /etc/profile
source <(nerdctl completion bash)
[root@K8s-master01 ~] source /etc/profile
#nerdctl默认使用defalut,需要切换至k8s.io的namespace才能显示对方的镜像信息
[root@K8s-master01 ~]#mkdir -p /etc/nerdctl/
[root@K8s-master01 ~]#vim /etc/nerdctl/nerdctl.toml
[root@K8s-master01 ~]#cat /etc/nerdctl/nerdctl.toml
namespace = "k8s.io"
#测试nerdctl查看kubernetes内镜像，如果非集群内镜像，可以不用添加
[root@K8s-master01 ubuntu]#nerdctl images
REPOSITORY TAG IMAGE ID CREATED PLATFORM SIZE BLOB SIZE
K8s-harbor01.mooreyxia.com/kubernetes/calico/cni v3.24.5 6d29e8402585 9 days ago linux/amd64 188.5 MiB 83.4 MiB
K8s-harbor01.mooreyxia.com/kubernetes/calico/cni <none> 6d29e8402585 9 days ago linux/amd64 188.5 MiB 83.4 MiB
K8s-harbor01.mooreyxia.com/kubernetes/calico/node v3.24.5 5c614b62b13d 9 days ago linux/amd64 224.4 MiB 77.8 MiB
K8s-harbor01.mooreyxia.com/kubernetes/calico/node <none> 5c614b62b13d 9 days ago linux/amd64 224.4 MiB 77.8 MiB
K8s-harbor01.mooreyxia.com/kubernetes/easzlab/pause 3.9 3ec9d4ec5512 9 days ago linux/amd64 732.0 KiB 312.4 KiB
K8s-harbor01.mooreyxia.com/kubernetes/easzlab/pause <none> 3ec9d4ec5512 9 days ago linux/amd64 732.0 KiB 312.4 KiB
nginx 1.20.2-alpine 74694f2de64c 2 days ago linux/amd64 26.1 MiB 9.6 MiB
nginx <none> 74694f2de64c 2 days ago linux/amd64 26.1 MiB 9.6 MiB
#创建容器cni运行环境，否则无法用nerdctl运行容器
#官方源码
https://github.com/containernetworking/plugins
#安装cni
[root@K8s-master01 ubuntu]#cd /opt/
[root@K8s-master01 opt]#wget https://github.com/containernetworking/plugins/releases/download/v1.2.0/cni-plugins-linux-amd64-v1.2.0.tgz
[root@K8s-master01 opt]#ls cni-plugins-linux-amd64-v1.2.0.tgz
cni-plugins-linux-amd64-v1.2.0.tgz
[root@K8s-master01 opt]#mkdir -p /opt/cni/bin
[root@K8s-master01 opt]#tar xvf cni-plugins-linux-amd64-v1.2.0.tgz -C /opt/cni/bin/
./
./loopback
./bandwidth
./ptp
./vlan
./host-device
./tuning
./vrf
./sbr
./dhcp
./static
./firewall
./macvlan
./dummy
./bridge
./ipvlan
./portmap
./host-local
[root@K8s-master01 opt]#tree /opt/cni/bin/
/opt/cni/bin/
├── bandwidth
├── bridge
├── dhcp
├── dummy
├── firewall
├── host-device
├── host-local
├── ipvlan
├── loopback
├── macvlan
├── portmap
├── ptp
├── sbr
├── static
├── tuning
├── vlan
└── vrf
0 directories, 17 files
#测试nerdctl运行容器
[root@K8s-master01 ~]#nerdctl run -p 81:80 K8s-harbor01.mooreyxia.com/baseimages/nginx-base:1.22.0
[root@K8s-dns ~]#curl http://192.168.11.211:81/
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Nginx 测试页面</title>
</head>
<body>
<h2>Nginx 测试web页面</h2>
<img src="./images/1.jpg">
<p>
<a href="http://www.jd.com" target="_blank">app跳转</a>
</p>
</body>
</html>- 部署buildkitd
- buildkitd组成部分
#官方源码
https://github.com/moby/buildkit
buildkitd(服务端)，⽬前⽀持runc和containerd作为镜像构建环境，默认是runc，可以更换为containerd。buildctl(客户端)，负责解析Dockerfile⽂件，并向服务端buildkitd发出构建请求。- 部署过程
- 注意：安装buildkit需要安装环境只有containerd没有docker,避免安装冲突
#下载buildkit源码并解压
[root@K8s-master01 ~]# cd /usr/local/src/
[root@K8s-master01 src]#wget https://github.com/moby/buildkit/releases/download/v0.11.5/buildkit-v0.11.5.linux-amd64.tar.gz
[root@K8s-master01 src]#ls
buildkit-v0.11.5.linux-amd64.tar.gz
[root@K8s-master01 src]#tar -xvf buildkit-v0.11.5.linux-amd64.tar.gz -C /usr/local/bin/
bin/
bin/buildctl
bin/buildkit-qemu-aarch64
bin/buildkit-qemu-arm
bin/buildkit-qemu-i386
bin/buildkit-qemu-mips64
bin/buildkit-qemu-mips64el
bin/buildkit-qemu-ppc64le
bin/buildkit-qemu-riscv64
bin/buildkit-qemu-s390x
bin/buildkit-runc
bin/buildkitd
#复制可文件到/usr/local/bin下
[root@K8s-master01 src]#mv /usr/local/bin/bin/buildctl /usr/local/bin/bin/buildkitd /usr/local/bin/
[root@K8s-master01 src]#ls /usr/local/bin/build*
/usr/local/bin/buildctl /usr/local/bin/buildkitd
#创建buildkit.socket套接字文件
[root@K8s-master01 src]#vim /lib/systemd/system/buildkit.socket
[root@K8s-master01 src]#cat /lib/systemd/system/buildkit.socket
[Unit]
Descriptinotallow=BuildKit
Documentatinotallow=https://github.com/moby/buildkit
[Socket]
ListenStream=%t/buildkit/buildkitd.sock
[Install]
WantedBy=sockets.target
#创建buildkitd.service
[root@K8s-master01 src]#vim /lib/systemd/system/buildkit.service
[root@K8s-master01 src]#cat /lib/systemd/system/buildkit.service
[Unit]
Descriptinotallow=BuildKit
Requires=buildkit.socket
After=buildkit.socketDocumentatinotallow=https://github.com/moby/buildkit
[Service]
ExecStart=/usr/local/bin/buildkitd --oci-worker=false --containerd-worker=true
[Install]
WantedBy=multi-user.target
#加载服务
[root@K8s-master01 src]#systemctl daemon-reload
[root@K8s-master01 src]#systemctl enable --now buildkit
Created symlink /etc/systemd/system/multi-user.target.wants/buildkit.service → /lib/systemd/system/buildkit.service.
[root@K8s-master01 src]#systemctl status buildkit
● buildkit.service - BuildKit
Loaded: loaded (/lib/systemd/system/buildkit.service; enabled; vendor preset: enabled)
Active: active (running) since Sat 2023-04-08 02:05:46 UTC; 57s ago
TriggeredBy: ● buildkit.socket
Main PID: 31137 (buildkitd)
Tasks: 8 (limit: 2234)
Memory: 7.6M
CPU: 68ms
CGroup: /system.slice/buildkit.service
└─31137 /usr/local/bin/buildkitd --oci-worker=false --containerd-worker=true
Apr 08 02:05:46 K8s-master01.mooreyxia.com systemd[1]: Started BuildKit.
Apr 08 02:05:46 K8s-master01.mooreyxia.com buildkitd[31137]: time="2023-04-08T02:05:46Z" level=warning msg="using host network as the default"
Apr 08 02:05:46 K8s-master01.mooreyxia.com buildkitd[31137]: time="2023-04-08T02:05:46Z" level=warning msg="git source cannot be enabled: failed to find git binary: exec: \"git\":
executable file not found in $PATH"
Apr 08 02:05:46 K8s-master01.mooreyxia.com buildkitd[31137]: time="2023-04-08T02:05:46Z" level=info msg="found worker \"ph4qqnwb8sk8awqhtrqse2xzs\", labels=map[org.mobyproject.bui
ldkit.worker.containerd.namespace:buildkit org.mobyproject.buildkit.worker.containerd.uuid:627774aa-294c-43ef-a5fa-01d506629339 org.mobyproject.buildkit.worker.executor:containerd
org.mobyproject.buildkit.worker.hostname:K8s-master01.mooreyxia.com org.mobyproject.buildkit.worker.network:host org.mobyproject.buildkit.worker.selinux.enabled:false org.mobypro
ject.buildkit.worker.snapshotter:overlayfs], platforms=[linux/amd64 linux/amd64/v2 linux/amd64/v3 linux/386]"
Apr 08 02:05:46 K8s-master01.mooreyxia.com buildkitd[31137]: time="2023-04-08T02:05:46Z" level=info msg="found 1 workers, default=\"ph4qqnwb8sk8awqhtrqse2xzs\""
Apr 08 02:05:46 K8s-master01.mooreyxia.com buildkitd[31137]: time="2023-04-08T02:05:46Z" level=warning msg="currently, only the default worker can be used."
Apr 08 02:05:46 K8s-master01.mooreyxia.com buildkitd[31137]: time="2023-04-08T02:05:46Z" level=info msg="running server on /run/buildkit/buildkitd.sock"构建镜像
- 加入nginx代理访问harbor
#DNS中加入nginx代理解析
k8s-nginx A 192.168.11.200
[root@K8s-dns ~]#dig k8s-nginx.mooreyxia.com
; <<>> DiG 9.18.1-1ubuntu1.3-Ubuntu <<>> k8s-nginx.mooreyxia.com
;; global options: +cmd
;; Got answer:
;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 61537
;; flags: qr rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1
;; OPT PSEUDOSECTION:
; EDNS: version: 0, flags:; udp: 65494
;; QUESTION SECTION:
;k8s-nginx.mooreyxia.com. IN A
;; ANSWER SECTION:
k8s-nginx.mooreyxia.com. 86400 IN A 192.168.11.200
;; Query time: 4 msec
;; SERVER: 127.0.0.53#53(127.0.0.53) (UDP)
;; WHEN: Sat Apr 08 02:23:29 UTC 2023
;; MSG SIZE rcvd: 68
#部署Nginx
[root@K8s-dns ~]#cat nginx_install.sh
#!/bin/bash
NGINX_FILE=nginx-1.22.1
NGINX_URL=http://nginx.org/download/
TAR=.tar.gz
SRC_DIR=/usr/local/src
NGINX_INSTALL_DIR=/apps/nginx
CPUS=`lscpu |awk '/^CPU\(s\)/{print $2}'`
. /etc/os-release
color () {
RES_COL=60
MOVE_TO_COL="echo -en \\033[${RES_COL}G"
SETCOLOR_SUCCESS="echo -en \\033[1;32m"
SETCOLOR_FAILURE="echo -en \\033[1;31m"
SETCOLOR_WARNING="echo -en \\033[1;33m"
SETCOLOR_NORMAL="echo -en \E[0m"
echo -n "$1" && $MOVE_TO_COL
echo -n "["
if [ $2 = "success" -o $2 = "0" ] ;then
${SETCOLOR_SUCCESS}
echo -n $" OK "
elif [ $2 = "failure" -o $2 = "1" ] ;then
${SETCOLOR_FAILURE}
echo -n $"FAILED"
else
${SETCOLOR_WARNING}
echo -n $"WARNING"
fi
${SETCOLOR_NORMAL}
echo -n "]"
echo
}
check () {
[ -e ${NGINX_INSTALL_DIR} ] && { color "nginx 已安装,请卸载后再安装" 1; exit; }
cd ${SRC_DIR}
if [ -e ${NGINX_FILE}${TAR} ];then
color "相关文件已准备好" 0
else
color '开始下载 nginx 源码包' 0
wget ${NGINX_URL}${NGINX_FILE}${TAR}
[ $? -ne 0 ] && { color "下载 ${NGINX_FILE}${TAR}文件失败" 1; exit; }
fi
}
install () {
color "开始安装 nginx" 0
if id nginx &> /dev/null;then
color "nginx 用户已存在" 1
else
useradd -s /sbin/nologin -r nginx
color "创建 nginx 用户" 0
fi
color "开始安装 nginx 依赖包" 0
if [ $ID == "centos" ] ;then
if [[ $VERSION_ID =~ ^7 ]];then
yum -y install gcc make pcre-devel openssl-devel zlib-devel perl-ExtUtils-Embed
elif [[ $VERSION_ID =~ ^8 ]];then
yum -y install make gcc-c++ libtool pcre pcre-devel zlib zlib-devel openssl openssl-devel perl-ExtUtils-Embed
else
color '不支持此系统!' 1
exit
fi
elif [ $ID == "rocky" ];then
yum -y install gcc make gcc-c++ libtool pcre pcre-devel zlib zlib-devel openssl openssl-devel perl-ExtUtils-Embed
else
apt update
apt -y install gcc make libpcre3 libpcre3-dev openssl libssl-dev zlib1g-dev
fi
[ $? -ne 0 ] && { color "安装依赖包失败" 1; exit; }
cd $SRC_DIR
tar xf ${NGINX_FILE}${TAR}
NGINX_DIR=`echo ${NGINX_FILE}${TAR}| sed -nr 's/^(.*[0-9]).*/\1/p'`
cd ${NGINX_DIR}
./configure --prefix=${NGINX_INSTALL_DIR} --user=nginx --group=nginx --with-http_ssl_module --with-http_v2_module --with-http_realip_module --with-http_stub_status_module --with-http_gzip_static_module --with-pcre --with-stream --with-stream_ssl_module --with-stream_realip_module
make -j $CPUS && make install
[ $? -eq 0 ] && color "nginx 编译安装成功" 0 || { color "nginx 编译安装失败,退出!" 1 ;exit; }
chown -R nginx.nginx ${NGINX_INSTALL_DIR}
echo "PATH=${NGINX_INSTALL_DIR}/sbin:${PATH}" > /etc/profile.d/nginx.sh
cat > /lib/systemd/system/nginx.service <<EOF
[Unit]
Descriptinotallow=The nginx HTTP and reverse proxy server
After=network.target remote-fs.target nss-lookup.target
[Service]
Type=forking
PIDFile=${NGINX_INSTALL_DIR}/logs/nginx.pid
ExecStartPre=/bin/rm -f ${NGINX_INSTALL_DIR}/logs/nginx.pid
ExecStartPre=${NGINX_INSTALL_DIR}/sbin/nginx -t
ExecStart=${NGINX_INSTALL_DIR}/sbin/nginx
ExecReload=/bin/kill -s HUP \$MAINPID
KillSignal=SIGQUIT
TimeoutStopSec=5
KillMode=process
PrivateTmp=true
LimitNOFILE=100000
[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable --now nginx &> /dev/null
systemctl is-active nginx &> /dev/null || { color "nginx 启动失败,退出!" 1 ; exit; }
color "nginx 安装完成" 0
}
check
install
exec bash
[root@K8s-dns ~]#bash nginx_install.sh
#创建一个证书目录并存放harbor生成的证书
[root@K8s-dns nginx-1.22.1]#cd
[root@K8s-dns ~]#mkdir -p /apps/nginx/certs
#将harbor复制到目录下
[root@K8s-harbor01 certs]#ls
K8s-harbor01.mooreyxia.com.cert K8s-harbor01.mooreyxia.com.crt K8s-harbor01.mooreyxia.com.csr K8s-harbor01.mooreyxia.com.key ca.crt ca.key scp.sh v3.ext
[root@K8s-harbor01 certs]#scp K8s-harbor01.mooreyxia.com.cert K8s-harbor01.mooreyxia.com.key 192.168.11.200:/apps/nginx/certs/
[root@K8s-dns ~]#tree /apps/nginx/certs
/apps/nginx/certs
├── K8s-harbor01.mooreyxia.com.cert
└── K8s-harbor01.mooreyxia.com.key
0 directories, 2 files
#编辑nginx配置⽂件：
[root@K8s-dns ~]#cat /apps/nginx/conf/nginx.conf
...
client_max_body_size 1000m;
server {
listen 80;
listen 443 ssl;
ssl_certificate /apps/nginx/certs/K8s-harbor01.mooreyxia.com.cert;
ssl_certificate_key /apps/nginx/certs/K8s-harbor01.mooreyxia.com.key;
ssl_session_cache shared:sslcache:20m;
ssl_session_timeout 10m;
server_name K8s-harbor01.mooreyxia.com;
#charset koi8-r;
#access_log logs/host.access.log main;
location / {
# root html;
# index index.html index.htm;
proxy_pass http://192.168.11.201;
}
...
[root@K8s-dns ~]#/apps/nginx/sbin/nginx -t
nginx: the configuration file /apps/nginx/conf/nginx.conf syntax is ok
nginx: configuration file /apps/nginx/conf/nginx.conf test is successful
#重新加载Nginx配置
[root@K8s-dns ~]#/apps/nginx/sbin/nginx -s reload
[root@K8s-dns ~]#ss -ntlp|grep 443
LISTEN 0 511 0.0.0.0:443 0.0.0.0:* users:(("nginx",pid=4788,fd=10),("nginx",pid=4435,fd=10))
LISTEN 0 511 0.0.0.0:80 0.0.0.0:* users:(("nginx",pid=4788,fd=6),("nginx",pid=4435,fd=6))
#浏览器访问
https://k8s-nginx.mooreyxia.com
- harbor证书分发到buildkit控制节点，授权登录
- 如果是临时使用可以省略这一步使用nerdctl login --insecure-registry harbor
[root@K8s-master01 ~]#mkdir -p /etc/containerd/certs.d/K8s-harbor01.mooreyxia.com
#复制证书
[root@K8s-harbor01 certs]#ls
K8s-harbor01.mooreyxia.com.cert K8s-harbor01.mooreyxia.com.crt K8s-harbor01.mooreyxia.com.csr K8s-harbor01.mooreyxia.com.key ca.crt ca.key scp.sh v3.ext
[root@K8s-harbor01 certs]#scp K8s-harbor01.mooreyxia.com.cert K8s-harbor01.mooreyxia.com.key ca.crt 192.168.11.211:/etc/containerd/certs.d/K8s-harbor01.mooreyxia.com
root@192.168.11.211's password:
K8s-harbor01.mooreyxia.com.cert 100% 2195 274.7KB/s 00:00
K8s-harbor01.mooreyxia.com.key 100% 3272 340.4KB/s 00:00
ca.crt 100% 2049 224.3KB/s 00:00
#验证登录
[root@K8s-master01 ~]#nerdctl login K8s-harbor01.mooreyxia.com
Enter Username: admin
Enter Password:
WARNING: Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store
Login Succeeded- 镜像构建
#用ubuntu官方镜像构建nginx镜像并上传到harbor
[root@K8s-master01 ubuntu]#cat Dockerfile
FROM ubuntu:22.04
MAINTAINER "mooreyxia mooreyxia@gmail.com"
RUN apt update && apt install -y iproute2 ntpdate tcpdump telnet traceroute nfs-kernel-server nfs-common lrzsz tree openssl libssl-dev libpcre3 libpcre3-dev zlib1g-dev ntpdate tcpdump telnet traceroute gcc openssh-server lrzsz tree openssl libssl-dev libpcre3 libpcre3-dev zlib1g-dev ntpdate tcpdump telnet traceroute iotop unzip zip make
ADD nginx-1.22.0.tar.gz /usr/local/src/
RUN cd /usr/local/src/nginx-1.22.0 && ./configure --prefix=/apps/nginx && make && make install && ln -sv /apps/nginx/sbin/nginx /usr/bin
RUN groupadd -g 2088 nginx && useradd -g nginx -s /usr/sbin/nologin -u 2088 nginx && chown -R nginx.nginx /apps/nginx
ADD nginx.conf /apps/nginx/conf/
ADD frontend.tar.gz /apps/nginx/html/
EXPOSE 80 443
CMD ["nginx","-g","daemon off;"]
#准备一个前端页面
[root@K8s-master01 ubuntu]#cat html/index.html
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Nginx 测试页面</title>
</head>
<body>
<h2>Nginx 测试web页面</h2>
<img src="./images/1.jpg">
<p>
<a href="http://www.jd.com" target="_blank">app跳转</a>
</p>
</body>
</html>
#构建镜像
[root@K8s-master01 ubuntu]#/usr/local/bin/nerdctl build -t K8s-harbor01.mooreyxia.com/baseimages/nginx-base:1.22.0 .
[+] Building 284.8s (12/12)
[+] Building 285.4s (12/12) FINISHED
=> [internal] load .dockerignore 0.5s
=> => transferring context: 2B 0.0s
=> [internal] load build definition from Dockerfile 0.6s
=> => transferring dockerfile: 931B 0.0s
=> [internal] load metadata for docker.io/library/ubuntu:22.04 13.3s
=> [1/7] FROM docker.io/library/ubuntu:22.04@sha256:67211c14fa74f070d27cc59d69a7fa9aeff8e28ea118ef3babc295a0428a6d21 7.0s
=> => resolve docker.io/library/ubuntu:22.04@sha256:67211c14fa74f070d27cc59d69a7fa9aeff8e28ea118ef3babc295a0428a6d21 0.3s
=> => sha256:2ab09b027e7f3a0c2e8bb1944ac46de38cebab7145f0bd6effebfe5492c818b6 29.53MB / 29.53MB 4.0s
=> => extracting sha256:2ab09b027e7f3a0c2e8bb1944ac46de38cebab7145f0bd6effebfe5492c818b6 2.3s
=> [internal] load build context 0.5s
=> => transferring context: 1.12MB 0.1s
=> [2/7] RUN apt update && apt install -y iproute2 ntpdate tcpdump telnet traceroute nfs-kernel-server nfs-common lrzsz tree openssl libssl-dev libpcre3 libpcre3-de 170.5s
=> [3/7] ADD nginx-1.22.0.tar.gz /usr/local/src/ 1.1s
=> [4/7] RUN cd /usr/local/src/nginx-1.22.0 && ./configure --prefix=/apps/nginx && make && make install && ln -sv /apps/nginx/sbin/nginx /usr/bin 39.9s
=> [5/7] RUN groupadd -g 2088 nginx && useradd -g nginx -s /usr/sbin/nologin -u 2088 nginx && chown -R nginx.nginx /apps/nginx 1.8s
=> [6/7] ADD nginx.conf /apps/nginx/conf/ 0.4s
=> [7/7] ADD frontend.tar.gz /apps/nginx/html/ 0.5s
=> exporting to docker image format 48.9s
=> => exporting layers 35.8s
=> => exporting manifest sha256:e0d4ee65783ce3c03e7c21eed4867d563be71611685d4eef7de2fb5dc5501daf 0.2s
=> => exporting config sha256:1be61ce233a3d46c7e7d826a4a3747cabb11fbf5c425df18a9cf33fc8314be2a 0.1s
=> => sending tarball 12.8s
Loaded image: K8s-harbor01.mooreyxia.com/baseimages/nginx-base:1.22.0
#上传镜像
[root@K8s-master01 ~]#nerdctl push K8s-harbor01.mooreyxia.com/baseimages/nginx-base:1.22.0
INFO[0000] pushing as a reduced-platform image (application/vnd.docker.distribution.manifest.v2+json, sha256:e0d4ee65783ce3c03e7c21eed4867d563be71611685d4eef7de2fb5dc5501daf)
manifest-sha256:e0d4ee65783ce3c03e7c21eed4867d563be71611685d4eef7de2fb5dc5501daf: done |++++++++++++++++++++++++++++++++++++++|
config-sha256:1be61ce233a3d46c7e7d826a4a3747cabb11fbf5c425df18a9cf33fc8314be2a: done |++++++++++++++++++++++++++++++++++++++|
elapsed: 6.6 s total: 5.5 Ki (858.0 B/s)- 分层构建镜像
#构建基础基础镜像
[root@K8s-master01 k8s-data]#cat dockerfile/system/centos/Dockerfile
#自定义Centos 基础镜像
FROM centos:7.9.2009
MAINTAINER mooreyxia mooreyxia@gmail.com
ADD filebeat-7.12.1-x86_64.rpm /tmp
RUN yum install -y /tmp/filebeat-7.12.1-x86_64.rpm vim wget tree lrzsz gcc gcc-c++ automake pcre pcre-devel zlib zlib-devel openssl openssl-devel iproute net-tools iotop && rm -rf /etc/localtime /tmp/filebeat-7.12.1-x86_64.rpm && ln -snf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && useradd nginx -u 2088
#生成镜像并上传
[root@K8s-master01 centos]#cat build-command.sh
#!/bin/bash
#docker build -t K8s-harbor01.mooreyxia.com/baseimages/magedu-centos-base:7.9.2009 .
#docker push K8s-harbor01.mooreyxia.com/baseimages/magedu-centos-base:7.9.2009
/usr/local/bin/nerdctl build -t K8s-harbor01.mooreyxia.com/baseimages/centos-base:7.9.2009 .
/usr/local/bin/nerdctl push K8s-harbor01.mooreyxia.com/baseimages/centos-base:7.9.2009
[root@K8s-master01 centos]#nerdctl images |grep centos-base
K8s-harbor01.mooreyxia.com/baseimages/centos-base 7.9.2009 eae0cec54c49 2 minutes ago linux/amd64 746.7 MiB 267.2 MiB
#构建中间层镜像
[root@K8s-master01 pub-images]#cd jdk-1.8.212/
[root@K8s-master01 jdk-1.8.212]#ls
Dockerfile build-command.sh jdk-8u212-linux-x64.tar.gz profile
[root@K8s-master01 jdk-1.8.212]#vim Dockerfile
[root@K8s-master01 jdk-1.8.212]#cat Dockerfile
#JDK Base Image
FROM K8s-harbor01.mooreyxia.com/baseimages/centos-base:7.9.2009
#FROM centos:7.9.2009
MAINTAINER mooreyxia mooreyxia@gmail.com
ADD jdk-8u212-linux-x64.tar.gz /usr/local/src/
RUN ln -sv /usr/local/src/jdk1.8.0_212 /usr/local/jdk
ADD profile /etc/profile
ENV JAVA_HOME /usr/local/jdk
ENV JRE_HOME $JAVA_HOME/jre
ENV CLASSPATH $JAVA_HOME/lib/:$JRE_HOME/lib/
ENV PATH $PATH:$JAVA_HOME/bin
[root@K8s-master01 jdk-1.8.212]#cat build-command.sh
/bin/bash
#docker build -t K8s-harbor01.mooreyxia.com/pub-images/jdk-base:v8.212 .
#sleep 1
#docker push K8s-harbor01.mooreyxia.com/pub-images/jdk-base:v8.212
nerdctl build -t K8s-harbor01.mooreyxia.com/pub-images/jdk-base:v8.212 .
nerdctl push K8s-harbor01.mooreyxia.com/pub-images/jdk-base:v8.212
#尝试利用harbor中的镜像进行二次构建会报错
[root@K8s-master01 jdk-1.8.212]#nerdctl build -t K8s-harbor01.mooreyxia.com/pub-images/jdk-base:v8.212 .
[+] Building 0.4s (3/3) FINISHED
=> [internal] load build definition from Dockerfile 0.1s
=> => transferring dockerfile: 447B 0.0s
=> [internal] load .dockerignore 0.1s
=> => transferring context: 2B 0.0s
=> ERROR [internal] load metadata for K8s-harbor01.mooreyxia.com/baseimages/centos-base:7.9.2009 0.1s
------
> [internal] load metadata for K8s-harbor01.mooreyxia.com/baseimages/centos-base:7.9.2009:
------
Dockerfile:2
--------------------
1 | #JDK Base Image
2 | >>> FROM K8s-harbor01.mooreyxia.com/baseimages/centos-base:7.9.2009
3 | #FROM centos:7.9.2009
4 |
--------------------
error: failed to solve: K8s-harbor01.mooreyxia.com/baseimages/centos-base:7.9.2009: failed to do request: Head "https://K8s-harbor01.mooreyxia.com/v2/baseimages/centos-base/manifests/7.9.2009": x509: certificate signed by unknown authority
FATA[0000] no image was built
#原因 - 证书对containerd起作用，镜像构建build时没有携带证书信息
[root@K8s-master01 jdk-1.8.212]#ls /etc/containerd/certs.d/K8s-harbor01.mooreyxia.com/
K8s-harbor01.mooreyxia.com.cert K8s-harbor01.mooreyxia.com.key ca.crt
#解决方案 - 基于nginx代理harbor并实现https- 基于nginx代理harbor并实现https
#停止harbor服务
[root@K8s-harbor01 certs]#cd /apps/harbor/
[root@K8s-harbor01 harbor]#ls
LICENSE common common.sh docker-compose.yml harbor.v2.7.1.tar.gz harbor.yml harbor.yml.tmpl install.sh prepare
[root@K8s-harbor01 harbor]#docker-compose stop
Stopping nginx ... done
Stopping harbor-jobservice ... done
Stopping harbor-core ... done
Stopping registry ... done
Stopping harbor-db ... done
Stopping harbor-portal ... done
Stopping redis ... done
Stopping registryctl ... done
Stopping harbor-log ... done
#取消https端口改为http
[root@K8s-harbor01 harbor]#cat harbor.yml
...
# https related config
#https:
# https port for harbor, default is 443
# port: 443
# The path of cert and key files for nginx
# certificate: /data/harbor/certs/K8s-harbor01.mooreyxia.com.crt
# private_key: /data/harbor/certs/K8s-harbor01.mooreyxia.com.key
...
#更新harbor配置，会自动生成默认配置文件，包含nginx配置文件
[root@K8s-harbor01 harbor]#./prepare
prepare base dir is set to /apps/harbor
WARNING:root:WARNING: HTTP protocol is insecure. Harbor will deprecate http protocol in the future. Please make sure to upgrade to https
Clearing the configuration file: /config/core/env
Clearing the configuration file: /config/core/app.conf
Clearing the configuration file: /config/nginx/nginx.conf
Clearing the configuration file: /config/jobservice/env
Clearing the configuration file: /config/jobservice/config.yml
Clearing the configuration file: /config/log/logrotate.conf
Clearing the configuration file: /config/log/rsyslog_docker.conf
Clearing the configuration file: /config/registryctl/env
Clearing the configuration file: /config/registryctl/config.yml
Clearing the configuration file: /config/portal/nginx.conf
Clearing the configuration file: /config/db/env
Clearing the configuration file: /config/registry/root.crt
Clearing the configuration file: /config/registry/passwd
Clearing the configuration file: /config/registry/config.yml
Generated configuration file: /config/portal/nginx.conf
Generated configuration file: /config/log/logrotate.conf
Generated configuration file: /config/log/rsyslog_docker.conf
Generated configuration file: /config/nginx/nginx.conf
Generated configuration file: /config/core/env
Generated configuration file: /config/core/app.conf
Generated configuration file: /config/registry/config.yml
Generated configuration file: /config/registryctl/env
Generated configuration file: /config/registryctl/config.yml
Generated configuration file: /config/db/env
Generated configuration file: /config/jobservice/env
Generated configuration file: /config/jobservice/config.yml
loaded secret from file: /data/secret/keys/secretkey
Generated configuration file: /compose_location/docker-compose.yml
Clean up the input dir
[root@K8s-harbor01 harbor]#ls common/config/portal/nginx.conf
common/config/portal/nginx.conf
#重启harbor
[root@K8s-harbor01 harbor]#docker-compose up -d
Creating network "harbor_harbor" with the default driver
Creating harbor-log ... done
Creating registry ... done
Creating harbor-portal ... done
Creating harbor-db ... done
Creating redis ... done
Creating registryctl ... done
Creating harbor-core ... done
Creating nginx ... done
Creating harbor-jobservice ... done
#测试http访问harbor
http://k8s-harbor01.mooreyxia.com/
#由于之前的节点访问harbor都是基于https认证，所以需要重新建立访问
#采用Nginx反向代理的方式最简便，证书不变
#nginx实现反向代理 - 前面已经实现，此处省略- 更改buildkitd配置⽂件
#build不信任自签发证书就指定http模式访问
[root@K8s-master01 ~]#mkdir -p /etc/buildkit/
[root@K8s-master01 ~]#vim /etc/buildkit/buildkitd.toml
[root@K8s-master01 ~]#cat /etc/buildkit/buildkitd.toml
[registry."K8s-harbor01.mooreyxia.com"]
http = true
insecure = true- 更改nerdctl配置⽂件：
[root@K8s-master01 ~]#cat /etc/nerdctl/nerdctl.toml
namespace = "k8s.io"
debug = false
debug_full = false
insecure_registry = true
#重启
[root@K8s-master01 ~]#systemctl restart buildkit
#尝试执行镜像构建 - 此时镜像的下载采用仓库http80端口，镜像的上传采用https
[root@K8s-master01 jdk-1.8.212]#nerdctl build -t k8s-nginx.mooreyxia.com/pub-images/jdk-base:v8.212 .
[+] Building 89.3s (10/10)
[+] Building 89.7s (10/10) FINISHED
=> [internal] load build definition from Dockerfile 0.5s
=> => transferring dockerfile: 447B 0.0s
=> [internal] load .dockerignore 0.5s
=> => transferring context: 2B 0.0s
=> [internal] load metadata for K8s-harbor01.mooreyxia.com/baseimages/centos-base:7.9.2009 0.4s
=> [auth] baseimages/centos-base:pull token for K8s-harbor01.mooreyxia.com 0.0s
=> [internal] load build context 5.9s
=> => transferring context: 195.05MB 4.2s
=> [1/4] FROM K8s-harbor01.mooreyxia.com/baseimages/centos-base:7.9.2009@sha256:eae0cec54c4942b50c38d000f7f20ceee592af11896b09bfeae076017630b832 0.2s
=> => resolve K8s-harbor01.mooreyxia.com/baseimages/centos-base:7.9.2009@sha256:eae0cec54c4942b50c38d000f7f20ceee592af11896b09bfeae076017630b832 0.1s
=> [2/4] ADD jdk-8u212-linux-x64.tar.gz /usr/local/src/ 15.0s
=> [3/4] RUN ln -sv /usr/local/src/jdk1.8.0_212 /usr/local/jdk 1.3s
=> [4/4] ADD profile /etc/profile 0.4s
=> exporting to docker image format 63.7s
=> => exporting layers 32.5s
=> => exporting manifest sha256:f0abc92cff14eae87bef62805cada35428b7c82d9718303aed91c746a4a47272 0.1s
=> => exporting config sha256:5c94a249a03f2a564bf979ed8aad16aa91a52e8ff0b2b6dae408f3a8c57e116d 0.1s
=> => sending tarball 31.0s
Loaded image: k8s-nginx.mooreyxia.com/pub-images/jdk-base:v8.212
#上传
[root@K8s-master01 jdk-1.8.212]#nerdctl push K8s-harbor01.mooreyxia.com/pub-images/jdk-base:v8.212
#注意若上传遇到401 Unauthorized报错，是harboe服务器未采用新方式访问服务，需要重新生成harbor服务
[root@K8s-harbor01 harbor]#docker-compose down -v
[root@K8s-harbor01 harbor]#./prepare
[root@K8s-harbor01 harbor]#docker-compose up -d
#如果还为解决，可重新生成harbor证书并分发 重启harbor服务即可我是moore,大家一起加油！！！
