1.构建部署kubernetes所需主机harbor172.25.254.200cpu1,内存2Gharbor仓库master172.25.254.100cpu2,内存4Gmaseter,k8s集群控制节点node1172.25.254.10cpu2,内存2Gworker,k8s集群工作节点node2172.25.254.20cpu2,内存2Gworker,k8s集群工作节点2.构建harbor镜像仓库部署docker本地仓库[rootharbor ~]# dnf remove podman -y 正在更新 Subscription Management 软件仓库。 无法读取客户身份 [rootharbor ~]# dnf install httpd -y 正在更新 Subscription Management 软件仓库。 无法读取客户身份 [rootharbor harbor]# vim /etc/httpd/conf/httpd.conf Listen 1111 [rootharbor docker]# systemctl enable --now httpd [rootharbor ~]# mkdir /var/www/html/docker -p复制这个下载地址[rootharbor ~]# cd /etc/yum.repos.d/ [rootharbor yum.repos.d]# vim docker.repo [docker] name docker baseurl https://mirrors.aliyun.com/docker-ce/linux/rhel/9.8/x86_64/stable/ gpgcheck 0 [rootharbor ~]# dnf install docker-ce --downloadonly --destdir /mnt/ 正在更新 Subscription Management 软件仓库。 无法读取客户身份 [rootharbor ~]# mv /mnt/*.rpm /var/www/html/docker/ [rootharbor ~]# cd /var/www/html/docker/ [rootharbor docker]# dnf install createrepo [rootharbor docker]# createrepo -v /var/www/html/docker/ [rootharbor docker]# vim /etc/yum.repos.d/docker.repo [docker] name docker baseurl http://172.25.254.200:1111/docker/ gpgcheck 0 [rootharbor ~]# dnf install docker-ce -y[rootharbor ~]# echo br_netfilter /etc/modules-load.d/docker_mod.conf [rootharbor ~]# modprobe -a br_netfilter [rootharbor ~]# vim /etc/sysctl.d/docker.conf net.bridge.bridge-nf-call-iptables 1 net.bridge.bridge-nf-call-ip6tables 1 net.ipv4.ip_forward 1 [rootharbor ~]# systemctl restart systemd-modules-load.service [rootharbor ~]# sysctl --system [rootharbor ~]# vim /lib/systemd/system/docker.service ExecStart/usr/bin/dockerd -H fd:// --containerd/run/containerd/containerd.sock --iptablestrue #加入 --iptablestrue [rootharbor ~]# systemctl daemon-reload [rootharbor ~]# systemctl enable docker.service生成仓库密钥key[rootharbor ~]# mkdir /data/certs -p [rootharbor ~]# openssl req -newkey rsa:4096 \ -nodes -sha256 -keyout /data/certs/xiao.org.key \ -addext subjectAltName DNS:reg.xiao.org \ -x509 -days 365 -out /data/certs/xiao.org.crt Country Name (2 letter code) [XX]:cn State or Province Name (full name) []:cq Locality Name (eg, city) [Default City]:ps Organization Name (eg, company) [Default Company Ltd]:k8s Organizational Unit Name (eg, section) []:harbor Common Name (eg, your name or your servers hostname) []:reg.xiao.org Email Address []:编辑harbor配置文件[rootharbor ~]# ls anaconda-ks.cfg harbor-offline-installer-v2.5.4.tgz [rootharbor ~]# tar zxf harbor-offline-installer-v2.5.4.tgz -C /opt/ [rootharbor ~]# cd /opt/harbor/ [rootharbor harbor]# ls common.sh harbor.v2.5.4.tar.gz harbor.yml.tmpl install.sh LICENSE prepare [rootharbor harbor]# cp harbor.yml.tmpl harbor.yml [rootharbor harbor]# vim harbor.yml hostname: reg.xiao.org certificate: /data/certs/xiao.org.crt private_key: /data/certs/xiao.org.key harbor_admin_password: 123 [rootharbor harbor]# ./install.sh --with-chartmuseum#编写启动脚本 [rootharbor ~]# vim /lib/systemd/system/harbor.service [Unit] Descriptionharbor with Docker Compose Documentationhttps://reg.timinglee.com/compose/ Afterdocker.service network-online.target Requiresdocker.service [Service] Typeoneshot WorkingDirectory/opt/harbor ExecStart/usr/bin/docker compose up -d ExecStop/usr/bin/docker compose down RemainAfterExityes [Install] WantedBymulti-user.target [rootharbor ~]# systemctl daemon-reload[rootharbor ~]# systemctl stop harbor.service [rootharbor harbor]# docker compose down #当关闭harbor时会自动删除容器开启时又自动启动容器 [rootharbor ~]# systemctl restart harbor.service [rootharbor harbor]# docker compose ps启动并验证[rootharbor harbor]# mkdir /etc/docker/certs.d/reg.xiao.org/ -p [rootharbor harbor]# cp /data/certs/xiao.org.crt /etc/docker/certs.d/reg.xiao. org/ca.crt [rootharbor harbor]# vim /etc/hosts [rootharbor harbor]# systemctl restart docker [rootharbor harbor]# docker compose up -d [rootharbor harbor]# docker login reg.xiao.org -u admin Password: WARNING! Your credentials are stored unencrypted in /root/.docker/config.json. Configure a credential helper to remove this warning. See https://docs.docker.com/go/credential-store/ Login Succeeded#将swap注释掉然后重启生效[rootmaster ~]# vim /etc/hosts 172.25.254.100 master 172.25.254.10 node1 172.25.254.20 node2 172.25.254.200 reg.xiao.org [rootmaster ~]# for i in 10 20 do scp /etc/hosts root172.25.254.$i:/etc/hosts done [rootmaster ~]# vim /etc/fstab #/dev/mapper/rhel-swap none swap defaults 0 0 #注释 swap [rootmaster ~]# reboot #重启3.利用Ansible部署kubernetes运行环境[rootharbor ~]# vim /etc/yum.repos.d/docker.repo [rootharbor ~]# cd /etc/yum.repos.d/ [rootharbor yum.repos.d]# ls aliyun.repo redhat.repo rhel9.repo [rootharbor yum.repos.d]# vim epel.repo [epel] name epel baseurl https://mirrors.aliyun.com/epel/9/Everything/x86_64/ gpgcheck 0 [rootharbor yum.repos.d]# dnf install ansible-core.x86_64 ansible.noarch -y 正在更新 Subscription Management 软件仓库。 无法读取客户身份配置所需用户[rootharbor yum.repos.d]# useradd devops [rootharbor yum.repos.d]# echo 123 | passwd --stdin devops 更改用户 devops 的密码 。 passwd所有的身份验证令牌已经成功更新。 [rootharbor yum.repos.d]# su - devops [devopsharbor ~]$ mkdir ansible [devopsharbor ~]$ cd ansible/生成ansible的临时配置文件并建立运行环境[devopsharbor ansible]$ vim inventory [servers] 172.25.254.100 172.25.254.10 172.25.254.20 #编辑主机清单文件 inventory用来定义 Ansible 要管理哪些主机。 [devopsharbor ansible]$ vim ansible.cfg [defaults] inventory /home/devops/ansible/inventory # 指定主机清单文件路径 remote_user root # 连接远端机器使用root用户 ask_pass false # 不交互式询问ssh密码使用密钥登录 host_key_checking false # 关闭主机指纹检查避免第一次连接确认yes/no卡住 [devopsharbor ansible]$ ansible all -m ping 172.25.254.20 | SUCCESS { ansible_facts: { discovered_interpreter_python: /usr/bin/python3 }, changed: false, ping: pong } 172.25.254.10 | SUCCESS { ansible_facts: { discovered_interpreter_python: /usr/bin/python3 }, changed: false, ping: pong } 172.25.254.100 | SUCCESS { ansible_facts: { discovered_interpreter_python: /usr/bin/python3 }, changed: false, ping: pong } #- ansible all对清单中所有主机执行任务 #- -m ping调用ping模块ansible 最基础连通性测试 #全部返回 pong代表环境完全通了#1 创建devops用户 ansible all -m user -a namedevops statepresent #2 设置密码123RHEL9兼容 ansible all -m shell -a echo devops:123 | chpasswd #3 安全配置sudo免密放到sudoers.d目录 ansible all -m lineinfile -a path/etc/sudoers.d/devops linedevops ALL(ALL) NOPASSWD: ALL createyes 生成ansible的主配置文件并测试[devopsharbor ansible]$ vim ansible.cfg [defaults] inventory /home/devops/ansible/inventory remote_user devops # SSH登录远端机器先用 devops 用户登录 ask_pass false # 不询问ssh登录密码依赖ssh密钥 host_key_checking false # 关闭主机指纹确认 [privilege_escalation] # ✅权限提升提权配置段 become true # 开启提权相当于命令行加 -b become_method sudo # 提权方式使用 sudo become_user root # 提权到 root 用户 become_ask_pass false # sudo不需要输入密码依赖NOPASSWD sudo权限 [devopsharbor ansible]$ ansible all -m shell -a whoami 172.25.254.100 | CHANGED | rc0 root 172.25.254.10 | CHANGED | rc0 root 172.25.254.20 | CHANGED | rc0 root利用ansible剧本部署docker[devopsharbor ansible]$ vim docker-devops.yml --- - name: devops k8s from ansible hosts: all tasks: - name: setup docker repo yum_repository: name: docker description: docker baseurl: http://172.25.254.200:1111/docker file: docker gpgcheck: no - name: install docker dnf: name: docker-ce state: present - name: setup docker.service replace: path: /lib/systemd/system/docker.service regexp: ExecStart/usr/bin/dockerd -H fd:// --containerd/run/containerd/containerd.sock replace: ExecStart/usr/bin/dockerd -H fd:// --containerd/run/containerd/containerd.sock --iptablestrue - name: setup docker registry copy: dest: /etc/docker/daemon.json content: {registry-mirrors: [https://reg.xiao.org]} - name: setup docker certs file: path: /etc/docker/certs.d/reg.timinglee.org state: directory - name: cp certs file copy: src: {{ item.src }} dest: {{ item.dest }} loop: - { src: /etc/docker/certs.d/reg.xiao.org/ca.crt, dest: /etc/docker/certs.d/reg.xiao.org/ca.crt } - { src: /etc/docker/certs.d/reg.xiao.org/ca.crt, dest: /etc/pki/ca-trust/source/anchors/ca.crt } - name: load module br_netfilter copy: dest: /etc/modules-load.d/docker_mod.conf content: br_netfilter - name: load module sysctl copy: dest: /etc/sysctl.d/docker.conf content: | net.bridge.bridge-nf-call-iptables1 net.bridge.bridge-nf-call-ip6tables1 net.ipv4.ip_forward1 - name: update ca cert shell: update-ca-trust extract sysctl --system - name: start services service: name: {{ item }} state: restarted enabled: yes loop: - docker - systemd-modules-load.service [devopsharbor ansible]$ ansible-playbook docker-devops.yml利用ansible安装cri-dockerd[devopsharbor ansible]$ ls ansible.cfg cri-dockerd-0.4.4.amd64.tgz docker-devops.yml inventory [devopsharbor ansible]$ tar zxf cri-dockerd-0.4.4.amd64.tgz [devopsharbor ansible]$ ls ansible.cfg cri-dockerd-0.4.4.amd64.tgz inventory cri-dockerd docker-devops.yml [devopsharbor ansible]$ cd cri-dockerd/ [devopsharbor cri-dockerd]$ ls cri-dockerd cri-docker.service cri-docker.socket [devopsharbor cri-dockerd]$ vim cri-docker.service ExecStart/usr/bin/cri-dockerd --container-runtime-endpoint fd:// --network-plugincni --pod-infra-container-imagereg.xiao.org/k8s/pause:3.10.1 #加入后面这一段[devopsharbor ansible]$ vim kubernetes-devops.yml 1 --- 2 - name: devops k8s from ansible 3 hosts: all 4 tasks: 5 - name: setup docker repo 6 yum_repository: 7 name: docker 8 description: docker 9 baseurl: http://172.25.254.200:1111/docker 10 file: docker 11 gpgcheck: no 12 13 - name: install docker 14 dnf: 15 name: docker-ce 16 state: present 17 18 - name: setup docker.service 19 replace: 20 path: /lib/systemd/system/docker.service 21 regexp: ExecStart/usr/bin/dockerd -H fd:// --containerd/run/containerd/containerd.sock 22 replace: ExecStart/usr/bin/dockerd -H fd:// --containerd/run/containerd/containerd.sock --iptablestrue 23 24 - name: setup docker registry 25 copy: 26 dest: /etc/docker/daemon.json 27 content: {registry-mirrors: [https://reg.xiao.org]} 28 29 - name: setup docker certs 30 file: 31 path: /etc/docker/certs.d/reg.xiao.org 32 state: directory 33 34 - name: cp certs file 35 copy: 36 src: {{ item.src }} 37 dest: {{ item.dest }} 38 loop: 39 - { src: /etc/docker/certs.d/reg.xiao.org/ca.crt, dest: /etc/docker/certs.d/reg.xiao.org/ca.crt } 40 - { src: /etc/docker/certs.d/reg.xiao.org/ca.crt, dest: /etc/pki/ca-trust/source/anchors/ca.crt } 41 42 - name: load module br_netfilter 43 copy: 44 dest: /etc/modules-load.d/docker_mod.conf 45 content: br_netfilter 46 47 - name: load module sysctl 48 copy: 49 dest: /etc/sysctl.d/docker.conf 50 content: | 51 net.bridge.bridge-nf-call-iptables1 52 net.bridge.bridge-nf-call-ip6tables1 53 net.ipv4.ip_forward1 54 55 - name: update ca cert 56 shell: update-ca-trust extract sysctl --system 57 58 - name: start services 59 service: 60 name: {{ item }} 61 state: restarted 62 enabled: yes 63 loop: 64 - docker 65 - systemd-modules-load.service 66 67 - name: setup cri-docker 68 copy: 69 src: {{item.src}} 70 dest: {{item.dest}} 71 mode: 0755 72 loop: 73 - src: ./cri-dockerd/cri-dockerd 74 dest: /usr/bin/cri-dockerd 75 - src: ./cri-dockerd/cri-docker.service 76 dest: /lib/systemd/system/cri-docker.service 77 - src: ./cri-dockerd/cri-docker.socket 78 dest: /lib/systemd/system/cri-docker.socket 79 80 - name: update ca cert 81 shell: update-ca-trust extract;sysctl --system;systemctl daemon-reload 82 83 - name: start services 84 service: 85 name: {{item}} 86 state: restarted 87 enabled: yes 88 loop: 89 - docker 90 - systemd-modules-load.service 91 - cri-docker [devopsharbor ansible]$ ansible-playbook docker-devops.yml用ansible部署k8s软件[rootharhor ~]# cd /etc/yum.repos.d/ [rootharhor yum.repos.d]# ls aliyun.repo docker.repo epel.repo redhat.repo rhel9.repo [rootharhor yum.repos.d]# vim kubernetes.repo [k8s] name k8s baseurl https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.35/rpm/ gpgcheck 0 [rootharhor yum.repos.d]# dnf list kubeadm --showduplicates kubeadm.x86_64 1.35.7-150500.1.1 k8s [rootharhor yum.repos.d]# dnf list kubeadm-1.35.7-150500.1.1 --showduplicates 正在更新 Subscription Management 软件仓库。 [rootharhor yum.repos.d]# dnf list kubeadm-1.35.7-150500.1.1 kubelet-1.35.7-150500.1.1 kubectl-1.35.7-150500.1.1 --showduplicates 正在更新 Subscription Management 软件仓库。[devopsharbor ansible]$ vim kubernetes-devops.yml - name: install kubernetes from ansible hosts: all tasks: - name: setup k8s repo yum_repository: name: kubernetes description: kubernetes baseurl: https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.35/rpm file: kubernetes gpgcheck: no - name: install software in master dnf: name: {{item}} state: present loop: - kubeadm-1.35.7-150500.1.1 - kubelet-1.35.7-150500.1.1 - kubectl-1.35.7-150500.1.1 when: inventory_hostname 172.25.254.100 - name: install software in work node dnf: name: {{item}} state: present loop: - kubeadm-1.35.7-150500.1.1 - kubelet-1.35.7-150500.1.1 when: inventory_hostname 172.25.254.10 or inventory_hostname 172.25.254.20 - name: setup kubectl kubeadm complication lineinfile: path: /root/.bashrc line: | source (kubectl completion bash) source (kubeadm completion bash) when: inventory_hostname 172.25.254.100 - name: start kubelet service: name: kubelet state: started enabled: yes#让补齐功能生效 [rootmaster mnt]# source ~/.bashrc [rootk8s-master mnt]# systemctl status kubelet.service #注意此时kubelet 不能启动是正常的4.下载kubernetes集群所需镜像查看镜像 [rootmaster ~]# kubeadm config images list I0820 23:18:27.955971 66565 version.go:260] remote version is much newer: v1.36.4; falling back to: stable-1.35 registry.k8s.io/kube-apiserver:v1.35.7 registry.k8s.io/kube-controller-manager:v1.35.7 registry.k8s.io/kube-scheduler:v1.35.7 registry.k8s.io/kube-proxy:v1.35.7 registry.k8s.io/coredns/coredns:v1.13.1 registry.k8s.io/pause:3.10.1 registry.k8s.io/etcd:3.6.6-0 下载镜像 [rootmaster ~]# kubeadm config images pull \ --image-repository registry.aliyuncs.com/google_containers \ --kubernetes-version v1.35.7 \ --cri-socketunix:///var/run/cri-dockerd.sock上传镜像到本地harbor#登录认证 [rootmaster ~]# docker login reg.timinglee.org -u admin Password: WARNING! Your credentials are stored unencrypted in /root/.docker/config.json. Configure a credential helper to remove this warning. See https://docs.docker.com/go/credential-store/ Login Succeeded #上传镜像 注意你的harbor里面必须有k8s才会成功 [rootmaster ~]# docker images --format {{.Repository}}:{{.Tag}} | awk -F / /google/{system(docker tag $0 reg.timinglee.org/k8s/$3)} [rootmaster ~]# docker images --format {{.Repository}}:{{.Tag}} | awk -F / /timinglee/{system(docker push $0)}上传成功5.在master中初始化kubernetes集群在master中完成集群初始化[rootmaster ~]# kubeadm init --pod-network-cidr10.244.0.0/16 \ --image-repository reg.xiao.org/k8s \ --kubernetes-version v1.35.7 \ --cri-socketunix:///var/run/cri-dockerd.sock Then you can join any number of worker nodes by running the following on each as root: kubeadm join 172.25.254.100:6443 --token nc59pb.vcn5hbqak2rphr7r \ --discovery-token-ca-cert-hash sha256:97be669e0881a04f8b3970fd0c504ee2b4 0932d6ebea1a2e49f06d796cd32262 #注意上面这一长串是自动生成的每个人都不一样。添加kubernets环境变量到本机[rootmaster ~]# echo export KUBECONFIG/etc/kubernetes/admin.conf ~/.bash_profile [rootmaster ~]# source ~/.bash_profile [rootmaster ~]# kubectl get nodes NAME STATUS ROLES AGE VERSION master NotReady control-plane 2m58s v1.35.7添加node节点到本集群[rootnode2 ~]# kubeadm join 172.25.254.100:6443 --token nc59pb.vcn5hbqak2rphr7r --discovery-token-ca-cert-hash sha256:97be669e0881a04f8b3970fd0c504ee2b40932d6ebea1a2e49f06d796cd32262 --cri-socketunix:///var/run/cri-dockerd.sock #在两台node主机上加入上面的命令使他们加入master 集群 #加入成功后在master主机上可以看到他们已经加入集群 [rootmaster ~]# kubectl get nodes NAME STATUS ROLES AGE VERSION master NotReady control-plane 10m v1.35.7 node1 NotReady none 43s v1.35.7 node2 NotReady none 5s v1.35.76.安装网络插件[rootmaster ~]# ls anaconda-ks.cfg kube-flannel-v0.28.9.tar.gz [rootmaster ~]# tar zxf kube-flannel-v0.28.9.tar.gz [rootmaster ~]# ls anaconda-ks.cfg flannel kube-flannel-v0.28.9.tar.gz [rootmaster ~]# cd flannel/ [rootmaster flannel]# ls kube-flannel-v0.28.9.tar kube-flannel.yml #导入镜像 [rootmaster flannel]# docker load -i kube-flannel-v0.28.9.tar Loaded image: ghcr.io/flannel-io/flannel-cni-plugin:v1.9.1-flannel3 Loaded image: ghcr.io/flannel-io/flannel:v0.28.9 #上传镜像 [rootmaster flannel]# docker tag ghcr.io/flannel-io/flannel-cni-plugin:v1.9.1-f lannel3 reg.xiao.org/flannel-io/flannel-cni-plugin:v1.9.0-flannel3 [rootmaster flannel]# docker push reg.xiao.org/flannel-io/flannel-cni-plugin :v1.9.0-flannel3 [rootmaster flannel]# docker tag ghcr.io/flannel-io/flannel:v0.28.9 reg.xiao.o rg/flannel-io/flannel:v0.28.9 [rootmaster flannel]# docker push reg.xiao.org/flannel-io/flannel:v0.28.9[rootmaster ~]# vim kube-flannel.yml image: flannel-io/flannel:v0.28.1 image: flannel-io/flannel-cni-plugin:v1.9.0-flannel1 image: flannel-io/flannel:v0.28.1 [rootmaster flannel]# grep -n image: kube-flannel.yml 148: image: ghcr.io/flannel-io/flannel:v0.28.9 175: image: ghcr.io/flannel-io/flannel-cni-plugin:v1.9.1-flannel3 186: image: ghcr.io/flannel-io/flannel:v0.28.9 [rootmaster flannel]# ls kube-flannel-v0.28.9.tar kube-flannel.yml [rootmaster flannel]# vim kube-flannel.yml [rootmaster flannel]# kubectl apply -f kube-flannel.yml namespace/kube-flannel created serviceaccount/flannel created clusterrole.rbac.authorization.k8s.io/flannel created clusterrolebinding.rbac.authorization.k8s.io/flannel created configmap/kube-flannel-cfg created daemonset.apps/kube-flannel-ds created [rootmaster flannel]# kubectl get nodes NAME STATUS ROLES AGE VERSION master Ready control-plane 36m v1.35.7 node1 Ready none 26m v1.35.7 node2 Ready none 25m v1.35.7 kubernetes 环境部署
1.构建部署kubernetes所需主机harbor172.25.254.200cpu1,内存2Gharbor仓库master172.25.254.100cpu2,内存4Gmaseter,k8s集群控制节点node1172.25.254.10cpu2,内存2Gworker,k8s集群工作节点node2172.25.254.20cpu2,内存2Gworker,k8s集群工作节点2.构建harbor镜像仓库部署docker本地仓库[rootharbor ~]# dnf remove podman -y 正在更新 Subscription Management 软件仓库。 无法读取客户身份 [rootharbor ~]# dnf install httpd -y 正在更新 Subscription Management 软件仓库。 无法读取客户身份 [rootharbor harbor]# vim /etc/httpd/conf/httpd.conf Listen 1111 [rootharbor docker]# systemctl enable --now httpd [rootharbor ~]# mkdir /var/www/html/docker -p复制这个下载地址[rootharbor ~]# cd /etc/yum.repos.d/ [rootharbor yum.repos.d]# vim docker.repo [docker] name docker baseurl https://mirrors.aliyun.com/docker-ce/linux/rhel/9.8/x86_64/stable/ gpgcheck 0 [rootharbor ~]# dnf install docker-ce --downloadonly --destdir /mnt/ 正在更新 Subscription Management 软件仓库。 无法读取客户身份 [rootharbor ~]# mv /mnt/*.rpm /var/www/html/docker/ [rootharbor ~]# cd /var/www/html/docker/ [rootharbor docker]# dnf install createrepo [rootharbor docker]# createrepo -v /var/www/html/docker/ [rootharbor docker]# vim /etc/yum.repos.d/docker.repo [docker] name docker baseurl http://172.25.254.200:1111/docker/ gpgcheck 0 [rootharbor ~]# dnf install docker-ce -y[rootharbor ~]# echo br_netfilter /etc/modules-load.d/docker_mod.conf [rootharbor ~]# modprobe -a br_netfilter [rootharbor ~]# vim /etc/sysctl.d/docker.conf net.bridge.bridge-nf-call-iptables 1 net.bridge.bridge-nf-call-ip6tables 1 net.ipv4.ip_forward 1 [rootharbor ~]# systemctl restart systemd-modules-load.service [rootharbor ~]# sysctl --system [rootharbor ~]# vim /lib/systemd/system/docker.service ExecStart/usr/bin/dockerd -H fd:// --containerd/run/containerd/containerd.sock --iptablestrue #加入 --iptablestrue [rootharbor ~]# systemctl daemon-reload [rootharbor ~]# systemctl enable docker.service生成仓库密钥key[rootharbor ~]# mkdir /data/certs -p [rootharbor ~]# openssl req -newkey rsa:4096 \ -nodes -sha256 -keyout /data/certs/xiao.org.key \ -addext subjectAltName DNS:reg.xiao.org \ -x509 -days 365 -out /data/certs/xiao.org.crt Country Name (2 letter code) [XX]:cn State or Province Name (full name) []:cq Locality Name (eg, city) [Default City]:ps Organization Name (eg, company) [Default Company Ltd]:k8s Organizational Unit Name (eg, section) []:harbor Common Name (eg, your name or your servers hostname) []:reg.xiao.org Email Address []:编辑harbor配置文件[rootharbor ~]# ls anaconda-ks.cfg harbor-offline-installer-v2.5.4.tgz [rootharbor ~]# tar zxf harbor-offline-installer-v2.5.4.tgz -C /opt/ [rootharbor ~]# cd /opt/harbor/ [rootharbor harbor]# ls common.sh harbor.v2.5.4.tar.gz harbor.yml.tmpl install.sh LICENSE prepare [rootharbor harbor]# cp harbor.yml.tmpl harbor.yml [rootharbor harbor]# vim harbor.yml hostname: reg.xiao.org certificate: /data/certs/xiao.org.crt private_key: /data/certs/xiao.org.key harbor_admin_password: 123 [rootharbor harbor]# ./install.sh --with-chartmuseum#编写启动脚本 [rootharbor ~]# vim /lib/systemd/system/harbor.service [Unit] Descriptionharbor with Docker Compose Documentationhttps://reg.timinglee.com/compose/ Afterdocker.service network-online.target Requiresdocker.service [Service] Typeoneshot WorkingDirectory/opt/harbor ExecStart/usr/bin/docker compose up -d ExecStop/usr/bin/docker compose down RemainAfterExityes [Install] WantedBymulti-user.target [rootharbor ~]# systemctl daemon-reload[rootharbor ~]# systemctl stop harbor.service [rootharbor harbor]# docker compose down #当关闭harbor时会自动删除容器开启时又自动启动容器 [rootharbor ~]# systemctl restart harbor.service [rootharbor harbor]# docker compose ps启动并验证[rootharbor harbor]# mkdir /etc/docker/certs.d/reg.xiao.org/ -p [rootharbor harbor]# cp /data/certs/xiao.org.crt /etc/docker/certs.d/reg.xiao. org/ca.crt [rootharbor harbor]# vim /etc/hosts [rootharbor harbor]# systemctl restart docker [rootharbor harbor]# docker compose up -d [rootharbor harbor]# docker login reg.xiao.org -u admin Password: WARNING! Your credentials are stored unencrypted in /root/.docker/config.json. Configure a credential helper to remove this warning. See https://docs.docker.com/go/credential-store/ Login Succeeded#将swap注释掉然后重启生效[rootmaster ~]# vim /etc/hosts 172.25.254.100 master 172.25.254.10 node1 172.25.254.20 node2 172.25.254.200 reg.xiao.org [rootmaster ~]# for i in 10 20 do scp /etc/hosts root172.25.254.$i:/etc/hosts done [rootmaster ~]# vim /etc/fstab #/dev/mapper/rhel-swap none swap defaults 0 0 #注释 swap [rootmaster ~]# reboot #重启3.利用Ansible部署kubernetes运行环境[rootharbor ~]# vim /etc/yum.repos.d/docker.repo [rootharbor ~]# cd /etc/yum.repos.d/ [rootharbor yum.repos.d]# ls aliyun.repo redhat.repo rhel9.repo [rootharbor yum.repos.d]# vim epel.repo [epel] name epel baseurl https://mirrors.aliyun.com/epel/9/Everything/x86_64/ gpgcheck 0 [rootharbor yum.repos.d]# dnf install ansible-core.x86_64 ansible.noarch -y 正在更新 Subscription Management 软件仓库。 无法读取客户身份配置所需用户[rootharbor yum.repos.d]# useradd devops [rootharbor yum.repos.d]# echo 123 | passwd --stdin devops 更改用户 devops 的密码 。 passwd所有的身份验证令牌已经成功更新。 [rootharbor yum.repos.d]# su - devops [devopsharbor ~]$ mkdir ansible [devopsharbor ~]$ cd ansible/生成ansible的临时配置文件并建立运行环境[devopsharbor ansible]$ vim inventory [servers] 172.25.254.100 172.25.254.10 172.25.254.20 #编辑主机清单文件 inventory用来定义 Ansible 要管理哪些主机。 [devopsharbor ansible]$ vim ansible.cfg [defaults] inventory /home/devops/ansible/inventory # 指定主机清单文件路径 remote_user root # 连接远端机器使用root用户 ask_pass false # 不交互式询问ssh密码使用密钥登录 host_key_checking false # 关闭主机指纹检查避免第一次连接确认yes/no卡住 [devopsharbor ansible]$ ansible all -m ping 172.25.254.20 | SUCCESS { ansible_facts: { discovered_interpreter_python: /usr/bin/python3 }, changed: false, ping: pong } 172.25.254.10 | SUCCESS { ansible_facts: { discovered_interpreter_python: /usr/bin/python3 }, changed: false, ping: pong } 172.25.254.100 | SUCCESS { ansible_facts: { discovered_interpreter_python: /usr/bin/python3 }, changed: false, ping: pong } #- ansible all对清单中所有主机执行任务 #- -m ping调用ping模块ansible 最基础连通性测试 #全部返回 pong代表环境完全通了#1 创建devops用户 ansible all -m user -a namedevops statepresent #2 设置密码123RHEL9兼容 ansible all -m shell -a echo devops:123 | chpasswd #3 安全配置sudo免密放到sudoers.d目录 ansible all -m lineinfile -a path/etc/sudoers.d/devops linedevops ALL(ALL) NOPASSWD: ALL createyes 生成ansible的主配置文件并测试[devopsharbor ansible]$ vim ansible.cfg [defaults] inventory /home/devops/ansible/inventory remote_user devops # SSH登录远端机器先用 devops 用户登录 ask_pass false # 不询问ssh登录密码依赖ssh密钥 host_key_checking false # 关闭主机指纹确认 [privilege_escalation] # ✅权限提升提权配置段 become true # 开启提权相当于命令行加 -b become_method sudo # 提权方式使用 sudo become_user root # 提权到 root 用户 become_ask_pass false # sudo不需要输入密码依赖NOPASSWD sudo权限 [devopsharbor ansible]$ ansible all -m shell -a whoami 172.25.254.100 | CHANGED | rc0 root 172.25.254.10 | CHANGED | rc0 root 172.25.254.20 | CHANGED | rc0 root利用ansible剧本部署docker[devopsharbor ansible]$ vim docker-devops.yml --- - name: devops k8s from ansible hosts: all tasks: - name: setup docker repo yum_repository: name: docker description: docker baseurl: http://172.25.254.200:1111/docker file: docker gpgcheck: no - name: install docker dnf: name: docker-ce state: present - name: setup docker.service replace: path: /lib/systemd/system/docker.service regexp: ExecStart/usr/bin/dockerd -H fd:// --containerd/run/containerd/containerd.sock replace: ExecStart/usr/bin/dockerd -H fd:// --containerd/run/containerd/containerd.sock --iptablestrue - name: setup docker registry copy: dest: /etc/docker/daemon.json content: {registry-mirrors: [https://reg.xiao.org]} - name: setup docker certs file: path: /etc/docker/certs.d/reg.timinglee.org state: directory - name: cp certs file copy: src: {{ item.src }} dest: {{ item.dest }} loop: - { src: /etc/docker/certs.d/reg.xiao.org/ca.crt, dest: /etc/docker/certs.d/reg.xiao.org/ca.crt } - { src: /etc/docker/certs.d/reg.xiao.org/ca.crt, dest: /etc/pki/ca-trust/source/anchors/ca.crt } - name: load module br_netfilter copy: dest: /etc/modules-load.d/docker_mod.conf content: br_netfilter - name: load module sysctl copy: dest: /etc/sysctl.d/docker.conf content: | net.bridge.bridge-nf-call-iptables1 net.bridge.bridge-nf-call-ip6tables1 net.ipv4.ip_forward1 - name: update ca cert shell: update-ca-trust extract sysctl --system - name: start services service: name: {{ item }} state: restarted enabled: yes loop: - docker - systemd-modules-load.service [devopsharbor ansible]$ ansible-playbook docker-devops.yml利用ansible安装cri-dockerd[devopsharbor ansible]$ ls ansible.cfg cri-dockerd-0.4.4.amd64.tgz docker-devops.yml inventory [devopsharbor ansible]$ tar zxf cri-dockerd-0.4.4.amd64.tgz [devopsharbor ansible]$ ls ansible.cfg cri-dockerd-0.4.4.amd64.tgz inventory cri-dockerd docker-devops.yml [devopsharbor ansible]$ cd cri-dockerd/ [devopsharbor cri-dockerd]$ ls cri-dockerd cri-docker.service cri-docker.socket [devopsharbor cri-dockerd]$ vim cri-docker.service ExecStart/usr/bin/cri-dockerd --container-runtime-endpoint fd:// --network-plugincni --pod-infra-container-imagereg.xiao.org/k8s/pause:3.10.1 #加入后面这一段[devopsharbor ansible]$ vim kubernetes-devops.yml 1 --- 2 - name: devops k8s from ansible 3 hosts: all 4 tasks: 5 - name: setup docker repo 6 yum_repository: 7 name: docker 8 description: docker 9 baseurl: http://172.25.254.200:1111/docker 10 file: docker 11 gpgcheck: no 12 13 - name: install docker 14 dnf: 15 name: docker-ce 16 state: present 17 18 - name: setup docker.service 19 replace: 20 path: /lib/systemd/system/docker.service 21 regexp: ExecStart/usr/bin/dockerd -H fd:// --containerd/run/containerd/containerd.sock 22 replace: ExecStart/usr/bin/dockerd -H fd:// --containerd/run/containerd/containerd.sock --iptablestrue 23 24 - name: setup docker registry 25 copy: 26 dest: /etc/docker/daemon.json 27 content: {registry-mirrors: [https://reg.xiao.org]} 28 29 - name: setup docker certs 30 file: 31 path: /etc/docker/certs.d/reg.xiao.org 32 state: directory 33 34 - name: cp certs file 35 copy: 36 src: {{ item.src }} 37 dest: {{ item.dest }} 38 loop: 39 - { src: /etc/docker/certs.d/reg.xiao.org/ca.crt, dest: /etc/docker/certs.d/reg.xiao.org/ca.crt } 40 - { src: /etc/docker/certs.d/reg.xiao.org/ca.crt, dest: /etc/pki/ca-trust/source/anchors/ca.crt } 41 42 - name: load module br_netfilter 43 copy: 44 dest: /etc/modules-load.d/docker_mod.conf 45 content: br_netfilter 46 47 - name: load module sysctl 48 copy: 49 dest: /etc/sysctl.d/docker.conf 50 content: | 51 net.bridge.bridge-nf-call-iptables1 52 net.bridge.bridge-nf-call-ip6tables1 53 net.ipv4.ip_forward1 54 55 - name: update ca cert 56 shell: update-ca-trust extract sysctl --system 57 58 - name: start services 59 service: 60 name: {{ item }} 61 state: restarted 62 enabled: yes 63 loop: 64 - docker 65 - systemd-modules-load.service 66 67 - name: setup cri-docker 68 copy: 69 src: {{item.src}} 70 dest: {{item.dest}} 71 mode: 0755 72 loop: 73 - src: ./cri-dockerd/cri-dockerd 74 dest: /usr/bin/cri-dockerd 75 - src: ./cri-dockerd/cri-docker.service 76 dest: /lib/systemd/system/cri-docker.service 77 - src: ./cri-dockerd/cri-docker.socket 78 dest: /lib/systemd/system/cri-docker.socket 79 80 - name: update ca cert 81 shell: update-ca-trust extract;sysctl --system;systemctl daemon-reload 82 83 - name: start services 84 service: 85 name: {{item}} 86 state: restarted 87 enabled: yes 88 loop: 89 - docker 90 - systemd-modules-load.service 91 - cri-docker [devopsharbor ansible]$ ansible-playbook docker-devops.yml用ansible部署k8s软件[rootharhor ~]# cd /etc/yum.repos.d/ [rootharhor yum.repos.d]# ls aliyun.repo docker.repo epel.repo redhat.repo rhel9.repo [rootharhor yum.repos.d]# vim kubernetes.repo [k8s] name k8s baseurl https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.35/rpm/ gpgcheck 0 [rootharhor yum.repos.d]# dnf list kubeadm --showduplicates kubeadm.x86_64 1.35.7-150500.1.1 k8s [rootharhor yum.repos.d]# dnf list kubeadm-1.35.7-150500.1.1 --showduplicates 正在更新 Subscription Management 软件仓库。 [rootharhor yum.repos.d]# dnf list kubeadm-1.35.7-150500.1.1 kubelet-1.35.7-150500.1.1 kubectl-1.35.7-150500.1.1 --showduplicates 正在更新 Subscription Management 软件仓库。[devopsharbor ansible]$ vim kubernetes-devops.yml - name: install kubernetes from ansible hosts: all tasks: - name: setup k8s repo yum_repository: name: kubernetes description: kubernetes baseurl: https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.35/rpm file: kubernetes gpgcheck: no - name: install software in master dnf: name: {{item}} state: present loop: - kubeadm-1.35.7-150500.1.1 - kubelet-1.35.7-150500.1.1 - kubectl-1.35.7-150500.1.1 when: inventory_hostname 172.25.254.100 - name: install software in work node dnf: name: {{item}} state: present loop: - kubeadm-1.35.7-150500.1.1 - kubelet-1.35.7-150500.1.1 when: inventory_hostname 172.25.254.10 or inventory_hostname 172.25.254.20 - name: setup kubectl kubeadm complication lineinfile: path: /root/.bashrc line: | source (kubectl completion bash) source (kubeadm completion bash) when: inventory_hostname 172.25.254.100 - name: start kubelet service: name: kubelet state: started enabled: yes#让补齐功能生效 [rootmaster mnt]# source ~/.bashrc [rootk8s-master mnt]# systemctl status kubelet.service #注意此时kubelet 不能启动是正常的4.下载kubernetes集群所需镜像查看镜像 [rootmaster ~]# kubeadm config images list I0820 23:18:27.955971 66565 version.go:260] remote version is much newer: v1.36.4; falling back to: stable-1.35 registry.k8s.io/kube-apiserver:v1.35.7 registry.k8s.io/kube-controller-manager:v1.35.7 registry.k8s.io/kube-scheduler:v1.35.7 registry.k8s.io/kube-proxy:v1.35.7 registry.k8s.io/coredns/coredns:v1.13.1 registry.k8s.io/pause:3.10.1 registry.k8s.io/etcd:3.6.6-0 下载镜像 [rootmaster ~]# kubeadm config images pull \ --image-repository registry.aliyuncs.com/google_containers \ --kubernetes-version v1.35.7 \ --cri-socketunix:///var/run/cri-dockerd.sock上传镜像到本地harbor#登录认证 [rootmaster ~]# docker login reg.timinglee.org -u admin Password: WARNING! Your credentials are stored unencrypted in /root/.docker/config.json. Configure a credential helper to remove this warning. See https://docs.docker.com/go/credential-store/ Login Succeeded #上传镜像 注意你的harbor里面必须有k8s才会成功 [rootmaster ~]# docker images --format {{.Repository}}:{{.Tag}} | awk -F / /google/{system(docker tag $0 reg.timinglee.org/k8s/$3)} [rootmaster ~]# docker images --format {{.Repository}}:{{.Tag}} | awk -F / /timinglee/{system(docker push $0)}上传成功5.在master中初始化kubernetes集群在master中完成集群初始化[rootmaster ~]# kubeadm init --pod-network-cidr10.244.0.0/16 \ --image-repository reg.xiao.org/k8s \ --kubernetes-version v1.35.7 \ --cri-socketunix:///var/run/cri-dockerd.sock Then you can join any number of worker nodes by running the following on each as root: kubeadm join 172.25.254.100:6443 --token nc59pb.vcn5hbqak2rphr7r \ --discovery-token-ca-cert-hash sha256:97be669e0881a04f8b3970fd0c504ee2b4 0932d6ebea1a2e49f06d796cd32262 #注意上面这一长串是自动生成的每个人都不一样。添加kubernets环境变量到本机[rootmaster ~]# echo export KUBECONFIG/etc/kubernetes/admin.conf ~/.bash_profile [rootmaster ~]# source ~/.bash_profile [rootmaster ~]# kubectl get nodes NAME STATUS ROLES AGE VERSION master NotReady control-plane 2m58s v1.35.7添加node节点到本集群[rootnode2 ~]# kubeadm join 172.25.254.100:6443 --token nc59pb.vcn5hbqak2rphr7r --discovery-token-ca-cert-hash sha256:97be669e0881a04f8b3970fd0c504ee2b40932d6ebea1a2e49f06d796cd32262 --cri-socketunix:///var/run/cri-dockerd.sock #在两台node主机上加入上面的命令使他们加入master 集群 #加入成功后在master主机上可以看到他们已经加入集群 [rootmaster ~]# kubectl get nodes NAME STATUS ROLES AGE VERSION master NotReady control-plane 10m v1.35.7 node1 NotReady none 43s v1.35.7 node2 NotReady none 5s v1.35.76.安装网络插件[rootmaster ~]# ls anaconda-ks.cfg kube-flannel-v0.28.9.tar.gz [rootmaster ~]# tar zxf kube-flannel-v0.28.9.tar.gz [rootmaster ~]# ls anaconda-ks.cfg flannel kube-flannel-v0.28.9.tar.gz [rootmaster ~]# cd flannel/ [rootmaster flannel]# ls kube-flannel-v0.28.9.tar kube-flannel.yml #导入镜像 [rootmaster flannel]# docker load -i kube-flannel-v0.28.9.tar Loaded image: ghcr.io/flannel-io/flannel-cni-plugin:v1.9.1-flannel3 Loaded image: ghcr.io/flannel-io/flannel:v0.28.9 #上传镜像 [rootmaster flannel]# docker tag ghcr.io/flannel-io/flannel-cni-plugin:v1.9.1-f lannel3 reg.xiao.org/flannel-io/flannel-cni-plugin:v1.9.0-flannel3 [rootmaster flannel]# docker push reg.xiao.org/flannel-io/flannel-cni-plugin :v1.9.0-flannel3 [rootmaster flannel]# docker tag ghcr.io/flannel-io/flannel:v0.28.9 reg.xiao.o rg/flannel-io/flannel:v0.28.9 [rootmaster flannel]# docker push reg.xiao.org/flannel-io/flannel:v0.28.9[rootmaster ~]# vim kube-flannel.yml image: flannel-io/flannel:v0.28.1 image: flannel-io/flannel-cni-plugin:v1.9.0-flannel1 image: flannel-io/flannel:v0.28.1 [rootmaster flannel]# grep -n image: kube-flannel.yml 148: image: ghcr.io/flannel-io/flannel:v0.28.9 175: image: ghcr.io/flannel-io/flannel-cni-plugin:v1.9.1-flannel3 186: image: ghcr.io/flannel-io/flannel:v0.28.9 [rootmaster flannel]# ls kube-flannel-v0.28.9.tar kube-flannel.yml [rootmaster flannel]# vim kube-flannel.yml [rootmaster flannel]# kubectl apply -f kube-flannel.yml namespace/kube-flannel created serviceaccount/flannel created clusterrole.rbac.authorization.k8s.io/flannel created clusterrolebinding.rbac.authorization.k8s.io/flannel created configmap/kube-flannel-cfg created daemonset.apps/kube-flannel-ds created [rootmaster flannel]# kubectl get nodes NAME STATUS ROLES AGE VERSION master Ready control-plane 36m v1.35.7 node1 Ready none 26m v1.35.7 node2 Ready none 25m v1.35.7