1.4 开始按步骤部署
通过 ansible 脚本初始化环境及部署 k8s 高可用集群。
可以使用ezctl命令一键安装或分布安装
[root@k8s-master1 /etc/kubeasz]# ezctl help
Usage: ezctl COMMAND [args]
-------------------------------------------------------------------------------------
Cluster setups:
list to list all of the managed clusters
checkout <cluster> to switch default kubeconfig of the cluster
new <cluster> to start a new k8s deploy with name 'cluster'
setup <cluster> <step> to setup a cluster, also supporting a step-by-step way
start <cluster> to start all of the k8s services stopped by 'ezctl stop'
stop <cluster> to stop all of the k8s services temporarily
upgrade <cluster> to upgrade the k8s cluster
destroy <cluster> to destroy the k8s cluster
backup <cluster> to backup the cluster state (etcd snapshot)
restore <cluster> to restore the cluster state from backups
start-aio to quickly setup an all-in-one cluster with 'default' settings
Cluster ops:
add-etcd <cluster> <ip> to add a etcd-node to the etcd cluster
add-master <cluster> <ip> to add a master node to the k8s cluster
add-node <cluster> <ip> to add a work node to the k8s cluster
del-etcd <cluster> <ip> to delete a etcd-node from the etcd cluster
del-master <cluster> <ip> to delete a master node from the k8s cluster
del-node <cluster> <ip> to delete a work node from the k8s cluster
Extra operation:
kcfg-adm <cluster> <args> to manage client kubeconfig of the k8s cluster
Use "ezctl help <command>" for more information about a given command.
[root@k8s-master1 /etc/kubeasz]#
[root@k8s-master1 /etc/kubeasz]# ezctl help setup
Usage: ezctl setup <cluster> <step>
available steps:
01 prepare to prepare CA/certs & kubeconfig & other system settings
02 etcd to setup the etcd cluster
03 container-runtime to setup the container runtime(docker or containerd)
04 kube-master to setup the master nodes
05 kube-node to setup the worker nodes
06 network to setup the network plugin
07 cluster-addon to setup other useful plugins
90 all to run 01~07 all at once
10 ex-lb to install external loadbalance for accessing k8s from outside
11 harbor to install a new harbor server or to integrate with an existed one
examples: ./ezctl setup test-k8s 01 (or ./ezctl setup test-k8s prepare)
./ezctl setup test-k8s 02 (or ./ezctl setup test-k8s etcd)
./ezctl setup test-k8s all
./ezctl setup test-k8s 04 -t restart_master
[root@k8s-master1 /etc/kubeasz]#
# 一键安装
ezctl setup waluna all1.4.1 环境初始化
3.0.0以后的版本，将 yml 文件放在了 playbooks 目录里面，早期版本直接在 /etc/ansible 目录中。早期版本的变量在各roles里，新版本将常用变量配置定义在config.yml文件中了。
# prepare文件
[root@k8s-master1 /etc/kubeasz]# tree roles/prepare/
roles/prepare/
├── files
│ └── sctp.conf
├── tasks
│ ├── centos.yml
│ ├── common.yml
│ ├── main.yml
│ ├── offline.yml
│ └── ubuntu.yml
└── templates
├── 10-k8s-modules.conf.j2
├── 30-k8s-ulimits.conf.j2
├── 95-k8s-journald.conf.j2
└── 95-k8s-sysctl.conf.j2
3 directories, 10 files
[root@k8s-master1 /etc/kubeasz]#
# deploy文件
[root@k8s-master1 /etc/kubeasz]# tree roles/deploy/
roles/deploy/
├── deploy.yml
├── tasks
│ ├── add-custom-kubectl-kubeconfig.yml
│ ├── create-kube-controller-manager-kubeconfig.yml
│ ├── create-kubectl-kubeconfig.yml
│ ├── create-kube-proxy-kubeconfig.yml
│ ├── create-kube-scheduler-kubeconfig.yml
│ └── main.yml
├── templates
│ ├── admin-csr.json.j2
│ ├── ca-config.json.j2
│ ├── ca-csr.json.j2
│ ├── crb.yaml.j2
│ ├── kube-controller-manager-csr.json.j2
│ ├── kube-proxy-csr.json.j2
│ ├── kube-scheduler-csr.json.j2
│ └── user-csr.json.j2
└── vars
└── main.yml
3 directories, 16 files
[root@k8s-master1 /etc/kubeasz]#
# 更改部署信息
[root@k8s-master1 /etc/kubeasz]# vim playbooks/01.prepare.yml
[root@k8s-master1 /etc/kubeasz]# cat playbooks/01.prepare.yml
# [optional] to synchronize system time of nodes with 'chrony'
- hosts:
- kube_master
- kube_node
- etcd
#- ex_lb # 注释此行，不对负载均衡做操作，因为已经配置好了
#- chrony # 注释此行，不进行时间同步，因为已经配置好时间同步
roles:
- { role: os-harden, when: "OS_HARDEN|bool" }
- { role: chrony, when: "groups['chrony']|length > 0" }
# to create CA, kubeconfig, kube-proxy.kubeconfig etc.
- hosts: localhost
roles:
- deploy
# prepare tasks for all nodes
- hosts:
- kube_master
- kube_node
- etcd
roles:
- prepare
[root@k8s-master1 /etc/kubeasz]#
# 证书信息都可以进行更改，这里使用默认信息。
[root@k8s-master1 /etc/kubeasz]# cat roles/deploy/templates/ca-csr.json.j2
{
"CN": "kubernetes",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "HangZhou",
"L": "XS",
"O": "k8s",
"OU": "System"
}
],
"ca": {
"expiry": "{{ CA_EXPIRY }}"
}
}
[root@k8s-master1 /etc/kubeasz]# grep -R CA_EXPIRY
roles/deploy/templates/ca-csr.json.j2: "expiry": "{{ CA_EXPIRY }}"
clusters/waluna/config.yml:CA_EXPIRY: "876000h"
Binary file down/kubeasz_3.1.0.tar matches
example/config.yml:CA_EXPIRY: "876000h"
[root@k8s-master1 /etc/kubeasz]#
# 证书有效期默认是变量，在clusters/waluna/config.yml文件中可以看到为876000h，即100年。
# 注意，不同版本的kubeasz，变量的定义文件路径不一样。
# 安装卸载包，会自动识别为Ubuntu系统还是centos系统
[root@k8s-master1 /etc/kubeasz]# ll roles/prepare/tasks/
total 28
drwxrwxr-x 2 root root 4096 Sep 25 15:30 ./
drwxrwxr-x 5 root root 4096 Sep 25 15:30 ../
-rw-rw-r-- 1 root root 1774 Sep 25 11:41 centos.yml
-rw-rw-r-- 1 root root 1850 Sep 25 11:41 common.yml
-rw-rw-r-- 1 root root 1668 Sep 25 11:41 main.yml
-rw-rw-r-- 1 root root 3469 Sep 25 11:41 offline.yml
-rw-rw-r-- 1 root root 1929 Sep 25 11:41 ubuntu.yml
[root@k8s-master1 /etc/kubeasz]#
# 内核参数的优化
[root@k8s-master1 /etc/kubeasz]# ll roles/prepare/templates/95-k8s-sysctl.conf.j2
-rw-rw-r-- 1 root root 515 Sep 25 11:41 roles/prepare/templates/95-k8s-sysctl.conf.j2
[root@k8s-master1 /etc/kubeasz]#
# 资源限制的优化
[root@k8s-master1 /etc/kubeasz]# ll roles/prepare/templates/30-k8s-ulimits.conf.j2
-rw-rw-r-- 1 root root 87 Sep 25 11:41 roles/prepare/templates/30-k8s-ulimits.conf.j2
[root@k8s-master1 /etc/kubeasz]#
# 变量设置
[root@k8s-master1 /etc/kubeasz]# grep -A 99 '# prepare' clusters/waluna/config.yml|grep -B 99 '# role:etcd'
# prepare
############################
# 可选离线安装系统软件包 (offline|online)
INSTALL_SOURCE: "online"
# 可选进行系统安全加固 github.com/dev-sec/ansible-collection-hardening
OS_HARDEN: false
# 设置时间源服务器【重要：集群内机器时间必须同步】
ntp_servers:
- "ntp1.aliyun.com"
- "time1.cloud.tencent.com"
- "0.cn.pool.ntp.org"
# 设置允许内部时间同步的网络段，比如"10.0.0.0/8"，默认全部允许
local_network: "0.0.0.0/0"
############################
# role:deploy
############################
# default: ca will expire in 100 years
# default: certs issued by the ca will expire in 50 years
CA_EXPIRY: "876000h"
CERT_EXPIRY: "438000h"
# kubeconfig 配置参数
CLUSTER_NAME: "cluster1"
CONTEXT_NAME: "context-{{ CLUSTER_NAME }}"
############################
# role:etcd
[root@k8s-master1 /etc/kubeasz]#
# 开始部署，-i指定hosts文件，-e @文件名 指定变量文件，注意：文件前面需要加@符号。
[root@k8s-master1 /etc/kubeasz]# ansible-playbook -i clusters/waluna/hosts -e @clusters/waluna/config.yml playbooks/01.prepare.yml
# 或者使用ezctl命令执行
[root@k8s-master1 /etc/kubeasz]# ./ezctl setup waluna 011.5.4.2 部署 etcd 集群
可选更改启动脚本路径，这里使用默认配置。
# ectd文件
[root@k8s-master1 /etc/kubeasz]# tree roles/etcd/
roles/etcd/
├── clean-etcd.yml
├── defaults
│ └── main.yml
├── tasks
│ └── main.yml
└── templates
├── etcd-csr.json.j2
└── etcd.service.j2
3 directories, 5 files
[root@k8s-master1 /etc/kubeasz]#
# etcd变量
[root@k8s-master1 /etc/kubeasz]# grep -A 99 '# role:etcd' clusters/waluna/config.yml|grep -B 99 '# role:runtime'
# role:etcd
############################
# 设置不同的wal目录，可以避免磁盘io竞争，提高性能
ETCD_DATA_DIR: "/var/lib/etcd"
ETCD_WAL_DIR: ""
############################
# role:runtime [containerd,docker]
[root@k8s-master1 /etc/kubeasz]#
# 部署etcd
[root@k8s-master1 /etc/kubeasz]# ezctl setup waluna etcd
#或ezctl setup waluna 02
#或ansible-playbook -i clusters/waluna/hosts -e @clusters/waluna/config.yml playbooks/02.etcd.yml各etcd服务器验证etcd服务
[root@etcd1 ~]# export NODE_IPS="10.0.0.91 10.0.0.92 10.0.0.93"
[root@etcd1 ~]# echo $NODE_IPS
10.0.0.91 10.0.0.92 10.0.0.93
[root@etcd1 ~]# for ip in ${NODE_IPS}; do ETCDCTL_API=3 /usr/bin/etcdctl --endpoints=https://${ip}:2379 --cacert=/etc/kubernetes/ssl/ca.pem --cert=/etc/kubernetes/ssl/etcd.pem --key=/etc/kubernetes/ssl/etcd-key.pem endpoint health;done
https://10.0.0.91:2379 is healthy: successfully committed proposal: took = 10.01138ms
https://10.0.0.92:2379 is healthy: successfully committed proposal: took = 8.586852ms
https://10.0.0.93:2379 is healthy: successfully committed proposal: took = 10.99684ms
[root@etcd1 ~]#1.5.4.3 部署 docker
可以自己手动安装，这里使用kubeasz安装。可选更改启动脚本路径，这里使用默认配置
# docker文件
[root@k8s-master1 /etc/kubeasz]# tree roles/docker/
roles/docker/
├── files
│ ├── docker
│ └── docker-tag
├── tasks
│ └── main.yml
├── templates
│ ├── daemon.json.j2
│ └── docker.service.j2
└── vars
└── main.yml
4 directories, 6 files
[root@k8s-master1 /etc/kubeasz]#
# 查看docker变量配置
[root@k8s-master1 /etc/kubeasz]# grep -A 99 '# role:runtime' clusters/waluna/config.yml|grep -B 99 '# role:kube-master'
# role:runtime [containerd,docker]
############################
# ------------------------------------------- containerd
# [.]启用容器仓库镜像
ENABLE_MIRROR_REGISTRY: true
# [containerd]基础容器镜像
SANDBOX_IMAGE: "easzlab/pause-amd64:3.5"
# [containerd]容器持久化存储目录
CONTAINERD_STORAGE_DIR: "/var/lib/containerd"
# ------------------------------------------- docker
# [docker]容器存储目录
DOCKER_STORAGE_DIR: "/var/lib/docker"
# [docker]开启Restful API
ENABLE_REMOTE_API: false
# [docker]信任的HTTP仓库
INSECURE_REG: '["127.0.0.1/8"]'
############################
# role:kube-master
[root@k8s-master1 /etc/kubeasz]#
# 部署docker
[root@k8s-master1 /etc/kubeasz]# ezctl setup waluna 03登录harbor会报错常见原因
# 没有配置证书
[root@k8s-master1 /etc/kubeasz]# docker login harbor.waluna.top
Username: admin
Password:
Error response from daemon: Get "https://harbor.waluna.top/v2/": x509: certificate signed by unknown authority
[root@k8s-master1 /etc/kubeasz]#
# 因为docker版本过高，其go版本高于1.15，因为为 go 1.15 版本开始废弃 CommonName
[root@k8s-master1 /etc/kubeasz]# docker login harbor.waluna.top
Username: admin
Password:
Error response from daemon: Get "https://harbor.waluna.top/v2/": x509: certificate relies on legacy Common Name field, use SANs or temporarily enable Common Name matching with GODEBUG=x509ignoreCN=0
[root@k8s-master1 /etc/kubeasz]# 配置harbor证书
# 创建目录
[root@k8s-master1 /etc/kubeasz]# mkdir /etc/docker/certs.d/harbor.waluna.top -p
# 拷贝证书
[root@k8s-master1 /etc/kubeasz]# scp 10.0.0.59:/apps/harbor/certs/harbor.crt /etc/docker/certs.d/harbor.waluna.top
# 添加host文件解析
# vim /etc/hosts
10.0.0.9 harbor.waluna.top
# 重启 docker，也可以不重启
# systemctl restart docker
# 登录验证
[root@k8s-master1 /etc/kubeasz]# docker login harbor.waluna.top
Username: admin
Password:
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store
Login Succeeded
[root@k8s-master1 /etc/kubeasz]# 利用脚本拷贝证书和hosts解析文件
[root@k8s-master1 ~]# vim scp-crt.sh
[root@k8s-master1 ~]# cat scp-crt.sh
#!/bin/bash
# 目标主机列表
IP="
10.0.0.9
10.0.0.19
10.0.0.29
10.0.0.69
10.0.0.79
10.0.0.89
"
for node in ${IP};do
ssh ${node} "mkdir /etc/docker/certs.d/harbor.waluna.top -p"
echo "Harbor 证书目录创建成功!"
scp /etc/docker/certs.d/harbor.waluna.top/harbor.crt ${node}:/etc/docker/certs.d/harbor.waluna.top/harbor.crt
echo "Harbor 证书拷贝成功!"
scp /etc/hosts ${node}:/etc/hosts
echo "hosts 文件拷贝完成"
scp -r /root/.docker ${node}:/root/
echo "Harbor 认证文件拷贝完成!"
done
[root@k8s-master1 ~]#
# 执行脚本
[root@k8s-master1 ~]# bash scp-crt.sh1.5.4.4 部署 master
可选更改启动脚本路径，这里使用默认配置。
# master文件
[root@k8s-master1 /etc/kubeasz]# tree roles/kube-master/
roles/kube-master/
├── tasks
│ └── main.yml
├── templates
│ ├── aggregator-proxy-csr.json.j2
│ ├── kube-apiserver.service.j2
│ ├── kube-controller-manager.service.j2
│ ├── kubernetes-csr.json.j2
│ ├── kube-scheduler-config.yaml.j2
│ └── kube-scheduler.service.j2
└── vars
└── main.yml
3 directories, 8 files
[root@k8s-master1 /etc/kubeasz]#
# 查看master变量配置
[root@k8s-master1 /etc/kubeasz]# grep -A 99 '# role:kube-master' clusters/waluna/config.yml|grep -B 99 '# role:kube-node'
# role:kube-master
############################
# k8s 集群 master 节点证书配置，可以添加多个ip和域名（比如增加公网ip和域名）
MASTER_CERT_HOSTS:
- "10.0.0.9"
- "k8s.waluna.top"
#- "www.test.com"
# node 节点上 pod 网段掩码长度（决定每个节点最多能分配的pod ip地址）
# 如果flannel 使用 --kube-subnet-mgr 参数，那么它将读取该设置为每个节点分配pod网段
# https://github.com/coreos/flannel/issues/847
NODE_CIDR_LEN: 24
############################
# role:kube-node
[root@k8s-master1 /etc/kubeasz]#
# 此版本存在bug，因为hosts文件中定义路径在/usr/bin下，但是默认在/opt/kube/bin下，所以需将docker放在/usr/bin下
[root@k8s-master1 /etc/kubeasz]# cp /opt/kube/bin/docker /usr/bin/
[root@k8s-master1 /etc/kubeasz]#
# 部署master
[root@k8s-master1 /etc/kubeasz]# ezctl setup waluna 04
# 查看master节点
[root@k8s-master1 /etc/kubeasz]# kubectl get node
NAME STATUS ROLES AGE VERSION
10.0.0.19 Ready,SchedulingDisabled master 5m52s v1.21.5
10.0.0.29 Ready,SchedulingDisabled master 5m52s v1.21.5
10.0.0.9 Ready,SchedulingDisabled master 5m51s v1.21.5
[root@k8s-master1 /etc/kubeasz]# 1.5.4.5 部署 node
node 节点必须安装 docker，部署前注意dns，详见1.7.1配置
# node文件
[root@k8s-master1 /etc/kubeasz]# tree roles/kube-node/
roles/kube-node/
├── tasks
│ ├── create-kubelet-kubeconfig.yml
│ └── main.yml
├── templates
│ ├── cni-default.conf.j2
│ ├── haproxy.service.j2
│ ├── kubelet-config.yaml.j2
│ ├── kubelet-csr.json.j2
│ ├── kubelet.service.j2
│ ├── kube-proxy-config.yaml.j2
│ └── kube-proxy.service.j2
└── vars
└── main.yml
3 directories, 10 files
[root@k8s-master1 /etc/kubeasz]#
# 查看node变量配置
[root@k8s-master1 /etc/kubeasz]# grep -A 99 '# role:kube-node' clusters/waluna/config.yml|grep -B 99 '# role:network'
# role:kube-node
############################
# Kubelet 根目录
KUBELET_ROOT_DIR: "/var/lib/kubelet"
# node节点最大pod 数
MAX_PODS: 110
# 配置为kube组件（kubelet,kube-proxy,dockerd等）预留的资源量
# 数值设置详见templates/kubelet-config.yaml.j2
KUBE_RESERVED_ENABLED: "no"
# k8s 官方不建议草率开启 system-reserved, 除非你基于长期监控，了解系统的资源占用状况；
# 并且随着系统运行时间，需要适当增加资源预留，数值设置详见templates/kubelet-config.yaml.j2
# 系统预留设置基于 4c/8g 虚机，最小化安装系统服务，如果使用高性能物理机可以适当增加预留
# 另外，集群安装时候apiserver等资源占用会短时较大，建议至少预留1g内存
SYS_RESERVED_ENABLED: "no"
# haproxy balance mode
BALANCE_ALG: "roundrobin"
############################
# role:network [flannel,calico,cilium,kube-ovn,kube-router]
[root@k8s-master1 /etc/kubeasz]#
# 查看基础镜像
[root@k8s-master1 /etc/kubeasz]# grep image roles/kube-node/ -R
roles/kube-node/tasks/main.yml: copy: src={{ base_dir }}/down/{{ dnscache_offline }} dest=/opt/kube/images/{{ dnscache_offline }}
roles/kube-node/tasks/main.yml: command: "ls /opt/kube/images"
roles/kube-node/tasks/main.yml: register: image_info
roles/kube-node/tasks/main.yml: shell: "{{ bin_dir }}/docker load -i /opt/kube/images/{{ dnscache_offline }}"
roles/kube-node/tasks/main.yml: when: 'dnscache_offline in image_info.stdout and CONTAINER_RUNTIME == "docker"'
roles/kube-node/tasks/main.yml: shell: "{{ bin_dir }}/ctr -n=k8s.io images import /opt/kube/images/{{ dnscache_offline }}"
roles/kube-node/tasks/main.yml: when: 'dnscache_offline in image_info.stdout and CONTAINER_RUNTIME == "containerd"'
roles/kube-node/templates/kubelet.service.j2: --image-pull-progress-deadline=5m \
roles/kube-node/templates/kubelet.service.j2: --pod-infra-container-image={{ SANDBOX_IMAGE }} \
roles/kube-node/templates/kubelet-config.yaml.j2: imagefs.available: 15%
roles/kube-node/templates/kubelet-config.yaml.j2:imageGCHighThresholdPercent: 85
roles/kube-node/templates/kubelet-config.yaml.j2:imageGCLowThresholdPercent: 80
roles/kube-node/templates/kubelet-config.yaml.j2:imageMinimumGCAge: 2m0s
[root@k8s-master1 /etc/kubeasz]#
[root@k8s-master1 /etc/kubeasz]# grep SANDBOX_IMAGE -R
roles/kube-node/templates/kubelet.service.j2: --pod-infra-container-image={{ SANDBOX_IMAGE }} \
roles/containerd/templates/config.toml.j2: sandbox_image = "{{ SANDBOX_IMAGE }}"
clusters/waluna/config.yml:SANDBOX_IMAGE: "easzlab/pause-amd64:3.5"
docs/setup/05-install_kube_node.md: --pod-infra-container-image={{ SANDBOX_IMAGE }} \
Binary file down/kubeasz_3.1.1.tar matches
example/config.yml:SANDBOX_IMAGE: "easzlab/pause-amd64:__pause__"
[root@k8s-master1 /etc/kubeasz]#
# 拉取镜像测试
[root@k8s-master1 /etc/kubeasz]# docker pull easzlab/pause-amd64:3.5
3.5: Pulling from easzlab/pause-amd64
Digest: sha256:2f4b437353f90e646504ec8317dacd6123e931152674628289c990a7a05790b0
Status: Image is up to date for easzlab/pause-amd64:3.5
docker.io/easzlab/pause-amd64:3.5
[root@k8s-master1 /etc/kubeasz]#
# 打标签上传至harbor
[root@k8s-master1 /etc/kubeasz]# docker tag easzlab/pause-amd64:3.5 harbor.waluna.top/baseimages/pause-amd64:3.5
[root@k8s-master1 /etc/kubeasz]# docker push harbor.waluna.top/baseimages/pause-amd64:3.5
The push refers to repository [harbor.waluna.top/baseimages/pause-amd64]
dee215ffc666: Pushed
3.5: digest: sha256:2f4b437353f90e646504ec8317dacd6123e931152674628289c990a7a05790b0 size: 526
[root@k8s-master1 /etc/kubeasz]#
# 更改config文件,将基础镜像改为本地harbor地址
[root@k8s-master1 /etc/kubeasz]# vim clusters/waluna/config.yml
......
# [containerd]基础容器镜像
#SANDBOX_IMAGE: "easzlab/pause-amd64:3.5"
SANDBOX_IMAGE: "harbor.waluna.top/baseimages/pause-amd64:3.5"
......
# 部署node
[root@k8s-master1 /etc/kubeasz]# ezctl setup waluna 05
# 验证node
[root@k8s-master1 /etc/kubeasz]# kubectl get node
NAME STATUS ROLES AGE VERSION
10.0.0.19 Ready,SchedulingDisabled master 13m v1.21.5
10.0.0.29 Ready,SchedulingDisabled master 13m v1.21.5
10.0.0.69 Ready node 37s v1.21.5
10.0.0.79 Ready node 37s v1.21.5
10.0.0.89 Ready node 37s v1.21.5
10.0.0.9 Ready,SchedulingDisabled master 13m v1.21.5
[root@k8s-master1 /etc/kubeasz]#
# 之前版本使用的是haproxy进行调度，新版本使用的是nginx进行调度，将nginx自定义为kube-lb了。
[root@node1 ~]# tree /etc/kube-lb/
/etc/kube-lb/
├── conf
│ └── kube-lb.conf
├── logs
│ ├── error.log
│ └── nginx.pid
└── sbin
└── kube-lb
3 directories, 4 files
[root@node1 ~]# cat /etc/kube-lb/conf/kube-lb.conf
user root;
worker_processes 1;
error_log /etc/kube-lb/logs/error.log warn;
events {
worker_connections 3000;
}
stream {
upstream backend {
server 10.0.0.9:6443 max_fails=2 fail_timeout=3s;
server 10.0.0.19:6443 max_fails=2 fail_timeout=3s;
server 10.0.0.29:6443 max_fails=2 fail_timeout=3s;
}
server {
listen 127.0.0.1:6443;
proxy_connect_timeout 1s;
proxy_pass backend;
}
}
[root@node1 ~]# 1.5.4.6 部署网络服务 flannel
可选更改启动脚本路径，这里使用默认配置。
# 查看flannel
[root@k8s-master1 /etc/kubeasz]# tree roles/flannel/
roles/flannel/
├── tasks
│ └── main.yml
└── templates
└── kube-flannel.yaml.j2
2 directories, 2 files
[root@k8s-master1 /etc/kubeasz]#
# 查看网络变量配置
[root@k8s-master1 /etc/kubeasz]# grep -A 99 '# role:network' clusters/waluna/config.yml|grep -B 99 '# role:cluster-addon'
# role:network [flannel,calico,cilium,kube-ovn,kube-router]
############################
# ------------------------------------------- flannel
# [flannel]设置flannel 后端"host-gw","vxlan"等
FLANNEL_BACKEND: "vxlan"
DIRECT_ROUTING: false
# [flannel] flanneld_image: "quay.io/coreos/flannel:v0.10.0-amd64"
flannelVer: "v0.13.0-amd64"
flanneld_image: "easzlab/flannel:{{ flannelVer }}"
# [flannel]离线镜像tar包
flannel_offline: "flannel_{{ flannelVer }}.tar"
# ------------------------------------------- calico
# [calico]设置 CALICO_IPV4POOL_IPIP=“off”,可以提高网络性能，条件限制详见 docs/setup/calico.md
CALICO_IPV4POOL_IPIP: "Always"
# [calico]设置 calico-node使用的host IP，bgp邻居通过该地址建立，可手工指定也可以自动发现
IP_AUTODETECTION_METHOD: "can-reach={{ groups['kube_master'][0] }}"
# [calico]设置calico 网络 backend: brid, vxlan, none
CALICO_NETWORKING_BACKEND: "brid"
# [calico]更新支持calico 版本: [v3.3.x] [v3.4.x] [v3.8.x] [v3.15.x]
calico_ver: "v3.19.2"
# [calico]calico 主版本
calico_ver_main: "{{ calico_ver.split('.')[0] }}.{{ calico_ver.split('.')[1] }}"
# [calico]离线镜像tar包
calico_offline: "calico_{{ calico_ver }}.tar"
# ------------------------------------------- cilium
# [cilium]CILIUM_ETCD_OPERATOR 创建的 etcd 集群节点数 1,3,5,7...
ETCD_CLUSTER_SIZE: 1
# [cilium]镜像版本
cilium_ver: "v1.4.1"
# [cilium]离线镜像tar包
cilium_offline: "cilium_{{ cilium_ver }}.tar"
# ------------------------------------------- kube-ovn
# [kube-ovn]选择 OVN DB and OVN Control Plane 节点，默认为第一个master节点
OVN_DB_NODE: "{{ groups['kube_master'][0] }}"
# [kube-ovn]离线镜像tar包
kube_ovn_ver: "v1.5.3"
kube_ovn_offline: "kube_ovn_{{ kube_ovn_ver }}.tar"
# ------------------------------------------- kube-router
# [kube-router]公有云上存在限制，一般需要始终开启 ipinip；自有环境可以设置为 "subnet"
OVERLAY_TYPE: "full"
# [kube-router]NetworkPolicy 支持开关
FIREWALL_ENABLE: "true"
# [kube-router]kube-router 镜像版本
kube_router_ver: "v0.3.1"
busybox_ver: "1.28.4"
# [kube-router]kube-router 离线镜像tar包
kuberouter_offline: "kube-router_{{ kube_router_ver }}.tar"
busybox_offline: "busybox_{{ busybox_ver }}.tar"
############################
# role:cluster-addon
[root@k8s-master1 /etc/kubeasz]#
# 查看flannel.yaml文件的位置
[root@k8s-master1 /etc/kubeasz]# grep flannel.yaml roles/flannel/tasks/main.yml
template: src=kube-flannel.yaml.j2 dest={{ cluster_dir }}/yml/flannel.yaml
shell: "{{ base_dir }}/bin/kubectl apply -f {{ cluster_dir }}/yml/flannel.yaml"
[root@k8s-master1 /etc/kubeasz]#
# 查看镜像地址
[root@k8s-master1 /etc/kubeasz]# grep image roles/flannel/templates/kube-flannel.yaml.j2
image: {{ flanneld_image }}
image: {{ flanneld_image }}
[root@k8s-master1 /etc/kubeasz]#
# 查看变量值
[root@k8s-master1 /etc/kubeasz]# grep flanneld_image -R
roles/flannel/templates/kube-flannel.yaml.j2: image: {{ flanneld_image }}
roles/flannel/templates/kube-flannel.yaml.j2: image: {{ flanneld_image }}
clusters/waluna/config.yml:# [flannel] flanneld_image: "quay.io/coreos/flannel:v0.10.0-amd64"
clusters/waluna/config.yml:flanneld_image: "easzlab/flannel:{{ flannelVer }}"
Binary file down/kubeasz_3.1.1.tar matches
example/config.yml:# [flannel] flanneld_image: "quay.io/coreos/flannel:v0.10.0-amd64"
example/config.yml:flanneld_image: "easzlab/flannel:{{ flannelVer }}"
[root@k8s-master1 /etc/kubeasz]#
[root@k8s-master1 /etc/kubeasz]# grep flannelVer -R
ezdown:flannelVer=v0.13.0-amd64
ezdown: if [[ ! -f "$imageDir/flannel_$flannelVer.tar" ]];then
ezdown: docker pull "easzlab/flannel:$flannelVer" && \
ezdown: docker save -o "$imageDir/flannel_$flannelVer.tar" "easzlab/flannel:$flannelVer"
clusters/waluna/config.yml:flannelVer: "v0.13.0-amd64"
clusters/waluna/config.yml:flanneld_image: "easzlab/flannel:{{ flannelVer }}"
clusters/waluna/config.yml:flannel_offline: "flannel_{{ flannelVer }}.tar"
Binary file down/kubeasz_3.1.1.tar matches
ezctl: flannelVer=$(grep 'flannelVer=' ezdown|cut -d'=' -f2)
ezctl: sed -i -e "s/__flannel__/$flannelVer/g" \
example/config.yml:flannelVer: "__flannel__"
example/config.yml:flanneld_image: "easzlab/flannel:{{ flannelVer }}"
example/config.yml:flannel_offline: "flannel_{{ flannelVer }}.tar"
[root@k8s-master1 /etc/kubeasz]#
# 在down目录也有flannel镜像
[root@k8s-master1 /etc/kubeasz]# ll down/flannel_v0.13.0-amd64.tar
-rw------- 1 root root 58150912 Nov 13 23:43 down/flannel_v0.13.0-amd64.tar
[root@k8s-master1 /etc/kubeasz]#
# 有多种方式安装，可以使用down目录的的镜像，直接上传至本地harbor，也可以区flannel的github项目里找取镜像地址，这里使用配置文件的镜像，拉去后打标签上传至本地harbor
[root@k8s-master1 /etc/kubeasz]# docker pull easzlab/flannel:v0.13.0-amd64
v0.13.0-amd64: Pulling from easzlab/flannel
Digest: sha256:34860ea294a018d392e61936f19a7862d5e92039d196cac9176da14b2bbd0fe3
Status: Image is up to date for easzlab/flannel:v0.13.0-amd64
docker.io/easzlab/flannel:v0.13.0-amd64
[root@k8s-master1 /etc/kubeasz]# docker tag easzlab/flannel:v0.13.0-amd64 harbor.waluna.top/baseimages/flannel:v0.13.0-amd64
[root@k8s-master1 /etc/kubeasz]# docker push harbor.waluna.top/baseimages/flannel:v0.13.0-amd64
The push refers to repository [harbor.waluna.top/baseimages/flannel]
1a6a4161ff3a: Pushed
8a984b390686: Pushed
bfb960ebd228: Pushed
24d8f5a426b6: Pushed
90679e912622: Pushed
0be670d27a91: Pushed
50644c29ef5a: Pushed
v0.13.0-amd64: digest: sha256:34860ea294a018d392e61936f19a7862d5e92039d196cac9176da14b2bbd0fe3 size: 1785
[root@k8s-master1 /etc/kubeasz]#
# 更改镜像地址
[root@k8s-master1 /etc/kubeasz]# vim clusters/waluna/config.yml
......
# [flannel] flanneld_image: "quay.io/coreos/flannel:v0.10.0-amd64"
flannelVer: "v0.13.0-amd64"
#flanneld_image: "easzlab/flannel:{{ flannelVer }}"
flanneld_image: "harbor.waluna.top/baseimages/flannel:{{ flannelVer }}"
......
# 部署网络组件flannel
[root@k8s-master1 /etc/kubeasz]# ezctl setup waluna 06验证 flannel
# 拉取apline镜像上传至本地harbor
[root@k8s-master1 /etc/kubeasz]# docker pull alpine
Using default tag: latest
latest: Pulling from library/alpine
a0d0a0d46f8b: Pull complete
Digest: sha256:e1c082e3d3c45cccac829840a25941e679c25d438cc8412c2fa221cf1a824e6a
Status: Downloaded newer image for alpine:latest
docker.io/library/alpine:latest
[root@k8s-master1 /etc/kubeasz]# docker tag alpine:latest harbor.waluna.top/baseimages/alpine:latest
[root@k8s-master1 /etc/kubeasz]# docker push harbor.waluna.top/baseimages/alpine:latest
The push refers to repository [harbor.waluna.top/baseimages/alpine]
e2eb06d8af82: Pushed
latest: digest: sha256:69704ef328d05a9f806b6b8502915e6a0a4faa4d72018dc42343f511490daf8a size: 528
[root@k8s-master1 /etc/kubeasz]#
# 创建pod进行测试
[root@k8s-master1 /etc/kubeasz]# kubectl run net-test1 --image=alpine sleep 360000
pod/net-test1 created
[root@k8s-master1 /etc/kubeasz]# kubectl run net-test2 --image=alpine sleep 360000
pod/net-test2 created
[root@k8s-master1 /etc/kubeasz]# kubectl get pod -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
net-test1 1/1 Running 0 20s 10.10.3.2 10.0.0.79 <none> <none>
net-test2 1/1 Running 0 10s 10.10.4.2 10.0.0.69 <none> <none>
[root@k8s-master1 /etc/kubeasz]#
# 验证网络
[root@k8s-master1 /etc/kubeasz]# kubectl exec -it net-test1 sh
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
/ # ifconfig
eth0 Link encap:Ethernet HWaddr 76:1C:F6:23:4D:EE
inet addr:10.10.3.2 Bcast:10.10.3.255 Mask:255.255.255.0
UP BROADCAST RUNNING MULTICAST MTU:1450 Metric:1
RX packets:12 errors:0 dropped:0 overruns:0 frame:0
TX packets:1 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:1024 (1.0 KiB) TX bytes:42 (42.0 B)
lo Link encap:Local Loopback
inet addr:127.0.0.1 Mask:255.0.0.0
UP LOOPBACK RUNNING MTU:65536 Metric:1
RX packets:0 errors:0 dropped:0 overruns:0 frame:0
TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:0 (0.0 B) TX bytes:0 (0.0 B)
/ # ping -c1 10.10.4.2
PING 10.10.4.2 (10.10.4.2): 56 data bytes
64 bytes from 10.10.4.2: seq=0 ttl=62 time=2.044 ms
--- 10.10.4.2 ping statistics ---
1 packets transmitted, 1 packets received, 0% packet loss
round-trip min/avg/max = 2.044/2.044/2.044 ms
/ # ping -c1 223.5.5.5
PING 223.5.5.5 (223.5.5.5): 56 data bytes
64 bytes from 223.5.5.5: seq=0 ttl=127 time=11.205 ms
--- 223.5.5.5 ping statistics ---
1 packets transmitted, 1 packets received, 0% packet loss
round-trip min/avg/max = 11.205/11.205/11.205 ms
/ # ping -c1 baidu.com
ping: bad address 'baidu.com'
/ # exit
command terminated with exit code 1
[root@k8s-master1 /etc/kubeasz]#
# 可以看到内外网都已经通，只是没有dns，没法ping通域名
Comments | NOTHING