Debian 11 部署 K8s 集群

K8s

一、集群规划

ip hostname role version
10.4.7.201 debian201 master v1.28.2
10.4.7.202 debian202 node v1.28.2
10.4.7.203 debian203 node v1.28.2

二、前期准备

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
# 检查mac地址和product_uuid（节点唯一）
$ ip link
$ sudo cat /sys/class/dmi/id/product_uuid

# 修改hosts（主机名略）
$ sudo vim /etc/hosts
10.4.7.201   debian201
10.4.7.202   debian202
10.4.7.203   debian203

# 临时开启内核模块（让iptables看到桥接流量）
$ sudo modprobe overlay
$ sudo modprobe br_netfilter

# 让Linux系统在系统引导期间启用内核模块
$ cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF

# 查看内核模块是否开启
$ lsmod | grep br_netfilter

# 配置sysctl
$ cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward                 = 1
EOF
$ sudo sysctl --system  # 在不重启机器的情况下应用配置

# 关闭SWAP
$ sudo sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab  # 永久关闭
$ sudo swapoff -a  # 当前会话关闭
$ free -m

基于安全性（如在官方文档中承诺的 Secret 只会在内存中读写，不会落盘）、利于保证节点同步一致性等原因，从 1.8 版开始，Kubernetes 就在它的文档中明确声明了它默认不支持 Swap 分区，在未关闭 Swap 分区的机器中，集群将直接无法启动

三、容器运行时

1. Containerd

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
# 1. containerd
$ wget -c https://github.com/containerd/containerd/releases/download/v1.7.6/containerd-1.7.6-linux-amd64.tar.gz
$ sudo tar Cxzvf /usr/local containerd-1.7.6-linux-amd64.tar.gz

$ wget -c https://raw.githubusercontent.com/containerd/containerd/main/containerd.service
$ sudo mkdir -p /usr/local/lib/systemd/system/
$ sudo cp containerd.service /usr/local/lib/systemd/system/
$ sudo chmod 755 /usr/local/lib/systemd/system/containerd.service

$ sudo mkdir -p /etc/containerd/
$ containerd config default | sudo tee /etc/containerd/config.toml
$ sudo vi /etc/containerd/config.toml
[plugins."io.containerd.grpc.v1.cri"]
  sandbox_image = "registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.9"
  #systemd_cgroup = false
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
  SystemdCgroup = true
[plugins."io.containerd.grpc.v1.cri".registry]
  [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
    [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
      endpoint = ["https://ung2thfc.mirror.aliyuncs.com","https://docker.mirrors.ustc.edu.cn"]
    [plugins."io.containerd.grpc.v1.cri".registry.mirrors."gcr.io"]
      endpoint = ["https://gcr.mirrors.ustc.edu.cn"]
    [plugins."io.containerd.grpc.v1.cri".registry.mirrors."k8s.gcr.io"]
      endpoint = ["https://gcr.mirrors.ustc.edu.cn/google-containers/"]
    [plugins."io.containerd.grpc.v1.cri".registry.mirrors."quay.io"]
      endpoint = ["https://quay.mirrors.ustc.edu.cn"]
    [plugins."io.containerd.grpc.v1.cri".registry.mirrors."registry.k8s.io"]
        endpoint = ["k8s.m.daocloud.io"]
    [plugins."io.containerd.grpc.v1.cri".registry.mirrors."10.4.7.200:88"]
      endpoint = ["http://10.4.7.200:88"]
  [plugins."io.containerd.grpc.v1.cri".registry.configs]
    [plugins."io.containerd.grpc.v1.cri".registry.configs."10.4.7.200:88".tls]
      insecure_skip_verify = true
    [plugins."io.containerd.grpc.v1.cri".registry.configs."10.4.7.200:88".auth]
      username = "admin"
      password = "Harbor12345"

$ sudo systemctl daemon-reload
$ sudo systemctl enable --now containerd

# 2. runc
$ wget -c https://github.com/opencontainers/runc/releases/download/v1.1.9/runc.arm64
$ sudo install -m 755 runc.amd64 /usr/local/sbin/runc

# 3. CNI plugins
$ wget -c https://github.com/containernetworking/plugins/releases/download/v1.3.0/cni-plugins-linux-amd64-v1.3.0.tgz
$ sudo mkdir -p /opt/cni/bin
$ sudo tar Cxzvf /opt/cni/bin cni-plugins-linux-amd64-v1.3.0.tgz

# 4. crictl
$ wget -c https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.28.0/crictl-v1.28.0-linux-amd64.tar.gz
$ sudo tar Czxvf /usr/local/bin crictl-v1.28.0-linux-amd64.tar.gz
$ cat << EOF | sudo tee /etc/crictl.yaml
runtime-endpoint: unix:///run/containerd/containerd.sock
image-endpoint: unix:///run/containerd/containerd.sock
timeout: 2
debug: false
pull-image-on-create: false
EOF

pause 版本可通过 sudo kubeadm config images list --kubernetes-version v1.28.2 查看

2. Docker

  • K8s 1.24 版本移除了 Dockershim，因此需要注意容器运行时的选择
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
# Uninstall old versions
$ for pkg in docker.io docker-doc docker-compose podman-docker containerd runc; do sudo apt-get remove $pkg; done

# Add Docker's official GPG key:
$ sudo apt-get update
$ sudo apt-get install ca-certificates curl gnupg
$ sudo install -m 0755 -d /etc/apt/keyrings
$ curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
$ sudo chmod a+r /etc/apt/keyrings/docker.gpg

# Add the repository to Apt sources:
$ echo \
  "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \
  "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \
  sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
$ sudo apt-get update

# Install the Docker packages
$ sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin

$ cat << EOF | sudo tee /etc/docker/daemon.json
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "storage-driver": "overlay2",
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "100m"
  },
  "insecure-registries": ["10.4.7.200:88"],
  "registry-mirrors": ["https://ung2thfc.mirror.aliyuncs.com","https://registry.docker-cn.com"]
}
EOF
$ sudo systemctl restart docker

$ sudo usermod -aG docker $USER

$ sudo curl -L "https://github.com/docker/compose/releases/download/v2.22.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose

3. CRI-O

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
# 安装依赖
$ sudo apt install gnupg2 apt-transport-https -y

# 添加CRI-O仓库和GPG key
$ export OS=Debian_11
$ export VERSION=1.24
$ curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/Release.key | sudo gpg --dearmor -o /usr/share/keyrings/libcontainers-archive-keyring.gpg
$ curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$VERSION/$OS/Release.key | sudo gpg --dearmor -o /usr/share/keyrings/libcontainers-crio-archive-keyring.gpg
$ echo "deb [signed-by=/usr/share/keyrings/libcontainers-archive-keyring.gpg] https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/ /" | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list
$ echo "deb [signed-by=/usr/share/keyrings/libcontainers-crio-archive-keyring.gpg] https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$VERSION/$OS/ /" | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:$VERSION.list

# 更新并刷新存储库和软件包索引
$ sudo apt update

# 安装CRI-O
$ sudo apt install cri-o cri-o-runc cri-tools -y

# 配置
$ sudo vim /etc/crio/crio.conf
[crio.network]
network_dir = "/etc/cni/net.d/"
plugin_dirs = [
    "/opt/cni/bin/",
]
$ sudo vim /etc/cni/net.d/100-crio-bridge.conf
"ranges": [  # 使用自定义子网（Pod）（确保和CNI插件的地址匹配）
    [{ "subnet": "10.10.0.0/24" }],
    [{ "subnet": "1100:200::/24" }]
]
$ sudo systemctl restart crio
$ sudo systemctl enable crio

三、K8s 集群

  • kubeadm：用于引导、安装和启动 Kubernetes 集群的命令行工具，只需要简单的几个命令工具（如 init、join、upgrade、reset 等），便可以快速完成集群的维护管理工作
  • kubelet：每个节点上的代理程序，负责节点上的大部分操作，比如由它来启停容器等，节点上只有它能与 apiserver 进行通信
  • kubectl：Kubernetes 客户端工具，用来操控 Kubernetes 集群，比如查看集群中的节点信息等
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
# 1. 注册apt软件源
$ sudo apt install -y apt-transport-https ca-certificates curl gnupg2
$ curl https://mirrors.aliyun.com/kubernetes/apt/doc/apt-key.gpg | sudo apt-key add -
$ cat <<EOF | sudo tee /etc/apt/sources.list.d/kubernetes.list
deb https://mirrors.aliyun.com/kubernetes/apt/ kubernetes-xenial main
EOF
$ sudo apt update

# 2. 安装k8s
$ sudo apt install -y kubeadm=1.28.2-00 kubelet=1.28.2-00 kubectl=1.28.2-00
$ sudo apt-mark hold kubeadm kubelet kubectl
$ sudo systemctl enable kubelet --now

# 3. master节点初始化集群
$ sudo kubeadm init \
 --image-repository registry.cn-hangzhou.aliyuncs.com/google_containers \
 --pod-network-cidr=10.10.0.0/16 \
 --apiserver-advertise-address=10.4.7.201 \
 --kubernetes-version=v1.28.2 \
 --v=5
$ mkdir -p $HOME/.kube
$ sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
$ sudo chown $(id -u):$(id -g) $HOME/.kube/config

# 4. node节点加入集群
$ sudo kubeadm join 10.4.7.201:6443 --token o1szra.s2i6veja96njcdq5 \
        --discovery-token-ca-cert-hash sha256:bf61c81dbacc0a4909a373c9b388e243ea37ce07142c5c79c60c28c7e39496b4

四、CNI 网络插件

1. calico

1
2
3
4
5
$ curl https://raw.githubusercontent.com/projectcalico/calico/v3.26.1/manifests/calico.yaml -O
#$ vi calico.yaml
#- name: CALICO_IPV4POOL_CIDR
#  value: "10.10.0.0/16"
$ kubectl apply -f calico.yaml

2. flannel

1
2
3
4
5
6
7
8
9
10
$ curl https://raw.githubusercontent.com/flannel-io/flannel/master/Documentation/kube-flannel.yml -O
#$ vi kube-flannel.yml
#net-conf.json: |
#  {
#    "Network": "10.10.0.0/16",
#    "Backend": {
#      "Type": "vxlan"
#    }
#  }
$ kubectl apply -f kube-flannel.yml

五、Helm

1
2
3
$ wget -c https://get.helm.sh/helm-v3.13.0-linux-amd64.tar.gz
$ tar -zxvf helm-v3.13.0-linux-amd64.tar.gz
$ sudo mv linux-amd64/helm /usr/local/bin/helm

六、其他命令

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
# 命令自动补全
$ echo "source <(kubectl completion bash)" >> ~/.bashrc

# 移除master上的污点
$ kubectl taint nodes debian201 node-role.kubernetes.io/control-plane:NoSchedule-

# 调整NodePort范围（默认30000~32767）
$ sudo vi /etc/kubernetes/manifests/kube-apiserver.yaml
  --service-node-port-range=5000-32767
$ export apiserver_pods=$(kubectl get pods --selector=component=kube-apiserver -n kube-system --output=jsonpath={.items..metadata.name})
$ kubectl delete pod $apiserver_pods -n kube-system
$ kubectl describe pod $apiserver_pods -n kube-system | grep 5000

# 生成node节点加入的token（默认24h）
$ kubeadm token create --print-join-command

参考