五、开始安装

所有k8s节点安装kubeadm、kubectl、kubelet

root@master1:~# apt-get update && apt-get install -y apt-transport-https  #安装依赖
root@master1:~# curl https://mirrors.aliyun.com/kubernetes/apt/doc/apt-key.gpg | apt-key add -  #导入密钥
root@master1:~# cat <<EOF >/etc/apt/sources.list.d/kubernetes.list  #导入阿里的kubernetes源
deb https://mirrors.aliyun.com/kubernetes/apt/ kubernetes-xenial main
EOF
root@master1:~# for i in  master2 master3 node1 node2 node3;do scp /etc/apt/sources.list.d/kubernetes.list 
$i:/etc/apt/sources.list.d/;done  #传输给其他节点
root@master1:~# for i in master2 master3 node1 node2 node3;do ssh $i apt update;done
root@master1:~# for i in master2 master3 node1 node2 node3;do ssh $i apt -y install kubelet kubeadm kubectl;done #安装最新版本

如需要安装指定版本需要按如下操作

root@master1:~# apt-cache madison kubeadm|head  #查询前10条，或者也可以用apt-cache show kubeadm |grep 1.25
   kubeadm |  1.25.2-00 | https://mirrors.aliyun.com/kubernetes/apt kubernetes-xenial/main amd64 Packages
   kubeadm |  1.25.1-00 | https://mirrors.aliyun.com/kubernetes/apt kubernetes-xenial/main amd64 Packages
   kubeadm |  1.25.0-00 | https://mirrors.aliyun.com/kubernetes/apt kubernetes-xenial/main amd64 Packages
   kubeadm |  1.24.6-00 | https://mirrors.aliyun.com/kubernetes/apt kubernetes-xenial/main amd64 Packages
   kubeadm |  1.24.5-00 | https://mirrors.aliyun.com/kubernetes/apt kubernetes-xenial/main amd64 Packages
   kubeadm |  1.24.4-00 | https://mirrors.aliyun.com/kubernetes/apt kubernetes-xenial/main amd64 Packages
   kubeadm |  1.24.3-00 | https://mirrors.aliyun.com/kubernetes/apt kubernetes-xenial/main amd64 Packages
   kubeadm |  1.24.2-00 | https://mirrors.aliyun.com/kubernetes/apt kubernetes-xenial/main amd64 Packages
   kubeadm |  1.24.1-00 | https://mirrors.aliyun.com/kubernetes/apt kubernetes-xenial/main amd64 Packages
   kubeadm |  1.24.0-00 | https://mirrors.aliyun.com/kubernetes/apt kubernetes-xenial/main amd64 Packages
root@master1:~# apt install -y  kubeadm=1.25.2-00 kubelet=1.25.2-00 kubectl=1.25.2-00

所有k8s节点安装cri-dockerd

Kubernetes自v1.24移除了对docker-shim的支持，而Docker Engine默认又不支持CRI规范，因而二者将无法直接完成整合。为此，Mirantis和Docker联合创建了cri-dockerd项目，用于为Docker Engine提供一个能够支持到CRI规范的垫片，从而能够让Kubernetes基于CRI控制Docker 。

root@master1:~#  wget https://github.com/Mirantis/cri-dockerd/releases/download/v0.2.6/cri-dockerd_0.2.6.3-0.ubuntu-focal_amd64.deb #在github中有二进制包也有deb和rpm包，为了方便我这里直接下载的deb包
root@master1:~# for i in  master2 master3 node1 node2 node3;do scp cri-dockerd_0.2.6.3-0.ubuntu-focal_amd64.deb 
$i:/root/;done  #传输给其他节点
root@master1:~# for i in  master2 master3 node1 node2 node3;do ssh $i dpkg -i cri-dockerd_0.2.6.3-0.ubuntu-focal_amd64.de ;done

修改cri-dockerd的service文件，否则会报错，嫌麻烦可以直接复制下面的文件

root@master1:~# cat /lib/systemd/system/cri-docker.service 
[Unit]
Description=CRI Interface for Docker Application Container Engine
Documentation=https://docs.mirantis.com
After=network-online.target firewalld.service docker.service
Wants=network-online.target
Requires=cri-docker.socket
[Service]
Type=notify
#ExecStart=/usr/bin/cri-dockerd --container-runtime-endpoint fd:// --pod-infracontainer-image registry.aliyuncs.com/google_containers/pause:3.7  
ExecStart=/usr/bin/cri-dockerd --network-plugin=cni --pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.7 #这里地址不改会起不来，修改仓库地址为阿里云是为了pull镜像加速
ExecReload=/bin/kill -s HUP $MAINPID
TimeoutSec=0
RestartSec=2
Restart=always
# Note that StartLimit* options were moved from "Service" to "Unit" in systemd 229.
# Both the old, and new location are accepted by systemd 229 and up, so using the old location
# to make them work for either version of systemd.
StartLimitBurst=3
# Note that StartLimitInterval was renamed to StartLimitIntervalSec in systemd 230.
# Both the old, and new name are accepted by systemd 230 and up, so using the old name to make
# this option work for either version of systemd.
StartLimitInterval=60s
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
# Comment TasksMax if your systemd version does not support it.
# Only systemd 226 and above support this option.
TasksMax=infinity
Delegate=yes
KillMode=process
[Install]
WantedBy=multi-user.target
root@master1:~# for i in  master2 master3 node1 node2 node3;do scp /lib/systemd/system/cri-docker.service  
$i:/lib/systemd/system/cri-docker.service ;done #传输给其他节点
root@master1:~# for i in  master2 master3 node1 node2 node3;do ssh $i systemctl daemon-reload;ssh $i systemctl enable --now cri-docker.service ;done

在master1初始化

root@master1:~#  kubeadm config print init-defaults > init.yaml     #生成初始化配置文件，方便修改参数
root@master1:~# cat init.yaml 
apiVersion: kubeadm.k8s.io/v1beta3
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 10.10.21.178    #填nginx的IP地址，如果是其他反向代理也填其他proxy的地址
  bindPort: 6443
nodeRegistration:
  criSocket:  unix:///var/run/cri-dockerd.sock  #由于是用的docker这里必须修改，否则初始化会报错
  imagePullPolicy: IfNotPresent
  name: master1             # 和节点名字对应
  taints: null
---
apiServer:
  timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns: {}
etcd:
  external:             #这里默认是local，由于我们用的外部etcd，所以需要修改
    endpoints:
      - https://10.10.21.170:2379
      - https://10.10.21.172:2379
      - https://10.10.21.175:2379
    #搭建etcd集群时生成的ca证书
    caFile: /etc/etcd/pki/ca.pem
    #搭建etcd集群时生成的客户端证书
    certFile: /etc/etcd/pki/client.pem
    #搭建etcd集群时生成的客户端密钥
    keyFile: /etc/etcd/pki/client-key.pem
imageRepository: registry.aliyuncs.com/google_containers
kind: ClusterConfiguration
kubernetesVersion: 1.25.2   #和版本需要一一对应
networking:
  dnsDomain: cluster.local
  podSubnet: 10.200.0.0/16      #pod的CIDR地址，不要和别的有交叉
  serviceSubnet: 10.100.0.0/16    #service的CIDR地址，不要和别的有交叉
scheduler: {} 
root@master1:~# kubeadm init --config=init.yaml #利用修改完成的文件进行初始化

由于已经配置从阿里拉取镜像，所以这一步并不会太久，出现类似以下提示就代表初始化完成了

root@master1:~# export KUBECONFIG=/etc/kubernetes/admin.conf #如果是root用户执行这条，普通用户需要执行上面的那几条，并且如果你将以上文件scp给其它节点，那么其它节点也能执行kubectl命令

安装CNI插件flannel

root@master1:~#  wget  https://raw.githubusercontent.com/flannelio/flannel/master/Documentation/kube-flannel.yml
root@master1:~#  cat kube-flannel.yml 
---
kind: Namespace
apiVersion: v1
metadata:
  name: kube-flannel
  labels:
    pod-security.kubernetes.io/enforce: privileged
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: flannel
rules:
- apiGroups:
  - ""
  resources:
  - pods
  verbs:
  - get
- apiGroups:
  - ""
  resources:
  - nodes
  verbs:
  - list
  - watch
- apiGroups:
  - ""
  resources:
  - nodes/status
  verbs:
  - patch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: flannel
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: flannel
subjects:
- kind: ServiceAccount
  name: flannel
  namespace: kube-flannel
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: flannel
  namespace: kube-flannel
---
kind: ConfigMap
apiVersion: v1
metadata:
  name: kube-flannel-cfg
  namespace: kube-flannel
  labels:
    tier: node
    app: flannel
data:
  cni-conf.json: |
    {
      "name": "cbr0",
      "cniVersion": "0.3.1",
      "plugins": [
        {
          "type": "flannel",
          "delegate": {
            "hairpinMode": true,
            "isDefaultGateway": true
          }
        },
        {
          "type": "portmap",
          "capabilities": {
            "portMappings": true
          }
        }
      ]
    }
  net-conf.json: |
    {
      "Network": "10.200.0.0/16",
      "Backend": {
        "Type": "vxlan"
      }
    }
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: kube-flannel-ds
  namespace: kube-flannel
  labels:
    tier: node
    app: flannel
spec:
  selector:
    matchLabels:
      app: flannel
  template:
    metadata:
      labels:
        tier: node
        app: flannel
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: kubernetes.io/os
                operator: In
                values:
                - linux
      hostNetwork: true
      priorityClassName: system-node-critical
      tolerations:
      - operator: Exists
        effect: NoSchedule
      serviceAccountName: flannel
      initContainers:
      - name: install-cni-plugin
       #image: flannelcni/flannel-cni-plugin:v1.1.0 for ppc64le and mips64le (dockerhub limitations may apply)
        image: docker.io/rancher/mirrored-flannelcni-flannel-cni-plugin:v1.1.0
        command:
        - cp
        args:
        - -f
        - /flannel
        - /opt/cni/bin/flannel
        volumeMounts:
        - name: cni-plugin
          mountPath: /opt/cni/bin
      - name: install-cni
       #image: flannelcni/flannel:v0.19.2 for ppc64le and mips64le (dockerhub limitations may apply)
        image: docker.io/rancher/mirrored-flannelcni-flannel:v0.19.2
        command:
        - cp
        args:
        - -f
        - /etc/kube-flannel/cni-conf.json
        - /etc/cni/net.d/10-flannel.conflist
        volumeMounts:
        - name: cni
          mountPath: /etc/cni/net.d
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      containers:
      - name: kube-flannel
       #image: flannelcni/flannel:v0.19.2 for ppc64le and mips64le (dockerhub limitations may apply)
        image: docker.io/rancher/mirrored-flannelcni-flannel:v0.19.2
        command:
        - /opt/bin/flanneld
        args:
        - --ip-masq
        - --kube-subnet-mgr
        resources:
          requests:
            cpu: "100m"
            memory: "50Mi"
          limits:
            cpu: "100m"
            memory: "50Mi"
        securityContext:
          privileged: false
          capabilities:
            add: ["NET_ADMIN", "NET_RAW"]
        env:
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        - name: EVENT_QUEUE_DEPTH
          value: "5000"
        volumeMounts:
        - name: run
          mountPath: /run/flannel
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
        - name: xtables-lock
          mountPath: /run/xtables.lock
      volumes:
      - name: run
        hostPath:
          path: /run/flannel
      - name: cni-plugin
        hostPath:
          path: /opt/cni/bin
      - name: cni
        hostPath:
          path: /etc/cni/net.d
      - name: flannel-cfg
        configMap:
          name: kube-flannel-cfg
      - name: xtables-lock
        hostPath:
          path: /run/xtables.lock
          type: FileOrCreate
# 下载不下来的可以复制我的使用，但是记得修改CIDR与前面定义的pod的一致，否则在后面起coredns的pod会失败

root@master1:~# kubectl apply -f kube-flannel.yml
root@master1:~# kubectl get pod -n kube-system  #查看coredns是否正常，如果是running就可以进行下一步，否则node是notready状态
root@master1:~# kubectl get node
NAME             STATUS   ROLES           AGE     VERSION
master1   Ready    control-plane   6d4h    v1.25.2
# 此时应该是这种状态

扩容work节点

正常初始化完成会直接有kubeadm join的命令提示，如果忘记保存或者是token过期可以重新生成

root@master1:~# kubeadm token create --print-join-command 
kubeadm join 10.10.21.178:6443 --token x4mb27.hidqr7ao758eafcx --discovery-token-ca-cert-hash sha256:44aba1ef82b6b34c40fe748a9c2cd321be91aa3c22dd23e706001b65affb9dc9

然后直接依次去其余work节点执行这条命令即可，但是我们这边的cri接口不上containerd，所以真实命令应该为

kubeadm join 10.10.21.178:6443 --token bnxg1m.6y89w1fsz73ztc34 --discovery-token-ca-cert-hash sha256:44aba1ef82b6b34c40fe748a9c2cd321be91aa3c22dd23e706001b65affb9dc9 --cri-socket unix:///run/cri-dockerd.sock

一定要在结尾加上*–cri-socket unix:///run/cri-dockerd.sock*

执行完成之后再查询node信息

root@master1:~# kubectl get node
NAME             STATUS   ROLES           AGE     VERSION
master1      Ready    control-plane   6d4h    v1.25.2
node1      Ready    <none>          5d12h   v1.25.2
node2        Ready    <none>          5d12h   v1.25.2
node3          Ready    <none>          5d12h   v1.25.2

扩容控制节点

扩容控制节点需要更新证书，正常命令应该如下

root@master1:~# kubeadm init phase upload-certs --upload-certs 
Found multiple CRI endpoints on the host. Please define which one do you wish to use by setting the 'criSocket' field in the kubeadm configuration file: unix:///var/run/containerd/containerd.sock, unix:///var/run/cri-dockerd.sock
To see the stack trace of this error execute with --v=5 or higher

可以看到我这边提示cri接口异常，然后我尝试加上

root@master1:~# kubeadm init phase upload-certs --upload-certs --cri-socket  unix:///var/run/cri-dockerd.sock
unknown flag: --cri-socket
To see the stack trace of this error execute with --v=5 or higher

还是报错

后来发现这其实主要原因是因为使用了外部etcd导致的，因此可以按如下方式更新证书,这里卡了我蛮久的，如果有遇到相同问题的可以试一下，init.yaml就是在kubeadm初始化的时候的那个文件

root@master1:~# kubeadm init phase upload-certs --upload-certs --config init.yaml 
[upload-certs] Storing the certificates in Secret "kubeadm-certs" in the "kube-system" Namespace
[upload-certs] Using certificate key:
a618b3a050dcc91d7304ca7f3ddf2f02f2825528d9801f9d1bd8d3e742898744

接下来的操作就比较简单了，将之前生成的token和这次生成的key拼接起来即可

root@master2:~# kubeadm join 10.10.21.178:6443 --token q5k1so.r9r6vg7lsj3zzec1 --discovery-token-ca-cert-hash sha256:44aba1ef82b6b34c40fe748a9c2cd321be91aa3c22dd23e706001b65affb9dc9  --control-plane --certificate-key 509228b6886d23b3f5d7e64d8d6d2b74429e9bf136494838e296a4f1b0e89c46 --cri-socket unix:///run/cri-dockerd.sock
#这里的token和key一定要和自己的对应好，我这边因为复制了太多次所以可能不是完全对应的
#此外，结尾也一定要加上 --cri-socket unix:///run/cri-dockerd.sock指定cri接口，否则会有报错

如果出现类似这种报错

unable to add a new control plane instance to a cluster that doesn’t have a stable controlPlaneEndpoint address

这是因为kubeadm-config 的configmap文件没有声明controlPlaneEndpoint，需要执行如下操作

kubectl edit cm kubeadm-config -n kube-system

在这行加入对应的controlPlaneEndpoint，填写对应的vip地址

在其他两个master节点依次执行命令等待初始化完成之后，master扩容就完成了

root@master2:~# kubectl get nodes,cs
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME                  STATUS   ROLES           AGE     VERSION
node/master1   Ready    control-plane   6d4h    v1.25.2
node/master2   Ready    control-plane   5d2h    v1.25.2
node/master3   Ready    control-plane   5d2h    v1.25.2
node/node1     Ready    <none>          5d12h   v1.25.2
node/node2     Ready    <none>          5d12h   v1.25.2
node/node3     Ready    <none>          5d12h   v1.25.2
NAME                                 STATUS    MESSAGE             ERROR
componentstatus/scheduler            Healthy   ok                  
componentstatus/controller-manager   Healthy   ok                  
componentstatus/etcd-2               Healthy   {"health":"true"}   
componentstatus/etcd-1               Healthy   {"health":"true"}   
componentstatus/etcd-0               Healthy   {"health":"true"}   
root@master2:~# kubectl cluster-info 
Kubernetes control plane is running at https://10.10.21.178:6443
CoreDNS is running at https://10.10.21.178:6443/api/v1/namespaces/kube-system/services/kube-dns:dns/proxy
To further debug and diagnose cluster problems, use 'kubectl cluster-info dump'.

到这里三主三从的k8s集群就部署完毕了，如果还需要部署dashboard可以参考之前的博客

修改k8s的service中nodeport端口范围

在 Kubernetes 集群中，NodePort 默认范围是 30000-32767，但是我们也可以进行修改

root@master2:~# cat /etc/kubernetes/manifests/kube-apiserver.yaml 
apiVersion: v1
kind: Pod
metadata:
  annotations:
    kubeadm.kubernetes.io/kube-apiserver.advertise-address.endpoint: 10.10.21.172:6443
  creationTimestamp: null
  labels:
    component: kube-apiserver
    tier: control-plane
  name: kube-apiserver
  namespace: kube-system
spec:
  containers:
  - command:
    - kube-apiserver
    - --advertise-address=10.10.21.172
    - --allow-privileged=true
    - --authorization-mode=Node,RBAC
    - --client-ca-file=/etc/kubernetes/pki/ca.crt
    - --enable-admission-plugins=NodeRestriction
    - --enable-bootstrap-token-auth=true
    - --etcd-cafile=/etc/etcd/pki/ca.pem
    - --etcd-certfile=/etc/etcd/pki/client.pem
    - --etcd-keyfile=/etc/etcd/pki/client-key.pem
    - --etcd-servers=https://10.10.21.170:2379,https://10.10.21.172:2379,https://10.10.21.175:2379
    - --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt
    - --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key
    - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
    - --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt
    - --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key
    - --requestheader-allowed-names=front-proxy-client
    - --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt
    - --requestheader-extra-headers-prefix=X-Remote-Extra-
    - --requestheader-group-headers=X-Remote-Group
    - --requestheader-username-headers=X-Remote-User
    - --secure-port=6443
    - --service-account-issuer=https://kubernetes.default.svc.cluster.local
    - --service-account-key-file=/etc/kubernetes/pki/sa.pub
    - --service-account-signing-key-file=/etc/kubernetes/pki/sa.key
    - --service-cluster-ip-range=10.100.0.0/16
    - --service-node-port-range=30000-50000     #这里定义的就是端口范围，如果没有的话也可以新增
    - --tls-cert-file=/etc/kubernetes/pki/apiserver.crt
    - --tls-private-key-file=/etc/kubernetes/pki/apiserver.key
    image: registry.aliyuncs.com/google_containers/kube-apiserver:v1.25.2
    imagePullPolicy: IfNotPresent
    livenessProbe:
      failureThreshold: 8
      httpGet:
        host: 10.10.21.172
        path: /livez
        port: 6443
        scheme: HTTPS
      initialDelaySeconds: 10
      periodSeconds: 10
      timeoutSeconds: 15
    name: kube-apiserver
    readinessProbe:
      failureThreshold: 3
      httpGet:
        host: 10.10.21.172
        path: /readyz
        port: 6443
        scheme: HTTPS
      periodSeconds: 1
      timeoutSeconds: 15
    resources:
      requests:
        cpu: 250m
    startupProbe:
      failureThreshold: 24
      httpGet:
        host: 10.10.21.172
        path: /livez
        port: 6443
        scheme: HTTPS
      initialDelaySeconds: 10
      periodSeconds: 10
      timeoutSeconds: 15
    volumeMounts:
    - mountPath: /etc/ssl/certs
      name: ca-certs
      readOnly: true
    - mountPath: /etc/ca-certificates
      name: etc-ca-certificates
      readOnly: true
    - mountPath: /etc/pki
      name: etc-pki
      readOnly: true
    - mountPath: /etc/etcd/pki
      name: etcd-certs-0
      readOnly: true
    - mountPath: /etc/kubernetes/pki
      name: k8s-certs
      readOnly: true
    - mountPath: /usr/local/share/ca-certificates
      name: usr-local-share-ca-certificates
      readOnly: true
    - mountPath: /usr/share/ca-certificates
      name: usr-share-ca-certificates
      readOnly: true
  hostNetwork: true
  priorityClassName: system-node-critical
  securityContext:
    seccompProfile:
      type: RuntimeDefault
  volumes:
  - hostPath:
      path: /etc/ssl/certs
      type: DirectoryOrCreate
    name: ca-certs
  - hostPath:
      path: /etc/ca-certificates
      type: DirectoryOrCreate
    name: etc-ca-certificates
  - hostPath:
      path: /etc/pki
      type: DirectoryOrCreate
    name: etc-pki
  - hostPath:
      path: /etc/etcd/pki
      type: DirectoryOrCreate
    name: etcd-certs-0
  - hostPath:
      path: /etc/kubernetes/pki
      type: DirectoryOrCreate
    name: k8s-certs
  - hostPath:
      path: /usr/local/share/ca-certificates
      type: DirectoryOrCreate
    name: usr-local-share-ca-certificates
  - hostPath:
      path: /usr/share/ca-certificates
      type: DirectoryOrCreate
    name: usr-share-ca-certificates
status: {}

修改完毕之后将这个文件传输至其余matser节点并delete原来的apiserver对应的pod

root@master1:~# for i in  master2 master3;do scp /etc/kubernetes/manifests/kube-apiserver.yaml  $i:/etc/kubernetes/manifests/kube-apiserver.yaml;done #传输给其他节点
root@master1:~# kubectl -n kube-system delete pod kube-apiserver-master2
pod "kube-apiserver-master2" deleted
root@master1:~# kubectl -n kube-system delete pod kube-apiserver-master3.hu.org
pod "kube-apiserver-master3" deleted
root@master1:~# kubectl -n kube-system delete pod kube-apiserver-master1.hu.org
pod "kube-apiserver-master1" deleted
root@master1:~# kubectl -n kube-system describe pod kube-apiserver-master1.hu.org |grep -i service
      --service-account-issuer=https://kubernetes.default.svc.cluster.local
      --service-account-key-file=/etc/kubernetes/pki/sa.pub
      --service-account-signing-key-file=/etc/kubernetes/pki/sa.key
      --service-cluster-ip-range=10.100.0.0/16
      --service-node-port-range=30000-50000

配置k8s使用ipvs进行流量转发

k8s的kube-proxy支持iptables、ipvs 模式，默认是iptables 模式

root@master1:~# apt-get update
root@master1:~# apt-get install -y ipvsadm ipset sysstat conntrack libseccomp-dev #安装ipvs
root@master1:~# uname -r
5.4.0-126-generic
root@master1:~# modprobe -- ip_vs
root@master1:~# modprobe -- ip_vs_rr
root@master1:~# modprobe -- ip_vs_wrr
root@master1:~# modprobe -- ip_vs_sh
root@master1:~# modprobe -- nf_conntrack
root@master1:~# cat >> /etc/modules-load.d/ipvs.conf <<EOF
ip_vs
ip_vs_lc
ip_vs_wlc
ip_vs_rr
ip_vs_wrr
ip_vs_lblc
ip_vs_lblcr
ip_vs_dh
ip_vs_sh
ip_vs_fo
ip_vs_nq
ip_vs_sed
ip_vs_ftp
ip_vs_sh
nf_conntrack #内核小于4.18，把这行改成nf_conntrack_ipv4
ip_tables
ip_set
xt_set
ipt_set
ipt_rpfilter
ipt_REJECT
ipip
EOF
root@master1:~# systemctl enable --now systemd-modules-load.service
The unit files have no installation config (WantedBy=, RequiredBy=, Also=,
Alias= settings in the [Install] section, and DefaultInstance= for template
units). This means they are not meant to be enabled using systemctl.
Possible reasons for having this kind of units are:
• A unit may be statically enabled by being symlinked from another unit's
  .wants/ or .requires/ directory.
• A unit's purpose may be to act as a helper for some other unit which has
  a requirement dependency on it.
• A unit may be started when needed via activation (socket, path, timer,
  D-Bus, udev, scripted systemctl call, ...).
• In case of template units, the unit is meant to be enabled with some
  instance name specified.
root@master1:~# systemctl status systemd-modules-load.service
● systemd-modules-load.service - Load Kernel Modules
     Loaded: loaded (/lib/systemd/system/systemd-modules-load.service; static; vendor preset: enabled)
     Active: active (exited) since Wed 2022-10-12 03:53:34 UTC; 5 days ago
       Docs: man:systemd-modules-load.service(8)
             man:modules-load.d(5)
   Main PID: 377 (code=exited, status=0/SUCCESS)
      Tasks: 0 (limit: 4640)
     Memory: 0B
     CGroup: /system.slice/systemd-modules-load.service
root@master1:~# lsmod |grep ip_vs
ip_vs_sh               16384  0
ip_vs_wrr              16384  0
ip_vs_rr               16384  10
ip_vs                 155648  16 ip_vs_rr,ip_vs_sh,ip_vs_wrr
nf_conntrack          139264  6 xt_conntrack,nf_nat,xt_nat,nf_conntrack_netlink,xt_MASQUERADE,ip_vs
nf_defrag_ipv6         24576  2 nf_conntrack,ip_vs
libcrc32c              16384  6 nf_conntrack,nf_nat,btrfs,xfs,raid456,ip_vs
#确定已经生效

修改kube-proxy配置

root@master1:~# kubectl edit configmaps kube-proxy -n kube-system
#在mode这行修改，后面加上ipvs：   *mode: "ipvs"*

然后删除kube-proxy对应的pod让他重新生成，重新生成完成之后可以进行查询检查

root@master1:~# kubectl -n kube-system logs kube-proxy-xkkhp 
I1013 03:09:02.586507       1 node.go:163] Successfully retrieved node IP: 10.10.21.173
I1013 03:09:02.586554       1 server_others.go:138] "Detected node IP" address="10.10.21.173"
I1013 03:09:02.602042       1 server_others.go:269] "Using ipvs Proxier"   #这里指明用的ipvs
I1013 03:09:02.602175       1 server_others.go:271] "Creating dualStackProxier for ipvs"
I1013 03:09:02.602192       1 server_others.go:501] "Detect-local-mode set to ClusterCIDR, but no IPv6 cluster CIDR defined, , defaulting to no-op detect-local for IPv6"
I1013 03:09:02.602873       1 proxier.go:449] "IPVS scheduler not specified, use rr by default"
I1013 03:09:02.603004       1 proxier.go:449] "IPVS scheduler not specified, use rr by default"
I1013 03:09:02.603023       1 ipset.go:113] "Ipset name truncated" ipSetName="KUBE-6-LOAD-BALANCER-SOURCE-CIDR" truncatedName="KUBE-6-LOAD-BALANCER-SOURCE-CID"
I1013 03:09:02.603030       1 ipset.go:113] "Ipset name truncated" ipSetName="KUBE-6-NODE-PORT-LOCAL-SCTP-HASH" truncatedName="KUBE-6-NODE-PORT-LOCAL-SCTP-HAS"
I1013 03:09:02.603166       1 server.go:661] "Version info" version="v1.25.2"
I1013 03:09:02.603177       1 server.go:663] "Golang settings" GOGC="" GOMAXPROCS="" GOTRACEBACK=""
I1013 03:09:02.608838       1 conntrack.go:52] "Setting nf_conntrack_max" nf_conntrack_max=131072
I1013 03:09:02.609416       1 config.go:317] "Starting service config controller"
I1013 03:09:02.609429       1 shared_informer.go:255] Waiting for caches to sync for service config
I1013 03:09:02.609461       1 config.go:444] "Starting node config controller"
I1013 03:09:02.609464       1 shared_informer.go:255] Waiting for caches to sync for node config
I1013 03:09:02.609479       1 config.go:226] "Starting endpoint slice config controller"
I1013 03:09:02.609482       1 shared_informer.go:255] Waiting for caches to sync for endpoint slice config
I1013 03:09:02.709868       1 shared_informer.go:262] Caches are synced for endpoint slice config
I1013 03:09:02.709923       1 shared_informer.go:262] Caches are synced for node config
I1013 03:09:02.711053       1 shared_informer.go:262] Caches are synced for service config
I1017 15:06:11.940342       1 graceful_termination.go:102] "Removed real server from graceful delete real server list" realServer="10.100.0.1:443/TCP/10.10.21.172:6443"
root@master1:~# kubectl -n kube-system logs kube-proxy-xkkhp  |grep -i ipvs
I1013 03:09:02.602042       1 server_others.go:269] "Using ipvs Proxier"
I1013 03:09:02.602175       1 server_others.go:271] "Creating dualStackProxier for ipvs"
I1013 03:09:02.602873       1 proxier.go:449] "IPVS scheduler not specified, use rr by default"
I1013 03:09:02.603004       1 proxier.go:449] "IPVS scheduler not specified, use rr by default"

基于docker部署高可用Kubernetes1.25.2（下）

五、开始安装

所有k8s节点安装kubeadm、kubectl、kubelet

所有k8s节点安装cri-dockerd

在master1初始化

安装CNI插件flannel

扩容work节点

扩容控制节点

修改k8s的service中nodeport端口范围

配置k8s使用ipvs进行流量转发

热门文章

最新文章

相关课程

相关电子书

相关实验场景

推荐镜像

基于docker部署高可用Kubernetes1.25.2（下）

五、 开始安装

所有k8s节点安装kubeadm、kubectl、kubelet

所有k8s节点安装cri-dockerd

在master1初始化

安装CNI插件flannel

扩容work节点

扩容控制节点

修改k8s的service中nodeport端口范围

配置k8s使用ipvs进行流量转发

热门文章

最新文章

相关课程

相关电子书

相关实验场景

推荐镜像

五、开始安装