如何在自建集群中部署cloud-provider-alibaba-cloud

前提条件

• Kubernetes集群已经部署完毕
• Master节点已经添加node-role.kubernetes.io/master: "" 标签

部署Cloud Controller Manager

配置Kubelet

为Kubelet配置ProviderID (需要为集群中所有节点执行此操作)

META_EP=http://100.100.100.200/latest/meta-data
echo `curl -s $META_EP/region-id`.`curl -s $META_EP/instance-id`
## for example
cn-shanghai.i-ufxxxxxxxxkb6xxo

为Node添加ProviderID

kubectl patch node xxxx -p '{"spec":{"providerID":"cn-shanghai.i-ufxxxxxxxxkb6xxo"}}'

设置AK

1）获取AK信息

2）对AK信息进行base64加密

echo -n "xxxxxxxxxxx" | base64

3）创建ConfigMap

apiVersion: v1
kind: ConfigMap
metadata:
  name: cloud-config
  namespace: kube-system
data:
  cloud-config.conf: |-
    {
        "Global": {
            "accessKeyID": "$Base64AccessKeyID",
            "accessKeySecret": "$Base64AccessKeySecret"
        }
    }

创建cloud-controller-manager.conf （需要在所有Master节点上执行）

将下述文件保存为 /etc/kubernetes/cloud-controller-manager.conf

其中， $CA_DATA 为 cat /etc/kubernetes/pki/ca.crt|base64 -w 0 的执行结果，

server地址为集群的apiserver地址。

kind: Config
contexts:
- context:
    cluster: kubernetes
    user: system:cloud-controller-manager
  name: system:cloud-controller-manager@kubernetes
current-context: system:cloud-controller-manager@kubernetes
users:
- name: system:cloud-controller-manager
  user:
    tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
apiVersion: v1
clusters:
- cluster:
    certificate-authority-data: $CA_DATA
    server: https://192.168.1.76:6443
  name: kubernetes

创建cloud-controller-manager

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: system:cloud-controller-manager
rules:
  - apiGroups:
      - ""
    resources:
      - persistentvolumes
      - services
      - secrets
      - endpoints
      - serviceaccounts
    verbs:
      - get
      - list
      - watch
      - create
      - update
      - patch
  - apiGroups:
      - ""
    resources:
      - nodes
    verbs:
      - get
      - list
      - watch
      - delete
      - patch
      - update
  - apiGroups:
      - ""
    resources:
      - services/status
    verbs:
      - update
      - patch
  - apiGroups:
      - ""
    resources:
      - nodes/status
    verbs:
      - patch
      - update
  - apiGroups:
      - ""
    resources:
      - events
      - endpoints
    verbs:
      - create
      - patch
      - update
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: cloud-controller-manager
  namespace: kube-system
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: system:cloud-controller-manager
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:cloud-controller-manager
subjects:
- kind: ServiceAccount
  name: cloud-controller-manager
  namespace: kube-system
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: system:shared-informers
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:cloud-controller-manager
subjects:
- kind: ServiceAccount
  name: shared-informers
  namespace: kube-system
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: system:cloud-node-controller
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:cloud-controller-manager
subjects:
- kind: ServiceAccount
  name: cloud-node-controller
  namespace: kube-system
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: system:pvl-controller
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:cloud-controller-manager
subjects:
- kind: ServiceAccount
  name: pvl-controller
  namespace: kube-system
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: system:route-controller
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:cloud-controller-manager
subjects:
- kind: ServiceAccount
  name: route-controller
  namespace: kube-system
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  labels:
    app: cloud-controller-manager
    tier: control-plane
  name: cloud-controller-manager
  namespace: kube-system
spec:
  selector:
    matchLabels:
      app: cloud-controller-manager
      tier: control-plane
  template:
    metadata:
      labels:
        app: cloud-controller-manager
        tier: control-plane
      annotations:
        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
      serviceAccountName: cloud-controller-manager
      tolerations:
      - effect: NoSchedule
        operator: Exists
        key: node-role.kubernetes.io/master
      - effect: NoSchedule
        operator: Exists
        key: node.cloudprovider.kubernetes.io/uninitialized
      nodeSelector:
         node-role.kubernetes.io/master: ""
      containers:
      - command:
        -  /cloud-controller-manager
        - --kubeconfig=/etc/kubernetes/cloud-controller-manager.conf
        - --address=127.0.0.1
        - --allow-untagged-cloud=true
        - --leader-elect=true
        - --cloud-provider=alicloud
        - --use-service-account-credentials=true
        - --cloud-config=/etc/kubernetes/config/cloud-config.conf
        ## 配置路由信息（Flannel网络插件）
        - --configure-cloud-routes=true
        - --allocate-node-cidrs=true
        - --route-reconciliation-period=3m
         # 替换为集群的podcidr
        - --cluster-cidr=172.20.0.0/16
        # 配置路由信息（Terway网络插件）
        #- --configure-cloud-routes=false
        #- --allocate-node-cidrs=false
        image: registry-vpc.${your-region}.aliyuncs.com/acs/cloud-controller-manager-amd64:v1.9.3.339-g9830b58-aliyun
        livenessProbe:
          failureThreshold: 8
          httpGet:
            host: 127.0.0.1
            path: /healthz
            port: 10258
            scheme: HTTP
          initialDelaySeconds: 15
          timeoutSeconds: 15
        name: cloud-controller-manager
        resources:
          requests:
            cpu: 200m
        volumeMounts:
        - mountPath: /etc/kubernetes/
          name: k8s
        - mountPath: /etc/ssl/certs
          name: certs
        - mountPath: /etc/pki
          name: pki
        - mountPath: /etc/kubernetes/config
          name: cloud-config
      hostNetwork: true
      volumes:
      - hostPath:
          path: /etc/kubernetes
        name: k8s
      - hostPath:
          path: /etc/ssl/certs
        name: certs
      - hostPath:
          path: /etc/pki
        name: pki
      - configMap:
          defaultMode: 420
          items:
          - key: cloud-config.conf
            path: cloud-config.conf
          name: cloud-config
        name: cloud-config

等待Pod running

kubectl -nkube-system get po|grep cloud-controller-manager

验证

1）创建deploy

kubectl create deploy nginx --image=nginx

2) 创建LoadBalancer类型svc

kubectl expose deploy nginx --name=test --port=80 --type=LoadBalancer