前言

KubeSphere? 是經 CNCF 認證的 Kubernetes 主流開源發行版之一，在 Kubernetes 之上提供多種以容器為資源載體的業務功能模塊，如多租戶管理、集群運維、應用管理、DevOps、微服務治理等功能，

最近微服務，要部署到k8s，采用KubeSphere應用為中心的容器管理平臺，于是搗鼓怎樣去部署，第一次部署成功，好像不穩定，再次恢復四臺服務器鏡像，重新部署，其中遇到很多的問題及挫折，在此記錄一下，以供大家參考，

準備服務器

master：172.16.7.12 centos7.5 + 8cpu + 16G（記憶體） + 20G（/） + 200G ( data）
node5：172.16.7.15 centos7.5 + 8cpu + 16G（記憶體） + 20G（/） + 1T ( data）
node3：172.16.7.16 centos7.5 + 8cpu + 16G（記憶體） + 20G（/） + 1T ( data）
node4：172.16.7.17 centos7.5 + 8cpu + 16G（記憶體） + 20G（/） + 1T ( data）

系統優化及安裝準備軟體

系統自動優化

參考“ansible安裝部署CDH集群,與手動安裝部署CDH集群，及CM配置和用戶權限配置”，先進行對系統進行優化：

sh deploy_robot.sh init_ssh
sh deploy_robot.sh init_sys

關于deploy_robot.sh的腳本在https://github.com/fleapx/cdh-deploy-robot.git

ansible工具軟體安裝

# 控制機器上安裝ansible
yum install -y ansible

配置修改 /etc/ansible/hosts ，對需要管理的主機進行配置，

[all]
172.16.7.12  master
172.16.7.15  node5
172.16.7.16  node3
172.16.7.17  node4

默認配置，需要修改編輯 /etc/ansible/ansible.cfg，

# uncomment this to disable SSH key host checking
host_key_checking = False

其他系統軟體安裝

ansible all -m shell -a "wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo"
ansible all -m shell -a "sed -i 's/^.*aliyuncs*/#&/g' /etc/yum.repos.d/CentOS-Base.repo"
ansible all -m shell -a "wget -O /etc/yum.repos.d/epel.repo https://mirrors.aliyun.com/repo/epel-7.repo"
ansible all -m shell -a "yum -y install ebtables socat ipset conntrack nfs-utils rpcbind"
ansible all -m shell -a "yum install -y vim wget yum-utils device-mapper-persistent-data lvm2"

集群時間校準

ansible all -m shell -a "yum install chrony -y"
ansible all -m shell -a "systemctl  start chronyd"
ansible all -m shell -a "sed -i -e '/^server/s/^/#/' -e '1a server ntp.aliyun.com iburst' /etc/chrony.conf"
ansible all -m shell -a "systemctl  restart chronyd"
ansible all -m shell -a "timedatectl set-timezone Asia/Shanghai"

系統其他優化

ansible all -m shell -a "echo '* soft nofile 655360' >> /etc/security/limits.conf"
ansible all -m shell -a "echo '* hard nofile 655360' >> /etc/security/limits.conf"
ansible all -m shell -a "echo '* soft nproc 655360'  >> /etc/security/limits.conf"
ansible all -m shell -a "echo '* hard nproc 655360'  >> /etc/security/limits.conf"
ansible all -m shell -a "echo '* soft  memlock  unlimited'  >> /etc/security/limits.conf"
ansible all -m shell -a "echo '* hard memlock  unlimited'  >> /etc/security/limits.conf"
ansible all -m shell -a "echo 'DefaultLimitNOFILE=1024000'  >> /etc/systemd/system.conf" 
ansible all -m shell -a "echo 'DefaultLimitNPROC=1024000'  >> /etc/systemd/system.conf"

開放所有埠

ansible all -m shell -a "iptables -P INPUT ACCEPT"
ansible all -m shell -a "iptables -P FORWARD ACCEPT"
ansible all -m shell -a "iptables -P OUTPUT ACCEPT"
ansible all -m shell -a "iptables -F"

安裝docker

CentOS 7系統下docker安裝，及配置阿里云加速，及所有配置請參加此文，

ansible all -m shell -a "yum remove docker  docker-common docker-selinux docker-engine"
ansible all -m shell -a "yum -y install docker-ce-19.03.8-3.el7"

安裝k8s的鏡像源

tee /etc/yum.repos.d/kubernetes.repo <<-'EOF'
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

安裝kubernetes-v1.18.6和kubesphere-v3.0

官方https://kubesphere.com.cn/多節點安裝地址：https://kubesphere.com.cn/docs/quick-start/all-in-one-on-linux/

下載kk安裝檔案

# 在國內先添加一個環境變數
export KKZONE=cn
curl -sfL https://get-kk.kubesphere.io | VERSION=v1.0.1 sh -

然后，使kk成為全域可執行檔案

mv ./kk /usr/local/bin

生成多節點集群的配置

# 創建一個組態檔模版
kk create config --with-kubernetes v1.18.6 --with-kubesphere v3.0.0 -f ./config-kubesphere.yaml

修改組態檔

修改組態檔config-kubesphere.yaml

apiVersion: kubekey.kubesphere.io/v1alpha1
kind: Cluster
metadata:
  name: sample
spec:
  hosts:
  - {name: master, address: 172.16.7.12, internalAddress: 172.16.7.12, user: root, password: azdebug_it}
  - {name: node3, address: 172.16.7.16, internalAddress: 172.16.7.16, user: root, password: azdebug_it}
  - {name: node4, address: 172.16.7.17, internalAddress: 172.16.7.17, user: root, password: azdebug_it}
  - {name: node5, address: 172.16.7.15, internalAddress: 172.16.7.15, user: root, password: azdebug_it}
  roleGroups:
    etcd:
    - node3
    master:
    - master
    worker:
    - node4
    - node3
    - node5
  controlPlaneEndpoint:
    domain: lb.kubesphere.local
    address: "172.16.7.12"
    port: "6443"
  kubernetes:
    version: v1.18.6
    imageRepo: kubesphere
    clusterName: cluster.local
  network:
    plugin: calico
    kubePodsCIDR: 10.233.64.0/18
    kubeServiceCIDR: 10.233.0.0/18
  registry:
    registryMirrors: []
    insecureRegistries: []
    #privateRegistry: dockerhub.kubekey.local
  addons: []


---
apiVersion: installer.kubesphere.io/v1alpha1
kind: ClusterConfiguration
metadata:
  name: ks-installer
  namespace: kubesphere-system
  labels:
    version: v3.0.0
spec:
  local_registry: ""
  persistence:
    storageClass: ""
  authentication:
    jwtSecret: ""
  etcd:
    monitoring: true
    endpointIps: 172.16.7.16
    port: 2379
    tlsEnable: true
  common:
    es:
      elasticsearchDataVolumeSize: 20Gi
      elasticsearchMasterVolumeSize: 4Gi
      elkPrefix: logstash
      logMaxAge: 7
    mysqlVolumeSize: 20Gi
    minioVolumeSize: 20Gi
    etcdVolumeSize: 20Gi
    openldapVolumeSize: 2Gi
    redisVolumSize: 2Gi
  console:
    enableMultiLogin: true  # enable/disable multi login
    port: 30880
  alerting:
    enabled: true
  auditing:
    enabled: true
  devops:
    enabled: true
    jenkinsMemoryLim: 5Gi
    jenkinsMemoryReq: 1500Mi
    jenkinsVolumeSize: 8Gi
    jenkinsJavaOpts_Xms: 1024m
    jenkinsJavaOpts_Xmx: 1024m
    jenkinsJavaOpts_MaxRAM: 2g
  events:
    enabled: true
    ruler:
      enabled: true
      replicas: 2
  logging:
    enabled: true
    logsidecarReplicas: 2
  metrics_server:
    enabled: true
  monitoring:
    prometheusMemoryRequest: 400Mi
    prometheusVolumeSize: 20Gi
  multicluster:
    clusterRole: none  # host | member | none
  networkpolicy:
    enabled: true
  notification:
    enabled: true
  openpitrix:
    enabled: true
  servicemesh:
    enabled: true

執行及安裝kubernetes v1.18.6和kubesphere v3.0.0

# 修改組態檔，添加上節點資訊（節點名稱，ip等）
kk create cluster -f ./config-kubesphere.yaml

正確執行結果

[root@master ~]# sudo -E /bin/sh -c "/usr/local/bin/kubeadm init --config=/etc/kubernetes/kubeadm-config.yaml"
W0105 22:45:15.009277   22248 utils.go:26] The recommended value for "clusterDNS" in "KubeletConfiguration" is: [10.233.0.10]; the provided value is: [169.254.25.10]
W0105 22:45:15.009521   22248 configset.go:202] WARNING: kubeadm cannot validate component configs for API groups [kubelet.config.k8s.io kubeproxy.config.k8s.io]
[init] Using Kubernetes version: v1.18.6
[preflight] Running pre-flight checks
error execution phase preflight: [preflight] Some fatal errors occurred:
	[ERROR ExternalEtcdVersion]: couldn't load external etcd's certificate and key pair /etc/ssl/etcd/ssl/node-node3.pem, /etc/ssl/etcd/ssl/node-node3-key.pem: open /etc/ssl/etcd/ssl/node-node3.pem: no such file or directory
	[ERROR ExternalEtcdClientCertificates]: /etc/ssl/etcd/ssl/node-node3.pem doesn't exist
	[ERROR ExternalEtcdClientCertificates]: /etc/ssl/etcd/ssl/node-node3-key.pem doesn't exist
[preflight] If you know what you are doing, you can make a check non-fatal with `--ignore-preflight-errors=...`
To see the stack trace of this error execute with --v=5 or higher
[root@master ~]# cd /home/k
k8s-script/                                       kubesphere-all-v3.0.0-offline-linux-amd64.tar.gz  
[root@master ~]# cd /home/k
k8s-script/                                       kubesphere-all-v3.0.0-offline-linux-amd64.tar.gz  
[root@master ~]# cd /home/k8s-script/
[root@master k8s-script]# export KKZONE=cn
[root@master k8s-script]# ./kk create cluster -f ./k8s-config.yaml
+--------+------+------+---------+----------+-------+-------+-----------+--------+------------+-------------+------------------+--------------+
| name   | sudo | curl | openssl | ebtables | socat | ipset | conntrack | docker | nfs client | ceph client | glusterfs client | time         |
+--------+------+------+---------+----------+-------+-------+-----------+--------+------------+-------------+------------------+--------------+
| node5  | y    | y    | y       | y        | y     | y     | y         | y      | y          |             |                  | CST 22:55:35 |
| node4  | y    | y    | y       | y        | y     | y     | y         | y      | y          |             |                  | CST 22:55:35 |
| node3  | y    | y    | y       | y        | y     | y     | y         | y      | y          |             |                  | CST 22:55:35 |
| master | y    | y    | y       | y        | y     | y     | y         | y      | y          |             |                  | CST 22:55:35 |
+--------+------+------+---------+----------+-------+-------+-----------+--------+------------+-------------+------------------+--------------+

This is a simple check of your environment.
Before installation, you should ensure that your machines meet all requirements specified at
https://github.com/kubesphere/kubekey#requirements-and-recommendations

Continue this installation? [yes/no]: yes
INFO[22:57:10 CST] Downloading Installation Files               
INFO[22:57:10 CST] Downloading kubeadm ...                      
INFO[22:57:10 CST] Downloading kubelet ...                      
INFO[22:57:10 CST] Downloading kubectl ...                      
INFO[22:57:11 CST] Downloading helm ...                         
INFO[22:57:11 CST] Downloading kubecni ...                      
INFO[22:57:11 CST] Configurating operating system ...           
[node5 172.16.7.15] MSG:
vm.swappiness = 1
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_tw_recycle = 1
net.ipv4.tcp_keepalive_time = 1200
net.ipv4.ip_local_port_range = 10000 65000
net.ipv4.tcp_max_syn_backlog = 8192
net.ipv4.tcp_max_tw_buckets = 5000
fs.file-max = 655350
net.ipv4.route.gc_timeout = 100
net.ipv4.tcp_syn_retries = 1
net.ipv4.tcp_synack_retries = 1
net.core.netdev_max_backlog = 16384
net.ipv4.tcp_max_orphans = 16384
net.ipv4.tcp_fin_timeout = 2
net.core.somaxconn = 32768
kernel.threads-max = 655360
kernel.pid_max = 655360
vm.max_map_count = 393210
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-arptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_local_reserved_ports = 30000-32767
[node4 172.16.7.17] MSG:
vm.swappiness = 1
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_tw_recycle = 1
net.ipv4.tcp_keepalive_time = 1200
net.ipv4.ip_local_port_range = 10000 65000
net.ipv4.tcp_max_syn_backlog = 8192
net.ipv4.tcp_max_tw_buckets = 5000
fs.file-max = 655350
net.ipv4.route.gc_timeout = 100
net.ipv4.tcp_syn_retries = 1
net.ipv4.tcp_synack_retries = 1
net.core.netdev_max_backlog = 16384
net.ipv4.tcp_max_orphans = 16384
net.ipv4.tcp_fin_timeout = 2
net.core.somaxconn = 32768
kernel.threads-max = 655360
kernel.pid_max = 655360
vm.max_map_count = 393210
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-arptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_local_reserved_ports = 30000-32767
[master 172.16.7.12] MSG:
vm.swappiness = 1
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_tw_recycle = 1
net.ipv4.tcp_keepalive_time = 1200
net.ipv4.ip_local_port_range = 10000 65000
net.ipv4.tcp_max_syn_backlog = 8192
net.ipv4.tcp_max_tw_buckets = 5000
fs.file-max = 655350
net.ipv4.route.gc_timeout = 100
net.ipv4.tcp_syn_retries = 1
net.ipv4.tcp_synack_retries = 1
net.core.netdev_max_backlog = 16384
net.ipv4.tcp_max_orphans = 16384
net.ipv4.tcp_fin_timeout = 2
net.core.somaxconn = 32768
kernel.threads-max = 655360
kernel.pid_max = 655360
vm.max_map_count = 393210
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-arptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_local_reserved_ports = 30000-32767
[node3 172.16.7.16] MSG:
vm.swappiness = 1
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_tw_recycle = 1
net.ipv4.tcp_keepalive_time = 1200
net.ipv4.ip_local_port_range = 10000 65000
net.ipv4.tcp_max_syn_backlog = 8192
net.ipv4.tcp_max_tw_buckets = 5000
fs.file-max = 655350
net.ipv4.route.gc_timeout = 100
net.ipv4.tcp_syn_retries = 1
net.ipv4.tcp_synack_retries = 1
net.core.netdev_max_backlog = 16384
net.ipv4.tcp_max_orphans = 16384
net.ipv4.tcp_fin_timeout = 2
net.core.somaxconn = 32768
kernel.threads-max = 655360
kernel.pid_max = 655360
vm.max_map_count = 393210
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-arptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_local_reserved_ports = 30000-32767
INFO[22:57:25 CST] Installing docker ...                        
INFO[22:57:35 CST] Start to download images on all nodes        
[node5] Downloading image: kubesphere/pause:3.2
[master] Downloading image: kubesphere/pause:3.2
[node3] Downloading image: kubesphere/etcd:v3.3.12
[node4] Downloading image: kubesphere/pause:3.2
[node5] Downloading image: kubesphere/kube-proxy:v1.18.6
[node4] Downloading image: kubesphere/kube-proxy:v1.18.6
[node3] Downloading image: kubesphere/pause:3.2
[master] Downloading image: kubesphere/kube-apiserver:v1.18.6
[node5] Downloading image: coredns/coredns:1.6.9
[node4] Downloading image: coredns/coredns:1.6.9
[master] Downloading image: kubesphere/kube-controller-manager:v1.18.6
[node5] Downloading image: kubesphere/k8s-dns-node-cache:1.15.12
[node3] Downloading image: kubesphere/kube-proxy:v1.18.6
[master] Downloading image: kubesphere/kube-scheduler:v1.18.6
[node5] Downloading image: calico/kube-controllers:v3.15.1
[node3] Downloading image: coredns/coredns:1.6.9
[node4] Downloading image: kubesphere/k8s-dns-node-cache:1.15.12
[node3] Downloading image: kubesphere/k8s-dns-node-cache:1.15.12
[master] Downloading image: kubesphere/kube-proxy:v1.18.6
[node5] Downloading image: calico/cni:v3.15.1
[node4] Downloading image: calico/kube-controllers:v3.15.1
[master] Downloading image: coredns/coredns:1.6.9
[node5] Downloading image: calico/node:v3.15.1
[node4] Downloading image: calico/cni:v3.15.1
[node3] Downloading image: calico/kube-controllers:v3.15.1
[node3] Downloading image: calico/cni:v3.15.1
[master] Downloading image: kubesphere/k8s-dns-node-cache:1.15.12
[node5] Downloading image: calico/pod2daemon-flexvol:v3.15.1
[node4] Downloading image: calico/node:v3.15.1
[master] Downloading image: calico/kube-controllers:v3.15.1
[node3] Downloading image: calico/node:v3.15.1
[node4] Downloading image: calico/pod2daemon-flexvol:v3.15.1
[node3] Downloading image: calico/pod2daemon-flexvol:v3.15.1
[master] Downloading image: calico/cni:v3.15.1
[master] Downloading image: calico/node:v3.15.1
[master] Downloading image: calico/pod2daemon-flexvol:v3.15.1
INFO[23:01:26 CST] Generating etcd certs                        
INFO[23:01:32 CST] Synchronizing etcd certs                     
INFO[23:01:36 CST] Creating etcd service                        
INFO[23:01:49 CST] Starting etcd cluster                        
[node3 172.16.7.16] MSG:
Configuration file already exists
Waiting for etcd to start
INFO[23:01:58 CST] Refreshing etcd configuration                
INFO[23:01:59 CST] Backup etcd data regularly                   
INFO[23:02:00 CST] Get cluster status                           
[master 172.16.7.12] MSG:
Cluster will be created.
INFO[23:02:01 CST] Installing kube binaries                     
Push /home/k8s-script/kubekey/v1.18.6/amd64/kubeadm to 172.16.7.16:/tmp/kubekey/kubeadm   Done
Push /home/k8s-script/kubekey/v1.18.6/amd64/kubeadm to 172.16.7.15:/tmp/kubekey/kubeadm   Done
Push /home/k8s-script/kubekey/v1.18.6/amd64/kubeadm to 172.16.7.17:/tmp/kubekey/kubeadm   Done
Push /home/k8s-script/kubekey/v1.18.6/amd64/kubeadm to 172.16.7.12:/tmp/kubekey/kubeadm   Done
Push /home/k8s-script/kubekey/v1.18.6/amd64/kubelet to 172.16.7.16:/tmp/kubekey/kubelet   Done
Push /home/k8s-script/kubekey/v1.18.6/amd64/kubelet to 172.16.7.12:/tmp/kubekey/kubelet   Done
Push /home/k8s-script/kubekey/v1.18.6/amd64/kubelet to 172.16.7.15:/tmp/kubekey/kubelet   Done
Push /home/k8s-script/kubekey/v1.18.6/amd64/kubelet to 172.16.7.17:/tmp/kubekey/kubelet   Done
Push /home/k8s-script/kubekey/v1.18.6/amd64/kubectl to 172.16.7.16:/tmp/kubekey/kubectl   Done
Push /home/k8s-script/kubekey/v1.18.6/amd64/kubectl to 172.16.7.17:/tmp/kubekey/kubectl   Done
Push /home/k8s-script/kubekey/v1.18.6/amd64/kubectl to 172.16.7.15:/tmp/kubekey/kubectl   Done
Push /home/k8s-script/kubekey/v1.18.6/amd64/kubectl to 172.16.7.12:/tmp/kubekey/kubectl   Done
Push /home/k8s-script/kubekey/v1.18.6/amd64/helm to 172.16.7.12:/tmp/kubekey/helm   Done
Push /home/k8s-script/kubekey/v1.18.6/amd64/helm to 172.16.7.16:/tmp/kubekey/helm   Done
Push /home/k8s-script/kubekey/v1.18.6/amd64/helm to 172.16.7.15:/tmp/kubekey/helm   Done
Push /home/k8s-script/kubekey/v1.18.6/amd64/helm to 172.16.7.17:/tmp/kubekey/helm   Done
Push /home/k8s-script/kubekey/v1.18.6/amd64/cni-plugins-linux-amd64-v0.8.6.tgz to 172.16.7.15:/tmp/kubekey/cni-plugins-linux-amd64-v0.8.6.tgz   Done
Push /home/k8s-script/kubekey/v1.18.6/amd64/cni-plugins-linux-amd64-v0.8.6.tgz to 172.16.7.16:/tmp/kubekey/cni-plugins-linux-amd64-v0.8.6.tgz   Done
Push /home/k8s-script/kubekey/v1.18.6/amd64/cni-plugins-linux-amd64-v0.8.6.tgz to 172.16.7.17:/tmp/kubekey/cni-plugins-linux-amd64-v0.8.6.tgz   Done
Push /home/k8s-script/kubekey/v1.18.6/amd64/cni-plugins-linux-amd64-v0.8.6.tgz to 172.16.7.12:/tmp/kubekey/cni-plugins-linux-amd64-v0.8.6.tgz   Done
INFO[23:02:50 CST] Initializing kubernetes cluster              
[master 172.16.7.12] MSG:
W0105 23:02:51.587457   23847 utils.go:26] The recommended value for "clusterDNS" in "KubeletConfiguration" is: [10.233.0.10]; the provided value is: [169.254.25.10]
W0105 23:02:51.587685   23847 configset.go:202] WARNING: kubeadm cannot validate component configs for API groups [kubelet.config.k8s.io kubeproxy.config.k8s.io]
[init] Using Kubernetes version: v1.18.6
[preflight] Running pre-flight checks
[preflight] Pulling images required for setting up a Kubernetes cluster
[preflight] This might take a minute or two, depending on the speed of your internet connection
[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Starting the kubelet
[certs] Using certificateDir folder "/etc/kubernetes/pki"
[certs] Generating "ca" certificate and key
[certs] Generating "apiserver" certificate and key
[certs] apiserver serving cert is signed for DNS names [master kubernetes kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluster.local lb.kubesphere.local kubernetes kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluster.local localhost lb.kubesphere.local master master.cluster.local node3 node3.cluster.local node4 node4.cluster.local node5 node5.cluster.local] and IPs [10.233.0.1 172.16.7.12 127.0.0.1 172.16.7.12 172.16.7.16 172.16.7.17 172.16.7.15 10.233.0.1]
[certs] Generating "apiserver-kubelet-client" certificate and key
[certs] Generating "front-proxy-ca" certificate and key
[certs] Generating "front-proxy-client" certificate and key
[certs] External etcd mode: Skipping etcd/ca certificate authority generation
[certs] External etcd mode: Skipping etcd/server certificate generation
[certs] External etcd mode: Skipping etcd/peer certificate generation
[certs] External etcd mode: Skipping etcd/healthcheck-client certificate generation
[certs] External etcd mode: Skipping apiserver-etcd-client certificate generation
[certs] Generating "sa" key and public key
[kubeconfig] Using kubeconfig folder "/etc/kubernetes"
[kubeconfig] Writing "admin.conf" kubeconfig file
[kubeconfig] Writing "kubelet.conf" kubeconfig file
[kubeconfig] Writing "controller-manager.conf" kubeconfig file
[kubeconfig] Writing "scheduler.conf" kubeconfig file
[control-plane] Using manifest folder "/etc/kubernetes/manifests"
[control-plane] Creating static Pod manifest for "kube-apiserver"
W0105 23:03:00.466175   23847 manifests.go:225] the default kube-apiserver authorization-mode is "Node,RBAC"; using "Node,RBAC"
[control-plane] Creating static Pod manifest for "kube-controller-manager"
W0105 23:03:00.474746   23847 manifests.go:225] the default kube-apiserver authorization-mode is "Node,RBAC"; using "Node,RBAC"
[control-plane] Creating static Pod manifest for "kube-scheduler"
W0105 23:03:00.476002   23847 manifests.go:225] the default kube-apiserver authorization-mode is "Node,RBAC"; using "Node,RBAC"
[wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory "/etc/kubernetes/manifests". This can take up to 4m0s
[apiclient] All control plane components are healthy after 32.002873 seconds
[upload-config] Storing the configuration used in ConfigMap "kubeadm-config" in the "kube-system" Namespace
[kubelet] Creating a ConfigMap "kubelet-config-1.18" in namespace kube-system with the configuration for the kubelets in the cluster
[upload-certs] Skipping phase. Please see --upload-certs
[mark-control-plane] Marking the node master as control-plane by adding the label "node-role.kubernetes.io/master=''"
[mark-control-plane] Marking the node master as control-plane by adding the taints [node-role.kubernetes.io/master:NoSchedule]
[bootstrap-token] Using token: 6zsarg.gxg5eijglkupq85j
[bootstrap-token] Configuring bootstrap tokens, cluster-info ConfigMap, RBAC Roles
[bootstrap-token] configured RBAC rules to allow Node Bootstrap tokens to get nodes
[bootstrap-token] configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials
[bootstrap-token] configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token
[bootstrap-token] configured RBAC rules to allow certificate rotation for all node client certificates in the cluster
[bootstrap-token] Creating the "cluster-info" ConfigMap in the "kube-public" namespace
[kubelet-finalize] Updating "/etc/kubernetes/kubelet.conf" to point to a rotatable kubelet client certificate and key
[addons] Applied essential addon: CoreDNS
[addons] Applied essential addon: kube-proxy

Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

You can now join any number of control-plane nodes by copying certificate authorities
and service account keys on each node and then running the following as root:

  kubeadm join lb.kubesphere.local:6443 --token 6zsarg.gxg5eijglkupq85j \
    --discovery-token-ca-cert-hash sha256:8e1405a3da9e80413ab9aec1952a8259490cb174dcc74ecb96c0c5eafa429fd9 \
    --control-plane 

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join lb.kubesphere.local:6443 --token 6zsarg.gxg5eijglkupq85j \
    --discovery-token-ca-cert-hash sha256:8e1405a3da9e80413ab9aec1952a8259490cb174dcc74ecb96c0c5eafa429fd9
[master 172.16.7.12] MSG:
service "kube-dns" deleted
[master 172.16.7.12] MSG:
service/coredns created
[master 172.16.7.12] MSG:
serviceaccount/nodelocaldns created
daemonset.apps/nodelocaldns created
[master 172.16.7.12] MSG:
configmap/nodelocaldns created
[master 172.16.7.12] MSG:
I0105 23:04:05.247536   26174 version.go:252] remote version is much newer: v1.20.1; falling back to: stable-1.18
W0105 23:04:06.468801   26174 configset.go:202] WARNING: kubeadm cannot validate component configs for API groups [kubelet.config.k8s.io kubeproxy.config.k8s.io]
[upload-certs] Storing the certificates in Secret "kubeadm-certs" in the "kube-system" Namespace
[upload-certs] Using certificate key:
13a993ef56fb292d7ecb9947a3095a0eca6c419dfa148569699c474a8d6c28df
[master 172.16.7.12] MSG:
secret/kubeadm-certs patched
[master 172.16.7.12] MSG:
secret/kubeadm-certs patched
[master 172.16.7.12] MSG:
secret/kubeadm-certs patched
[master 172.16.7.12] MSG:
W0105 23:04:08.563212   26292 configset.go:202] WARNING: kubeadm cannot validate component configs for API groups [kubelet.config.k8s.io kubeproxy.config.k8s.io]
kubeadm join lb.kubesphere.local:6443 --token cfoibt.jdyzk3oc1aze53ri     --discovery-token-ca-cert-hash sha256:8e1405a3da9e80413ab9aec1952a8259490cb174dcc74ecb96c0c5eafa429fd9
[master 172.16.7.12] MSG:
NAME     STATUS     ROLES    AGE   VERSION   INTERNAL-IP   EXTERNAL-IP   OS-IMAGE                KERNEL-VERSION                CONTAINER-RUNTIME
master   NotReady   master   39s   v1.18.6   172.16.7.12   <none>        CentOS Linux 7 (Core)   3.10.0-1160.11.1.el7.x86_64   docker://19.3.9
INFO[23:04:09 CST] Deploying network plugin ...                 
[master 172.16.7.12] MSG:
configmap/calico-config created
customresourcedefinition.apiextensions.k8s.io/bgpconfigurations.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/bgppeers.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/blockaffinities.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/clusterinformations.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/felixconfigurations.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/globalnetworkpolicies.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/globalnetworksets.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/hostendpoints.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/ipamblocks.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/ipamconfigs.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/ipamhandles.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/ippools.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/kubecontrollersconfigurations.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/networkpolicies.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/networksets.crd.projectcalico.org created
clusterrole.rbac.authorization.k8s.io/calico-kube-controllers created
clusterrolebinding.rbac.authorization.k8s.io/calico-kube-controllers created
clusterrole.rbac.authorization.k8s.io/calico-node created
clusterrolebinding.rbac.authorization.k8s.io/calico-node created
daemonset.apps/calico-node created
serviceaccount/calico-node created
deployment.apps/calico-kube-controllers created
serviceaccount/calico-kube-controllers created
INFO[23:04:13 CST] Joining nodes to cluster                     
[node5 172.16.7.15] MSG:
W0105 23:04:14.035833   52825 join.go:346] [preflight] WARNING: JoinControlPane.controlPlane settings will be ignored when control-plane flag is not set.
[preflight] Running pre-flight checks
[preflight] Reading configuration from the cluster...
[preflight] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -oyaml'
W0105 23:04:20.030576   52825 utils.go:26] The recommended value for "clusterDNS" in "KubeletConfiguration" is: [10.233.0.10]; the provided value is: [169.254.25.10]
[kubelet-start] Downloading configuration for the kubelet from the "kubelet-config-1.18" ConfigMap in the kube-system namespace
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Starting the kubelet
[kubelet-start] Waiting for the kubelet to perform the TLS Bootstrap...

This node has joined the cluster:
* Certificate signing request was sent to apiserver and a response was received.
* The Kubelet was informed of the new secure connection details.

Run 'kubectl get nodes' on the control-plane to see this node join the cluster.
[node4 172.16.7.17] MSG:
W0105 23:04:14.432448   53936 join.go:346] [preflight] WARNING: JoinControlPane.controlPlane settings will be ignored when control-plane flag is not set.
[preflight] Running pre-flight checks
[preflight] Reading configuration from the cluster...
[preflight] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -oyaml'
W0105 23:04:19.870838   53936 utils.go:26] The recommended value for "clusterDNS" in "KubeletConfiguration" is: [10.233.0.10]; the provided value is: [169.254.25.10]
[kubelet-start] Downloading configuration for the kubelet from the "kubelet-config-1.18" ConfigMap in the kube-system namespace
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Starting the kubelet
[kubelet-start] Waiting for the kubelet to perform the TLS Bootstrap...

This node has joined the cluster:
* Certificate signing request was sent to apiserver and a response was received.
* The Kubelet was informed of the new secure connection details.

Run 'kubectl get nodes' on the control-plane to see this node join the cluster.
[node3 172.16.7.16] MSG:
W0105 23:04:14.376894   57091 join.go:346] [preflight] WARNING: JoinControlPane.controlPlane settings will be ignored when control-plane flag is not set.
[preflight] Running pre-flight checks
[preflight] Reading configuration from the cluster...
[preflight] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -oyaml'
W0105 23:04:20.949568   57091 utils.go:26] The recommended value for "clusterDNS" in "KubeletConfiguration" is: [10.233.0.10]; the provided value is: [169.254.25.10]
[kubelet-start] Downloading configuration for the kubelet from the "kubelet-config-1.18" ConfigMap in the kube-system namespace
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Starting the kubelet
[kubelet-start] Waiting for the kubelet to perform the TLS Bootstrap...

This node has joined the cluster:
* Certificate signing request was sent to apiserver and a response was received.
* The Kubelet was informed of the new secure connection details.

Run 'kubectl get nodes' on the control-plane to see this node join the cluster.
[node3 172.16.7.16] MSG:
node/node3 labeled
[node5 172.16.7.15] MSG:
node/node5 labeled
[node4 172.16.7.17] MSG:
node/node4 labeled
[master 172.16.7.12] MSG:
storageclass.storage.k8s.io/local created
serviceaccount/openebs-maya-operator created
clusterrole.rbac.authorization.k8s.io/openebs-maya-operator created
clusterrolebinding.rbac.authorization.k8s.io/openebs-maya-operator created
configmap/openebs-ndm-config created
daemonset.apps/openebs-ndm created
deployment.apps/openebs-ndm-operator created
deployment.apps/openebs-localpv-provisioner created
INFO[23:04:51 CST] Deploying KubeSphere ...                     
v3.0.0
[master 172.16.7.12] MSG:
namespace/kubesphere-system created
namespace/kubesphere-monitoring-system created
[master 172.16.7.12] MSG:
secret/kube-etcd-client-certs created
[master 172.16.7.12] MSG:
namespace/kubesphere-system unchanged
serviceaccount/ks-installer unchanged
customresourcedefinition.apiextensions.k8s.io/clusterconfigurations.installer.kubesphere.io unchanged
clusterrole.rbac.authorization.k8s.io/ks-installer unchanged
clusterrolebinding.rbac.authorization.k8s.io/ks-installer unchanged
deployment.apps/ks-installer unchanged
clusterconfiguration.installer.kubesphere.io/ks-installer created


INFO[23:10:23 CST] Installation is complete.

Please check the result using the command:

       kubectl logs -n kubesphere-system $(kubectl get pod -n kubesphere-system -l app=ks-install -o jsonpath='{.items[0].metadata.name}') -f

遇到問題及解決方案

Push /home/k8s-script/kubekey/v1.18.6/amd64/cni-plugins-linux-amd64-v0.8.6.tgz to 172.16.8.35:/tmp/kubekey/cni-plugins-linux-amd64-v0.8.6.tgz   Done
Push /home/k8s-script/kubekey/v1.18.6/amd64/cni-plugins-linux-amd64-v0.8.6.tgz to 172.16.8.36:/tmp/kubekey/cni-plugins-linux-amd64-v0.8.6.tgz   Done
INFO[16:20:31 CST] Initializing kubernetes cluster              
[master 172.16.8.36] MSG:
[preflight] Running pre-flight checks
W0105 16:20:38.445541   19396 removeetcdmember.go:79] [reset] No kubeadm config, using etcd pod spec to get data directory
[reset] No etcd config found. Assuming external etcd
[reset] Please, manually reset etcd to prevent further issues
[reset] Stopping the kubelet service
[reset] Unmounting mounted directories in "/var/lib/kubelet"
W0105 16:20:38.453323   19396 cleanupnode.go:99] [reset] Failed to evaluate the "/var/lib/kubelet" directory. Skipping its unmount and cleanup: lstat /var/lib/kubelet: no such file or directory
[reset] Deleting contents of config directories: [/etc/kubernetes/manifests /etc/kubernetes/pki]
[reset] Deleting files: [/etc/kubernetes/admin.conf /etc/kubernetes/kubelet.conf /etc/kubernetes/bootstrap-kubelet.conf /etc/kubernetes/controller-manager.conf /etc/kubernetes/scheduler.conf]
[reset] Deleting contents of stateful directories: [/var/lib/dockershim /var/run/kubernetes /var/lib/cni]

The reset process does not clean CNI configuration. To do so, you must remove /etc/cni/net.d

The reset process does not reset or clean up iptables rules or IPVS tables.
If you wish to reset iptables, you must do so manually by using the "iptables" command.

If your cluster was setup to utilize IPVS, run ipvsadm --clear (or similar)
to reset your system's IPVS tables.

The reset process does not clean your kubeconfig files and you must remove them manually.
Please, check the contents of the $HOME/.kube/config file.
[master 172.16.8.36] MSG:
[preflight] Running pre-flight checks
W0105 16:20:40.305612   19617 removeetcdmember.go:79] [reset] No kubeadm config, using etcd pod spec to get data directory
[reset] No etcd config found. Assuming external etcd
[reset] Please, manually reset etcd to prevent further issues
[reset] Stopping the kubelet service
[reset] Unmounting mounted directories in "/var/lib/kubelet"
W0105 16:20:40.310273   19617 cleanupnode.go:99] [reset] Failed to evaluate the "/var/lib/kubelet" directory. Skipping its unmount and cleanup: lstat /var/lib/kubelet: no such file or directory
[reset] Deleting contents of config directories: [/etc/kubernetes/manifests /etc/kubernetes/pki]
[reset] Deleting files: [/etc/kubernetes/admin.conf /etc/kubernetes/kubelet.conf /etc/kubernetes/bootstrap-kubelet.conf /etc/kubernetes/controller-manager.conf /etc/kubernetes/scheduler.conf]
[reset] Deleting contents of stateful directories: [/var/lib/dockershim /var/run/kubernetes /var/lib/cni]

The reset process does not clean CNI configuration. To do so, you must remove /etc/cni/net.d

The reset process does not reset or clean up iptables rules or IPVS tables.
If you wish to reset iptables, you must do so manually by using the "iptables" command.

If your cluster was setup to utilize IPVS, run ipvsadm --clear (or similar)
to reset your system's IPVS tables.

The reset process does not clean your kubeconfig files and you must remove them manually.
Please, check the contents of the $HOME/.kube/config file.
ERRO[16:20:41 CST] Failed to init kubernetes cluster: Failed to exec command: sudo -E /bin/sh -c "/usr/local/bin/kubeadm init --config=/etc/kubernetes/kubeadm-config.yaml" 
W0105 16:20:40.826437   19657 utils.go:26] The recommended value for "clusterDNS" in "KubeletConfiguration" is: [10.233.0.10]; the provided value is: [169.254.25.10]
W0105 16:20:40.826682   19657 configset.go:202] WARNING: kubeadm cannot validate component configs for API groups [kubelet.config.k8s.io kubeproxy.config.k8s.io]
[init] Using Kubernetes version: v1.18.6
[preflight] Running pre-flight checks
error execution phase preflight: [preflight] Some fatal errors occurred:
	[ERROR ExternalEtcdVersion]: couldn't load external etcd's certificate and key pair /etc/ssl/etcd/ssl/node-node3.pem, /etc/ssl/etcd/ssl/node-node3-key.pem: open /etc/ssl/etcd/ssl/node-node3.pem: no such file or directory
	[ERROR ExternalEtcdClientCertificates]: /etc/ssl/etcd/ssl/node-node3.pem doesn't exist
	[ERROR ExternalEtcdClientCertificates]: /etc/ssl/etcd/ssl/node-node3-key.pem doesn't exist
[preflight] If you know what you are doing, you can make a check non-fatal with `--ignore-preflight-errors=...`
To see the stack trace of this error execute with --v=5 or higher: Process exited with status 1  node=172.16.8.36
WARN[16:20:41 CST] Task failed ...                              
WARN[16:20:41 CST] error: interrupted by error                  
Error: Failed to init kubernetes cluster: interrupted by error
Usage:
  kk create cluster [flags]

Flags:
  -f, --filename string          Path to a configuration file
  -h, --help                     help for cluster
      --skip-pull-images         Skip pre pull images
      --with-kubernetes string   Specify a supported version of kubernetes
      --with-kubesphere          Deploy a specific version of kubesphere (default v3.0.0)
  -y, --yes                      Skip pre-check of the installation

Global Flags:
      --debug   Print detailed information (default true)

Failed to init kubernetes cluster: interrupted by error

在master節點執行

 sudo -E /bin/sh -c "/usr/local/bin/kubeadm init --config=/etc/kubernetes/kubeadm-config.yaml"

提取關鍵錯誤

[ERROR ExternalEtcdVersion]: couldn't load external etcd's certificate and key pair /etc/ssl/etcd/ssl/node-node3.pem, /etc/ssl/etcd/ssl/node-node3-key.pem: open /etc/ssl/etcd/ssl/node-node3.pem: no such file or directory
	[ERROR ExternalEtcdClientCertificates]: /etc/ssl/etcd/ssl/node-node3.pem doesn't exist
	[ERROR ExternalEtcdClientCertificates]: /etc/ssl/etcd/ssl/node-node3-key.pem doesn't exist

在master節點執行，再次查看

[root@master ~]# ls -lh /etc/ssl/etcd/ssl/
總用量 32K
-rw-r--r--. 1 root root 1.7K 1月   5 19:59 admin-node3-key.pem
-rw-r--r--. 1 root root 1.4K 1月   5 19:59 admin-node3.pem
-rw-r--r--. 1 root root 1.7K 1月   5 19:59 ca-key.pem
-rw-r--r--. 1 root root 1.1K 1月   5 19:59 ca.pem
-rw-r--r--. 1 root root 1.7K 1月   5 19:59 member-node3-key.pem
-rw-r--r--. 1 root root 1.4K 1月   5 19:59 member-node3.pem
-rw-r--r--. 1 root root 1.7K 1月   5 19:59 node-master-key.pem
-rw-r--r--. 1 root root 1.4K 1月   5 19:59 node-master.pem

原來在/etc/ssl/etcd/ssl/中真不存在node-node3-key.pem、node-node3.pem，真么辦？原來是我選擇的是member模式，

解決方法

[root@master ~]# cp  /etc/ssl/etcd/ssl/member-node3-key.pem /etc/ssl/etcd/ssl/node-node3-key.pem
[root@master ~]# cp /etc/ssl/etcd/ssl/member-node3.pem /etc/ssl/etcd/ssl/node-node3.pem

再次執行

export KKZONE=cn
./kk create cluster -f ./k8s-config.yaml

查看安裝KubeSphere日志：

kubectl logs -n kubesphere-system $(kubectl get pod -n kubesphere-system -l app=ks-install -o jsonpath='{.items[0].metadata.name}') -f

成功安裝KubeSphere3.0

登陸kubesphere

#ip:30880
#用戶名：admin
#默認密碼：P@88w0rd

優秀參考

https://www.cnblogs.com/elfcafe/p/13779619.html
https://www.cnblogs.com/carriezhangyan/p/11551192.html
https://www.cnblogs.com/xiao987334176/p/13267339.html
https://www.cnblogs.com/wenyang321/p/14086162.html
https://www.cnblogs.com/technology178/p/13547342.html
https://www.cnblogs.com/wuchangblog/p/14091717.html
https://www.cnblogs.com/it-peng/p/11393812.html

推薦這個博文：https://blog.csdn.net/weixin_43141746/article/details/110261158

轉載請註明出處，本文鏈接：https://www.uj5u.com/qita/245639.html

標籤：其他

上一篇：Shell腳本——業務上線前如何去掃描指定網段的所有IP地址呢？

下一篇：PHP實作爬取資料

采用KubeSphere的kk，部署安裝多節點服務的kubernetes-v1.18.6和kubesphere-v3.0.0的踩坑程序記錄，及反思

前言