目錄
- 一、多mater加入之前的單節點
- 二、lvs(keepalived+nginx)
- 三、dashboard
- 1、 創建rdac控制管理資源(kind:Role)
- 2、 創建secret安全資源(kind:Secret)
- 3、 創建configmap配置管理資源(kind:ConfigMap)
- 4、創建控制資源(kind:ServiceAccount、Deployment)
- 5、 創建service資源(kind:Service)
- 6、查看資源
- 7、訪問測驗
- 8、解決瀏覽器無法訪問的問題
- 9、訪問web網站頁面
etcd
flannel
master&node
一、多mater加入之前的單節點
systemctl stop firewalld
setenforce 0
#部署master2
將master01上的kuberetes目錄拷貝至master02上
scp -r /opt/kubernetes/ root@192.168.241.5:/opt
將master01上的三個組件啟動腳本拷貝至master02上
scp /usr/lib/systemd/system/{kube-apiserver,kube-controller-manager,kube-scheduler}.service root@192.168.241.5:/usr/lib/systemd/system/
修改master02組態檔 kube-apiserver中的IP地址
cd /opt/kubernetes/cfg
vim kube-apiserver
KUBE_APISERVER_OPTS="--logtostderr=true \
--v=4 \
--etcd-servers=https://192.168.241.20:2379,https://192.168.241.3:2379,https://192.168.241.4:2379 \
--bind-address=192.168.241.5 \ #修改master2的IP地址
--secure-port=6443 \
--advertise-address=192.168.241.5 \ #修改master2的IP地址
拷貝master01上已有的etcd證書給master02使用
注意:master02一定要有etcd證書(不裝etcd也需要etcd證書,因為master02也是要與etcd互動的)
scp -r /opt/etcd/ root@192.168.241.5:/opt
#啟動master02的三個組件服務
systemctl start kube-apiserver
systemctl enable kube-apiserver
systemctl status kube-apiserver
systemctl start kube-controller-manager
systemctl enable kube-controller-manager
systemctl status kube-controller-manager
systemctl start kube-scheduler
systemctl enable kube-scheduler
systemctl status kube-scheduler
#增加環境變數
vim /etc/profile
#末尾添加
export PATH=$path:/opt/kubernetes/bin
source /etc/profile
# 查看master02是否可以檢測到node節點
kubectl get node
此時的master2無法控制node節點,只能訪問etcd
二、lvs(keepalived+nginx)
所有nginx節點都需要操作
#負載均衡部署
systemctl stop firewalld
setenforce 0
#部署nginx
vim /etc/yum.repos.d/nginx.repo
[nginx]
name=nginx repo
baseurl=http://nginx.org/packages/centos/7/$basearch/
gpgcheck=0
yum list
yum -y install nginx
#添加四層轉發(添加stream模塊)
#在events模塊和http模塊中間添加一個獨立的stream模塊
vim /etc/nginx/nginx.conf
…………省略內容
stream {
log_format main '$remote_addr $upstream_addr - [$time_local] $status $upstream_bytes_sent';
access_log /var/log/nginx/k8s-access.log main; #指定日志存放目錄
upstream k8s-apiserver {
#master01的ip地址和埠
server 192.168.241.20:6443; #6443是apiserver的埠號
#master02的ip地址和埠
server 192.168.241.5:6443;
}
server {
listen 6443;
proxy_pass k8s-apiserver;
}
}
…………省略內容
# 開啟nginx服務
nginx -t #檢查組態檔是否有語法錯誤
systemctl start nginx #開啟nginx服務
netstat -ntap | grep nginx #查看nginx狀態及監聽埠6443
#部署keepalived高可用 #在2臺nginx服務器上配置
yum -y install keepalived
#修改組態檔
洗掉原有組態檔,重新定義添加
vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
# 接收郵件地址
notification_email {
acassen@firewall.loc
failover@firewall.loc
sysadmin@firewall.loc
}
# 郵件發送地址
notification_email_from Alexandre.Cassen@firewall.loc
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id NGINX_MASTER
}
vrrp_script check_nginx {
script "/etc/nginx/check_nginx.sh" #監控nginx腳本的路徑,稍后會創建
}
vrrp_instance VI_1 {
state MASTER #lb01該節點為MASTER,lb02設為BACKUP
interface ens33
virtual_router_id 51
priority 100 #優先級,lb01為master,優先級100,lb02為backup,優先級設90
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.241.100/24 #VIP地址(虛擬IP)
}
track_script {
check_nginx
}
}
#創建nginx監控腳本
vim /etc/nginx/check_nginx.sh
count=$(ps -ef |grep nginx |egrep -cv "grep|$$")
if [ "$count" -eq 0 ];then
systemctl stop keepalived
fi
chmod +x /etc/nginx/check_nginx.sh
#開啟服務
systemctl start keepalived.service
systemctl status keepalived.service
#查看漂移地址
ip a
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:91:1c:7f brd ff:ff:ff:ff:ff:ff
inet 192.168.241.6/24 brd 192.168.241.255 scope global ens33
valid_lft forever preferred_lft forever
inet 192.168.241.100/24 scope global secondary ens33 ##vip已經起來了
valid_lft forever preferred_lft forever
inet6 fe80::38d2:d1fa:bd9c:3f26/64 scope link
valid_lft forever preferred_lft forever
#驗證漂移地址
lb01中使用pkill nginx,再在lb02中使用ip a查看vip地址
#結束lb01上的nginx
pkill nginx
#keepalived也關閉了
systemctl status keepalived
#查看lb01地址
[root@localhost nginx]# ip a
......
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:91:1c:7f brd ff:ff:ff:ff:ff:ff
inet 192.168.241.6/24 brd 192.168.241.255 scope global ens33
valid_lft forever preferred_lft forever
inet6 fe80::38d2:d1fa:bd9c:3f26/64 scope link
valid_lft forever preferred_lft forever
......
這時lb01上的漂移地址就沒有了
#查看lb02地址
[root@localhost nginx]# ip a
......
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:2a:02:b2 brd ff:ff:ff:ff:ff:ff
inet 192.168.241.7/24 brd 192.168.241.255 scope global ens33
valid_lft forever preferred_lft forever
inet 192.168.241.100/24 scope global secondary ens33
valid_lft forever preferred_lft forever
inet6 fe80::fda:8925:c9d0:1438/64 scope link
valid_lft forever preferred_lft forever
......
# 恢復操作(在lb01中先啟動給nginx服務,在啟動keepalived服務)
因為有nginx監控,如果先啟動keepalived是啟不了的,
systemctl start nginx
systemctl start keepalived
#再次使用ip a查看lb01地址
漂移地址就回到了lb01上,因為lb01是主節點,優先級高
[root@localhost nginx]# ip a
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:91:1c:7f brd ff:ff:ff:ff:ff:ff
inet 192.168.241.6/24 brd 192.168.241.255 scope global ens33
valid_lft forever preferred_lft forever
inet 192.168.241.100/24 scope global secondary ens33
valid_lft forever preferred_lft forever
inet6 fe80::38d2:d1fa:bd9c:3f26/64 scope link
valid_lft forever preferred_lft forever
#node節點指向VIP漂移地址
兩個node均需要操作
#修改兩個node節點組態檔(bootstrap.kubeconfig 、kubelet.kubeconfig、kube-proxy.kubeconfig),server ip統一VIP地址
vim /opt/kubernetes/cfg/bootstrap.kubeconfig
server: https://192.168.241.100:6443
vim /opt/kubernetes/cfg/kubelet.kubeconfig
server: https://192.168.241.100:6443
vim /opt/kubernetes/cfg/kube-proxy.kubeconfig
server: https://192.168.241.100:6443
#重啟服務
systemctl restart kubelet.service
systemctl restart kube-proxy.service
#替換完成后自檢
grep 100 *
bootstrap.kubeconfig: server: https://192.168.241.100:6443
kubelet.kubeconfig: server: https://192.168.241.100:6443
kube-proxy.kubeconfig: server: https://192.168.241.100:6443
#在lb01上查看nginx的k8s日志
#檢查日志是否完成了訪問,建立了負載均衡
cat /var/log/nginx/k8s-access.log
192.168.241.3 192.168.241.20:6443 - [14/Apr/2021:17:33:07 +0800] 200 1119
192.168.241.3 192.168.241.5:6443 - [14/Apr/2021:17:33:07 +0800] 200 1120
192.168.241.4 192.168.241.5:6443 - [14/Apr/2021:17:33:11 +0800] 200 1120
192.168.241.4 192.168.241.5:6443 - [14/Apr/2021:17:33:11 +0800] 200 1118
##k8s集群測驗
master01操作
# 測驗創建pod
kubectl run nginx --image=nginx
#查看pod狀態
kubectl get pods
#該指令可以查看到資源具體資訊,IP及所在節點
kubectl get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE
nginx-dbddb74b8-8qrm4 1/1 Running 0 7m18s 172.17.69.3 192.168.241.3 <none>
#查看日志問題
#查看pod資源的日志
[root@localhost ~]# kubectl logs nginx-dbddb74b8-8qrm4
Error from server (Forbidden): Forbidden (user=system:anonymous, verb=get, resource=nodes, subresource=proxy) ( pods/log nginx-dbddb74b8-8qrm4)
原因:出現 error 是由于權限不足,需要提權解決辦法(添加匿名用戶授予權限):
kubectl create clusterrolebinding cluster-system-anonymous --clusterrole=cluster-admin --user=system:anonymous
提權后查看日志記錄是空的,因為現在這個容器并沒有被訪問
[root@localhost ~]# kubectl logs nginx-dbddb74b8-8qrm4
/docker-entrypoint.sh: /docker-entrypoint.d/ is not empty, will attempt to perform configuration
/docker-entrypoint.sh: Looking for shell scripts in /docker-entrypoint.d/
/docker-entrypoint.sh: Launching /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh
10-listen-on-ipv6-by-default.sh: info: Getting the checksum of /etc/nginx/conf.d/default.conf
10-listen-on-ipv6-by-default.sh: info: Enabled listen on IPv6 in /etc/nginx/conf.d/default.conf
/docker-entrypoint.sh: Launching /docker-entrypoint.d/20-envsubst-on-templates.sh
/docker-entrypoint.sh: Launching /docker-entrypoint.d/30-tune-worker-processes.sh
/docker-entrypoint.sh: Configuration complete; ready for start up
#訪問測驗(創建的資源是在192.168.241.3節點上的,所以在該節點進行訪問)
[root@localhost ~]# kubectl get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE
nginx-dbddb74b8-8qrm4 1/1 Running 0 7m18s 172.17.69.3 192.168.241.3 <none>
去對應的node節點上使用curl 172.17.69.3
[root@hzh ~]# curl 172.17.69.3
#訪問后就會產生日志記錄(再次回到master01查看日志)
[root@localhost ~]# kubectl logs nginx-dbddb74b8-8qrm4
/docker-entrypoint.sh: /docker-entrypoint.d/ is not empty, will attempt to perform configuration
/docker-entrypoint.sh: Looking for shell scripts in /docker-entrypoint.d/
/docker-entrypoint.sh: Launching /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh
10-listen-on-ipv6-by-default.sh: info: Getting the checksum of /etc/nginx/conf.d/default.conf
10-listen-on-ipv6-by-default.sh: info: Enabled listen on IPv6 in /etc/nginx/conf.d/default.conf
/docker-entrypoint.sh: Launching /docker-entrypoint.d/20-envsubst-on-templates.sh
/docker-entrypoint.sh: Launching /docker-entrypoint.d/30-tune-worker-processes.sh
/docker-entrypoint.sh: Configuration complete; ready for start up
172.17.69.1 - - [14/Apr/2021:09:42:34 +0000] "GET / HTTP/1.1" 200 612 "-" "curl/7.29.0" "-"
172.17.79.0 - - [14/Apr/2021:09:43:31 +0000] "GET / HTTP/1.1" 200 612 "-" "curl/7.29.0" "-"
三、dashboard
# 在master01上操作
#創建dashboard作業目錄
[root@master01 k8s]# mkdir dashboard
[root@master01 k8s]# cd dashboard
#拷貝官方的yaml檔案(此處已經提前下載好,直接拷貝至dashboard作業目錄)
官網下載地址:https://github.com/kubernetes/tree/master/cluster/addons/dashboard
1、 創建rdac控制管理資源(kind:Role)
1)創建
[root@master01 dashboard]# kubectl create -f dashboard-rbac.yaml #-f:以檔案(yaml檔案)的形式創建資源
role.rbac.authorization.k8s.io/kubernetes-dashboard-minimal created
rolebinding.rbac.authorization.k8s.io/kubernetes-dashboard-minimal created
2)查看yaml檔案,查看name名稱及namespace命名空間
[root@master01 dashboard]# vim dashboard-rbac.yaml
里面創建的資源kind是Role角色
3)查看Role角色資源
[root@master01 dashboard]# kubectl get Role -n kube-system #-n:指向命名空間
NAME AGE
extension-apiserver-authentication-reader 27h
kubernetes-dashboard-minimal 91s
system::leader-locking-kube-controller-manager 27h
system::leader-locking-kube-scheduler 27h
system:controller:bootstrap-signer 27h
system:controller:cloud-provider 27h
system:controller:token-cleaner 27h
2、 創建secret安全資源(kind:Secret)
1)創建
[root@master01 dashboard]# kubectl create -f dashboard-secret.yaml
secret/kubernetes-dashboard-certs created
secret/kubernetes-dashboard-key-holder created
2)查看yaml檔案,查看namespace命名空間及name名稱
[root@master01 dashboard]# vim dashboard-secret.yaml
3) 查看創建的資源
[root@master01 dashboard]# kubectl get Secret -n kube-system
NAME TYPE DATA AGE
default-token-4vhn6 kubernetes.io/service-account-token 3 27h
kubernetes-dashboard-certs Opaque 0 23s
kubernetes-dashboard-key-holder Opaque 0 23s
3、 創建configmap配置管理資源(kind:ConfigMap)
1)創建
[root@master01 dashboard]# kubectl create -f dashboard-configmap.yaml
configmap/kubernetes-dashboard-settings created
2)查看yaml檔案,查看namespace命名空間及name名稱
[root@master01 dashboard]# vim dashboard-configmap.yaml
3)查看創建的資源
[root@master01 dashboard]# kubectl get ConfigMap -n kube-system
NAME DATA AGE
extension-apiserver-authentication 1 27h
kubernetes-dashboard-settings 0 22s
4、創建控制資源(kind:ServiceAccount、Deployment)
1)創建
[root@master01 dashboard]# kubectl create -f dashboard-controller.yaml
serviceaccount/kubernetes-dashboard created
deployment.apps/kubernetes-dashboard created
創建了兩個資源(ServiceAccount服務訪問、Deployment控制器資源)
2) 查看yaml檔案,查看namespace命名空間及name名稱
[root@master01 dashboard]# vim dashboard-controller.yaml
3) 查看創建的資源
[root@master01 dashboard]# kubectl get ServiceAccount -n kube-system
NAME SECRETS AGE
default 1 27h
kubernetes-dashboard 1 27s
[root@master01 dashboard]# kubectl get Deployment -n kube-system
NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE
kubernetes-dashboard 1 1 1 1 59s
5、 創建service資源(kind:Service)
1) 創建
[root@master01 dashboard]# kubectl create -f dashboard-service.yaml
service/kubernetes-dashboard created
2) 查看yaml檔案,查看namespace命名空間及name名稱
service資源一旦使用,則說明服務就已經提供出去并且被訪問,所以必定會提供埠
[root@master01 dashboard]# vim dashboard-service.yaml
pod提供的埠,用戶不能直接訪問
node節點埠,用戶可以訪問node節點埠,映射到pod提供的埠
3) 查看創建的資源
[root@master01 dashboard]# kubectl get Service -n kube-system
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes-dashboard NodePort 10.0.0.60 <none> 443:30001/TCP 18s
6、查看資源
1) 創建完成后查看指定的命名空間kube-system下的pod資源
這時資源創建完成,可以查看整個pods資源
[root@master01 dashboard]# kubectl get pods -n kube-system
NAME READY STATUS RESTARTS AGE
kubernetes-dashboard-65f974f565-2xl7d 1/1 Running 0 2m15s
2) 查看多個資源(可以用逗號" ," 隔開)
[root@master01 dashboard]# kubectl get pods,service -n kube-system
NAME READY STATUS RESTARTS AGE
pod/kubernetes-dashboard-65f974f565-2xl7d 1/1 Running 0 2m39s
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/kubernetes-dashboard NodePort 10.0.0.60 <none> 443:30001/TCP 80s
3) 查看pods資源具體創建在哪個節點
[root@master01 dashboard]# kubectl get pods -n kube-system -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE
kubernetes-dashboard-65f974f565-2xl7d 1/1 Running 0 3m17s 172.17.69.4 192.168.241.3 <none>
7、訪問測驗
訪問nodeIP加埠號進行測驗:https://192.168.241.3:30001,如出現以下問題,屬于瀏覽器證書不被信任(谷歌瀏覽器的問題),一些老版本的瀏覽器可以進行訪問

下面我們就來解決證書不被信任的問題
8、解決瀏覽器無法訪問的問題
1、創建證書
[root@master01 dashboard]# vim dashboard-cert.sh
cat > dashboard-csr.json <<EOF
{
"CN": "Dashboard",
"hosts": [],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "BeiJing",
"ST": "BeiJing"
}
]
}
EOF
#產生ca證書
K8S_CA=$1
cfssl gencert -ca=$K8S_CA/ca.pem -ca-key=$K8S_CA/ca-key.pem -config=$K8S_CA/ca-config.json -profile=kubernetes dashboard-csr.json | cfssljson -bare dashboard
#洗掉原本的證書并重新創建證書
kubectl delete secret kubernetes-dashboard-certs -n kube-system
kubectl create secret generic kubernetes-dashboard-certs --from-file=./ -n kube-system
2、生成證書到指定目錄
[root@master01 dashboard]# bash dashboard-cert.sh /root/k8s/k8s-cert/
3、更改yaml檔案指向證書位置
[root@master01 dashboard]# vim dashboard-controller.yaml
……省略內容
args:
# PLATFORM-SPECIFIC ARGS HERE
- --auto-generate-certificates
- --tls-key-file=dashboard-key.pem #添加
- --tls-cert-file=dashboard.pem #添加
……省略內容
4、apply重新部署
變更過的yaml檔案需要重新部署資源
[root@master01 dashboard]# kubectl apply -f dashboard-controller.yaml
注意:重新部署后,資源所在的節點有可能會變動,建議重新檢查下資源資訊及所在節點
[root@master01 dashboard]# kubectl get pods -n kube-system -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE
kubernetes-dashboard-7dffbccd68-pt2st 1/1 Running 0 35s 172.17.79.3 192.168.241.4 <none>
5、訪問驗證
瀏覽器訪問:https://192.168.241.4:30001,這時就可以正常訪問了

9、訪問web網站頁面
輸入nodeIP及埠訪問后,會進入登錄方式的選擇,正常生產環境會選擇令牌的方式區登錄
這里我們也選擇令牌的方式登錄

1、生成令牌,創建資源
1) 創建admin賬戶角色資源,相當于管理員
[root@master01 dashboard]# kubectl create -f k8s-admin.yaml
serviceaccount/dashboard-admin created
clusterrolebinding.rbac.authorization.k8s.io/dashboard-admin created
2) 查看secret安全資源
[root@master01 dashboard]# kubectl get secret -n kube-system
NAME TYPE DATA AGE
dashboard-admin-token-8xp9n kubernetes.io/service-account-token 3 15s #生成了admin-token資源
default-token-4vhn6 kubernetes.io/service-account-token 3 27h
kubernetes-dashboard-certs Opaque 11 3m42s
kubernetes-dashboard-key-holder Opaque 2 12m
kubernetes-dashboard-token-mxlp9 kubernetes.io/service-account-token 3 9m50s
3) 查看admin-token
[root@master01 dashboard]# kubectl describe secret dashboard-admin-token-8xp9n -n kube-system
Name: dashboard-admin-token-8xp9n
Namespace: kube-system
Labels: <none>
Annotations: kubernetes.io/service-account.name: dashboard-admin
kubernetes.io/service-account.uid: a502f1dd-9d0e-11eb-9757-000c297eb227
Type: kubernetes.io/service-account-token
Data
====
ca.crt: 1359 bytes
namespace: 11 bytes
token: eyJhbGciOiJSUzI1NiIsImtpZCI6IiJ9.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJkYXNoYm9hcmQtYWRtaW4tdG9rZW4tOHhwOW4iLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC5uYW1lIjoiZGFzaGJvYXJkLWFkbWluIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQudWlkIjoiYTUwMmYxZGQtOWQwZS0xMWViLTk3NTctMDAwYzI5N2ViMjI3Iiwic3ViIjoic3lzdGVtOnNlcnZpY2VhY2NvdW50Omt1YmUtc3lzdGVtOmRhc2hib2FyZC1hZG1pbiJ9.gH8HMFx1lLsXEP9Lh1wq4hhG-Kq6MyCUNsUo30hNVAduVomgHYFIKNeFxC82oBrSbZX2keM2D2qfIQJk-LSImehDuHrqje67btaQxGGb0bk3RAN4-GDF4JdeFjGYQdIgXfrajbYqICYg1EsvQVWTjQEP5cJ3VJUKXOg4_8Yee3b8h6J5EsX-r7R4I68nghQeh9hiMb5FS_iVPrc2CHHGNbavekI671NwnrFJ_IkwFguHHJ8yNx3pve3UYRPRWAyhcSP16EJfoHFgUK4m7JdzLl1oEhjxjf5hE8N3LnFFmphahGrM_cjLBctLx-hkoL-gPxv5mVi-OCyW60xzJHUjPw
末尾token:后面的內容就是令牌碼,記錄下來用于web網站的登錄
2、復制令牌碼,進入web登錄頁面,登錄

現在我們就進入到了k8s的web網站頁面,可以在里面進行相應的查看及操作,這里就不一一演示了,
至此,我們整個k8s集群的二進制部署就全部完成了,包括單master節點,多master節點以及web網站頁面的顯示
轉載請註明出處,本文鏈接:https://www.uj5u.com/qita/276737.html
標籤:其他
