1 環境概述
192.168.38.50 master
192.168.38.51 node1
192.168.38.52 node2
192.168.38.100 harbor/nfs
192.168.38.45 gitlab/sonarqube
2 Kubernetes安裝Jenkins
2.1 PV/PVC
#創建命名空間
kubectl create namespace jenkins-k8s
(1) 創建PV
vim jenkins-pv.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
name: jenkins-pv
spec:
capacity:
storage: 10Gi
accessModes:
- ReadWriteMany
nfs:
path: /usr/local/kubernetes/volumes/jenkins
server: 192.168.38.100
#應用
kubectl apply -f jenkins-pv.yaml

(2) 創建PVC
vim jenkins-pvc.yaml
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: jenkins-pvc
namespace: jenkins-k8s
spec:
resources:
requests:
storage: 10Gi
accessModes:
- ReadWriteMany
#應用
kubectl apply -f jenkins-pvc.yaml

2.2 ServiceAccount
#創建一個sa賬號
kubectl create sa jenkins-k8s-sa -n jenkins-k8s
#授權,kubectl create clusterrolebinding 名稱、名稱空間、系結clusterrole=cluster-admin
kubectl create clusterrolebinding jenkins-k8s-sa-cluster -n jenkins-k8s --clusterrole=cluster-admin --serviceaccount=jenkins-k8s:jenkins-k8s-sa

2.3 整合Jenkins
2.3.1 下載鏡像
#node1、node2
docker pull jenkins/jenkins:latest
2.3.2 jenkins-deployment.yaml
vim jenkins-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: jenkins
namespace: jenkins-k8s
spec:
replicas: 1
selector:
matchLabels:
app: jenkins
template:
metadata:
labels:
app: jenkins
spec:
serviceAccountName: jenkins-k8s-sa
containers:
- name: jenkins
image: jenkins/jenkins:latest
imagePullPolicy: IfNotPresent
ports:
- containerPort: 8080
name: web
protocol: TCP
- containerPort: 50000
name: agent
protocol: TCP
resources:
limits:
cpu: 1000m
memory: 1Gi
requests:
cpu: 500m
memory: 500Mi
livenessProbe:
httpGet:
port: 8080
path: /login
initialDelaySeconds: 60
timeoutSeconds: 5
failureThreshold: 12
readinessProbe:
httpGet:
port: 8080
path: /login
initialDelaySeconds: 60
volumeMounts:
- mountPath: /var/jenkins_home
name: jenkins-volume
subPath: jenkins-home
volumes:
- name: jenkins-volume
persistentVolumeClaim:
claimName: jenkins-pvc
#應用
kubectl apply -f jenkins-deployment.yaml
#查看
kubectl get pod -n jenkins-k8s
#驗證
kubectl get pod -o wide -n jenkins-k8s
curl 10.244.2.23:8080

2.3.3 jenkins-svc.yaml
#創建
vim jenkins-svc.yaml
apiVersion: v1
kind: Service
metadata:
name: jenkins-service
namespace: jenkins-k8s
labels:
app: jenkins
spec:
selector:
app: jenkins
type: ClusterIP
ports:
- port: 8080
name: web
targetPort: web
- name: agent
port: 50000
targetPort: agent
#應用
kubectl apply -f jenkins-svc.yaml
#查看
kubectl get svc -n jenkins-k8s

2.3.4 jenkins-ingress.yaml
vim jenkins-ingress.yaml
apiVersion: networking.k8s.io/v1beta1
kind: Ingress
metadata:
name: jenkins
namespace: jenkins-k8s
spec:
rules:
- host: rosh.jenkins.com
http:
paths:
- backend:
serviceName: jenkins-service
servicePort: 8080
#應用
kubectl apply -f jenkins-ingress.yaml
#查看
kubectl get ingress -n jenkins-k8s

2.4 Jenkins 初始化
(1) 查看初始化密碼
#查看掛載日志,記錄初始化密碼,填入初始化頁面
84f98993d6594792b7e420c74934eb89

(2)初始化插件


(3)初始化用戶


2.5 中文插件



2.6 kubernetes插件
(1) 安裝插件
(2) 配置Kubernetes



2.7 Jenkins從節點配置
2.7.1 制作鏡像

(1) setting.xml
<?xml version="1.0" encoding="UTF-8"?>
<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
<localRepository>/usr/local/apache-maven/repo</localRepository>
<pluginGroups></pluginGroups>
<proxies></proxies>
<servers></servers>
<profiles></profiles>
<mirrors>
<mirror>
<id>central</id>
<mirrorOf>central</mirrorOf>
<name>aliyun maven</name>
<url>https://maven.aliyun.com/repository/public</url>
</mirror>
</mirrors>
</settings>
(2) Dockerfile
FROM cnych/jenkins:jnlp6
MAINTAINER rosh
# 切換到 root 賬戶進行操作
USER root
# 安裝 maven
COPY apache-maven-3.6.2-bin.tar.gz .
RUN tar -zxf apache-maven-3.6.2-bin.tar.gz && \
mv apache-maven-3.6.2 /usr/local && \
rm -f apache-maven-3.6.2-bin.tar.gz && \
ln -s /usr/local/apache-maven-3.6.2/bin/mvn /usr/bin/mvn && \
ln -s /usr/local/apache-maven-3.6.2 /usr/local/apache-maven && \
mkdir -p /usr/local/apache-maven/repo
COPY settings.xml /usr/local/apache-maven/conf/settings.xml
(3) 制作鏡像
docker build -t jenkins-slave-maven .

(4) 上傳至harbor
docker tag jenkins-slave-maven:latest 192.168.38.100/library/jenkins-slave-maven:latest
docker push 192.168.38.100/library/jenkins-slave-maven:latest
(5) 查看

2.7.2 配置模板


master的/root/.kube目錄拷貝到node1和node2節點:
scp -r /root/.kube node1:
scp -r /root/.kube node2:
2.7.3 測驗
(1) 創建流水線

(2) 測驗腳本
//參考jenkins-slave的pod模塊來構建Jenkins-Slave的pod
node("jenkins-slave"){
stage('k8s-測驗'){
sh "kubectl get node"
}
stage('maven-測驗'){
sh "mvn -v"
}
stage('docker 測驗'){
sh "docker version"
}
}
(3) 結果


3 整合Gitlab
3.1 憑證
jenkins添加訪問憑證

3.2 創建pipeline工程

3.3 整合Hook


3.4 Gitlab



3.5 Jenkinsfile
(1) 流水線配置

(2) 生成腳本

(3) Jenkinsfile
//參考jenkins-slave的pod模塊來構建Jenkins-Slave的pod
node("jenkins-slave"){
stage('拉取代碼'){
git branch: 'develop', credentialsId: 'ad4d8da3-c362-4b92-b8da-e218f372d193', url: 'http://192.168.38.45/root/springboot-cloud-k8s.git'
echo "拉取代碼成功"
}
}
(4) 推送代碼到develop分支
(5) 查看jenkins 日志


3.6 Generic Webhook Trigger
查看Gitlab推送請求,防止頻繁構建,現在限制develop分支推送代碼時,觸發自動構建,別的分支不觸發構建,

3.6.1 修改Pipeline配置



3.6.2 修改Gitlab鉤子

3.6.3 測驗
master分支merge develop分支代碼,推送master分支,沒有觸發jenkins自動化構建,查看gitlab回應資訊

4 制作鏡像、上傳Harbor
4.1 Dockerfile

#######################網關#######################
FROM java:8
RUN /bin/cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
RUN echo "Asia/Shanghai" > /etc/timezone
RUN bash -c 'touch /app.jar'
ARG JAR_FILE
COPY ${JAR_FILE} app.jar
EXPOSE 9000
ENTRYPOINT ["java","-jar","/app.jar"]
#######################student#######################
FROM java:8
RUN /bin/cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
RUN echo "Asia/Shanghai" > /etc/timezone
RUN bash -c 'touch /app.jar'
ARG JAR_FILE
COPY ${JAR_FILE} app.jar
EXPOSE 8001
ENTRYPOINT ["java","-jar","/app.jar"]
#######################teacher#######################
FROM java:8
RUN /bin/cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
RUN echo "Asia/Shanghai" > /etc/timezone
RUN bash -c 'touch /app.jar'
ARG JAR_FILE
COPY ${JAR_FILE} app.jar
EXPOSE 8002
ENTRYPOINT ["java","-jar","/app.jar"]
4.2 Maven插件
<!-- docker的maven插件-->
<plugin>
<groupId>com.spotify</groupId>
<artifactId>dockerfile-maven-plugin</artifactId>
<version>1.3.6</version>
<configuration>
<repository>${project.artifactId}</repository>
<buildArgs>
<JAR_FILE>target/${project.build.finalName}.jar</JAR_FILE>
</buildArgs>
</configuration>
</plugin>
4.3 集成Harbor
(1) 憑證

(2) 生成腳本

4.4 nfs創建maven目錄
#nfs服務器
cd /usr/local/kubernetes/volumes
mkdir maven
chmod 777 ./maven/
chown nfsnobody ./maven/
vim /etc/exports
/usr/local/kubernetes/volumes/maven/ *(rw,sync,insecure,no_subtree_check,no_root_squash)
#重啟
systemctl restart rpcbind && systemctl restart nfs
#其它節點測驗
showmount -e 192.168.38.100

4.5 Jenkinsfile
//harbor地址
def harborUrl = "192.168.38.100"
//harbor倉庫名稱
def harborRepository = "springcloud-k8s"
//構建版本tag
def tag = "latest"
//harbor 憑證
def harborAuth = "3db8138a-792d-4d9b-8b4e-c7c62d9f6cf4"
//微服務串列
def microServerList = ['gateway-server','student-server','teacher-server']
//參考jenkins-slave的pod模塊來構建Jenkins-Slave的pod
node("jenkins-slave"){
stage('拉取代碼'){
git branch: 'develop', credentialsId: 'ad4d8da3-c362-4b92-b8da-e218f372d193', url: 'http://192.168.38.45/root/springboot-cloud-k8s.git'
echo "拉取代碼成功"
}
stage('制作鏡像,上傳harbor'){
for(int i=0;i<microServerList.size();i++){
echo "制作鏡像: ${microServerList[i]}"
//服務名稱
def serverName = "${microServerList[i]}"
//定義鏡像名稱
def imageName="${serverName}:${tag}"
//打包,構建鏡像
sh "mvn -f ${serverName} clean package dockerfile:build"
// 打tag
sh "docker tag ${imageName} ${harborUrl}/${harborRepository}/${imageName}"
// 上傳鏡像
withCredentials([usernamePassword(credentialsId: '3db8138a-792d-4d9b-8b4e-c7c62d9f6cf4', passwordVariable: 'password', usernameVariable: 'username')]) {
// some block
//登錄
sh "docker login -u ${username} -p ${password} ${harborUrl}"
//上傳
sh "docker push ${harborUrl}/${harborRepository}/${imageName}"
}
//洗掉
sh "docker rmi -f ${imageName}"
sh "docker rmi -f ${harborUrl}/${harborRepository}/${imageName}"
}
}
}
4.6 推送dev分支
4.7 jenkins查看結果


5 SonarQube
5.1 Sonarqube token


#記錄token
c06945196968cf0a3e5e326e1e4b2646fc97b135
mvn sonar:sonar \
-Dsonar.host.url=http://192.168.38.45:9000/sonar \
-Dsonar.login=c06945196968cf0a3e5e326e1e4b2646fc97b135

5.2 Jenkins整合Sonarqube
(1) 安裝插件

(2) 創建憑證

(3) 系統配置

(4) 全域配置


5.3 sonar-project.properties
#################################網關##########################################
#SonarQube 實體名稱,唯一
sonar.projectKey=gateway-server
#專案名稱
sonar.projectName=gateway-server
#版本號
sonar.projectVersion=1.0
#指定掃描目錄
sonar.sources=.
sonar.exclusions=**/test/**
#jdk版本
sonar.java.source=1.8
sonar.java.target=1.8
#掃描撰寫類的專案
sonar.java.binaries=target/classes
#編碼格式
sonar.sourceEncoding=UTF-8
#################################學生##########################################
#SonarQube 實體名稱,唯一
sonar.projectKey=student-server
#專案名稱
sonar.projectName=student-server
#版本號
sonar.projectVersion=1.0
#指定掃描目錄
sonar.sources=.
sonar.exclusions=**/test/**
#jdk版本
sonar.java.source=1.8
sonar.java.target=1.8
#掃描撰寫類的專案
sonar.java.binaries=target/classes
#編碼格式
sonar.sourceEncoding=UTF-8
#################################老師##########################################
#SonarQube 實體名稱,唯一
sonar.projectKey=teacher-server
#專案名稱
sonar.projectName=teacher-server
#版本號
sonar.projectVersion=1.0
#指定掃描目錄
sonar.sources=.
sonar.exclusions=**/test/**
#jdk版本
sonar.java.source=1.8
sonar.java.target=1.8
#掃描撰寫類的專案
sonar.java.binaries=target/classes
#編碼格式
sonar.sourceEncoding=UTF-8
5.4 Jenkinsfile
//harbor地址
def harborUrl = "192.168.38.100"
//harbor倉庫名稱
def harborRepository = "springcloud-k8s"
//構建版本tag
def tag = "latest"
//harbor 憑證
def harborAuth = "3db8138a-792d-4d9b-8b4e-c7c62d9f6cf4"
//微服務串列
def microServerList = ['gateway-server','student-server','teacher-server']
//參考jenkins-slave的pod模塊來構建Jenkins-Slave的pod
node("jenkins-slave"){
stage('拉取代碼'){
git branch: 'develop', credentialsId: 'ad4d8da3-c362-4b92-b8da-e218f372d193', url: 'http://192.168.38.45/root/springboot-cloud-k8s.git'
echo "拉取代碼成功"
}
stage('制作鏡像,上傳harbor'){
for(int i=0;i<microServerList.size();i++){
echo "制作鏡像: ${microServerList[i]}"
//服務名稱
def serverName = "${microServerList[i]}"
//定義鏡像名稱
def imageName="${serverName}:${tag}"
//打包,構建鏡像
sh "mvn -f ${serverName} clean package dockerfile:build"
// 打tag
sh "docker tag ${imageName} ${harborUrl}/${harborRepository}/${imageName}"
// 上傳鏡像
withCredentials([usernamePassword(credentialsId: '3db8138a-792d-4d9b-8b4e-c7c62d9f6cf4', passwordVariable: 'password', usernameVariable: 'username')]) {
// some block
//登錄
sh "docker login -u ${username} -p ${password} ${harborUrl}"
//上傳
sh "docker push ${harborUrl}/${harborRepository}/${imageName}"
}
//洗掉
sh "docker rmi -f ${imageName}"
sh "docker rmi -f ${harborUrl}/${harborRepository}/${imageName}"
}
}
stage('代碼審查'){
//定義當前Jenkins的SonarQubeScanner工具
def scannerHome = tool 'sonarqube-scanner'
for(int i=0;i<microServerList.size();i++){
//參考當前JenkinsSonarQube環境
withSonarQubeEnv('sonarqube') {
sh """cd ${microServerList[i]}
${scannerHome}/bin/sonar-scanner"""
}
}
}
}
5.5 推代碼
5.6 查看結果


6 部署
6.1創建日志PV/PVC
(1) gateway
vim gateway-pv-pvc.yaml
kubectl apply -f gateway-pv-pvc.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
name: gateway-logs-pv
spec:
capacity:
storage: 2Gi
accessModes:
- ReadWriteMany
nfs:
path: "/usr/local/kubernetes/volumes/logs/gateway"
server: 192.168.38.100
---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: gateway-logs-pvc
namespace: jenkins-k8s
spec:
resources:
requests:
storage: 2Gi
accessModes:
- ReadWriteMany
(2) teacher
vim teacher-pv-pvc.yaml
kubectl apply -f teacher-pv-pvc.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
name: teacher-logs-pv
spec:
capacity:
storage: 2Gi
accessModes:
- ReadWriteMany
nfs:
path: "/usr/local/kubernetes/volumes/logs/teacher"
server: 192.168.38.100
---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: teacher-logs-pvc
namespace: jenkins-k8s
spec:
resources:
requests:
storage: 2Gi
accessModes:
- ReadWriteMany
(3) student
vim student-pv-pvc.yaml
kubectl apply -f student-pv-pvc.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
name: student-logs-pv
spec:
capacity:
storage: 2Gi
accessModes:
- ReadWriteMany
nfs:
path: "/usr/local/kubernetes/volumes/logs/student"
server: 192.168.38.100
---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: student-logs-pvc
namespace: jenkins-k8s
spec:
resources:
requests:
storage: 2Gi
accessModes:
- ReadWriteMany

6.2 創建權限
kubectl create secret docker-registry registry-secret --namespace=jenkins-k8s \
--docker-server=192.168.38.100 --docker-username=admin \
--docker-password=Harbor12345
kubectl get secret

6.3 Deployment
(1) gateway
##############Deployment######################
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: gateway-server
name: gateway-server
namespace: jenkins-k8s
spec:
replicas: 2
selector:
matchLabels:
app: gateway-server
template:
metadata:
labels:
app: gateway-server
spec:
imagePullSecrets:
- name: registry-secret
containers:
- name: gateway-server
image: 192.168.38.100/springcloud-k8s/gateway-server:latest
imagePullPolicy: Always
volumeMounts:
- mountPath: /gateway-server/logs
name: gateway-server
volumes:
- name: gateway-server
persistentVolumeClaim:
claimName: gateway-logs-pvc
---
##############Service####################
apiVersion: v1
kind: Service
metadata:
labels:
app: gateway-server
name: gateway-server
spec:
ports:
- port: 9000
protocol: TCP
targetPort: 9000
selector:
app: gateway-server
type: ClusterIP
---
##############Ingress####################
apiVersion: networking.k8s.io/v1beta1
kind: Ingress
metadata:
name: gateway-server
spec:
rules:
- host: rosh.gateway-server.com
http:
paths:
- backend:
serviceName: gateway-server
servicePort: 9000
(2) student
##############Deployment####################
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: student-server
name: student-server
namespace: jenkins-k8s
spec:
replicas: 2
selector:
matchLabels:
app: student-server
template:
metadata:
labels:
app: student-server
spec:
imagePullSecrets:
- name: registry-secret
containers:
- name: student-server
image: 192.168.38.100/springcloud-k8s/student-server:latest
imagePullPolicy: Always
volumeMounts:
- mountPath: /student-server/logs
name: student-server
volumes:
- name: student-server
persistentVolumeClaim:
claimName: student-logs-pvc
---
##############Service####################
apiVersion: v1
kind: Service
metadata:
labels:
app: student-server
name: student-server
spec:
ports:
- port: 8001
protocol: TCP
targetPort: 8001
selector:
app: student-server
type: ClusterIP
---
##############Ingress####################
apiVersion: networking.k8s.io/v1beta1
kind: Ingress
metadata:
name: student-server
spec:
rules:
- host: rosh.student-server.com
http:
paths:
- backend:
serviceName: student-server
servicePort: 8001
(3) teacher
##############Deployment####################
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: teacher-server
name: teacher-server
namespace: jenkins-k8s
spec:
replicas: 2
selector:
matchLabels:
app: teacher-server
template:
metadata:
labels:
app: teacher-server
spec:
imagePullSecrets:
- name: registry-secret
containers:
- name: teacher-server
image: 192.168.38.100/springcloud-k8s/teacher-server:latest
imagePullPolicy: Always
volumeMounts:
- mountPath: /teacher-server/logs
name: teacher-server
volumes:
- name: teacher-server
persistentVolumeClaim:
claimName: teacher-logs-pvc
---
##############Service####################
apiVersion: v1
kind: Service
metadata:
labels:
app: teacher-server
name: teacher-server
spec:
ports:
- port: 8002
protocol: TCP
targetPort: 8002
selector:
app: teacher-server
type: ClusterIP
---
##############Ingress####################
apiVersion: networking.k8s.io/v1beta1
kind: Ingress
metadata:
name: teacher-server
spec:
rules:
- host: rosh.teacher-server.com
http:
paths:
- backend:
serviceName: teacher-server
servicePort: 8002
6.4 Jenkinsfile
//harbor地址
def harborUrl = "192.168.38.100"
//harbor倉庫名稱
def harborRepository = "springcloud-k8s"
//構建版本tag
def tag = "latest"
//harbor 憑證
def harborAuth = "3db8138a-792d-4d9b-8b4e-c7c62d9f6cf4"
//微服務串列
def microServerList = ['gateway-server','student-server','teacher-server']
//參考jenkins-slave的pod模塊來構建Jenkins-Slave的pod
node("jenkins-slave"){
stage('拉取代碼'){
git branch: 'develop', credentialsId: 'ad4d8da3-c362-4b92-b8da-e218f372d193', url: 'http://192.168.38.45/root/springboot-cloud-k8s.git'
echo "拉取代碼成功"
}
stage('制作鏡像,上傳harbor'){
for(int i=0;i<microServerList.size();i++){
echo "制作鏡像: ${microServerList[i]}"
//服務名稱
def serverName = "${microServerList[i]}"
//定義鏡像名稱
def imageName="${serverName}:${tag}"
//打包,構建鏡像
sh "mvn -f ${serverName} clean package dockerfile:build"
// 打tag
sh "docker tag ${imageName} ${harborUrl}/${harborRepository}/${imageName}"
// 上傳鏡像
withCredentials([usernamePassword(credentialsId: '3db8138a-792d-4d9b-8b4e-c7c62d9f6cf4', passwordVariable: 'password', usernameVariable: 'username')]) {
// some block
//登錄
sh "docker login -u ${username} -p ${password} ${harborUrl}"
//上傳
sh "docker push ${harborUrl}/${harborRepository}/${imageName}"
}
}
}
stage('代碼審查'){
//定義當前Jenkins的SonarQubeScanner工具
def scannerHome = tool 'sonarqube-scanner'
for(int i=0;i<microServerList.size();i++){
//參考當前JenkinsSonarQube環境
withSonarQubeEnv('sonarqube') {
sh """cd ${microServerList[i]}
${scannerHome}/bin/sonar-scanner"""
}
}
}
stage('k8s 部署'){
for(int i=0;i<microServerList.size();i++){
sh "kubectl apply -f ${microServerList[i]}/Deployment.yaml"
}
}
stage('洗掉本地鏡像 '){
for(int i=0;i<microServerList.size();i++){
//服務名稱
def serverName = "${microServerList[i]}"
//定義鏡像名稱
def imageName="${serverName}:${tag}"
//洗掉
sh "docker rmi -f ${imageName}"
sh "docker rmi -f ${harborUrl}/${harborRepository}/${imageName}"
}
}
}
6.5 結果




轉載請註明出處,本文鏈接:https://www.uj5u.com/qita/147424.html
標籤:其他
