CICD: jenkins发布服务到kubernetes
- TAGS: cicd
jenkins 页面构建通过 helm 发布到 k8s 集群中
jenkins1
jenkins job 选项参数
branch: master SVC_ENV: prod PORT: 8080 JAVA_OPT: 空 Nacos_Server: http://nacos-headless.nacos.svc:8848 Nacos_Namespace: 0096540d-c49f-4f87-b3d8-274ef675efda replicas: 2 k8s_namespace: xxx-dev skywalking: xxx:11800
流水线
- SCM
- git
- repository url 代码仓库
- 脚本路径: deploy/Jenkinsfile
- git
deploy/Jenkinsfile
#!groovy node('master') { def jobname = env.JOB_BASE_NAME def name = jobname.split(/(dev-|qa-|prod-)/)[1] def workdir = env.WORKSPACE def port = env.PORT def nacosserver = env.Nacos_Server def nacosns = env.Nacos_Namespace def dockerid = env.BUILD_TAG def appenv = env.SVC_ENV def k8sns = env.k8s_namespace def k8sreplicas = env.replicas def skywalking = env.skywalking stage('代码获取') { dir("${workdir}") { checkout scm } } stage('构建代码') { sh "mvn clean package" sh "cp ${name}/target/${name}.jar deploy" } stage('构建Docker镜像') { docker.withRegistry('https://registry-vpc.cn-beijing.aliyuncs.com', 'e87917f6-1219-4d80-8c3e-d054ecbe268d') { def Image = docker.build("registry-vpc.cn-beijing.aliyuncs.com/xxx-${appenv}/${name}:latest", "./deploy --build-arg JAR_FILE=${name}.jar --build-arg SERVER_NAME=${name} --build-arg PORT=${port} --build-arg ENV=${appenv} --build-arg NACOS_SERVER=${nacosserver} --build-arg NACOS_NS=${nacosns} --build-arg SKYWALKING_SERVER=${skywalking}") Image.push("${dockerid}") Image.push("latest") } } try { stage('卸载服务') { sh "helm --kubeconfig /root/.kube/cici/${appenv}-config uninstall ${name} -n ${k8sns}" sh "sed -i 's/service-name/${name}/g' deploy/helm/Chart.yaml" } } catch (Exception e) { echo "continue" } stage('发布服务') { sh "helm --kubeconfig /root/.kube/cici/${appenv}-config upgrade ${name} deploy/helm \ --install \ --timeout 5m0s \ --namespace ${k8sns} \ --set service.port=${port} \ --set name=${name} \ --set replicas=${k8sreplicas} \ --set deployment.image.registry=registry-vpc.cn-beijing.aliyuncs.com/cici-${appenv}/${name} \ --set deployment.image.version=${dockerid}" } stage('服务列表') { sh "helm --kubeconfig /root/.kube/cici/${appenv}-config list -A | grep ${name}" } }
deploy/Dockerfile
FROM registry-vpc.cn-beijing.aliyuncs.com/cici-dev/openjdk8:oap-latest ARG JAR_FILE ARG PORT ARG ENV ARG NACOS_SERVER ARG NACOS_NS ARG JAVA_OPT ARG SKYWALKING_SERVER ARG SERVER_NAME ENV TZ=Asia/Shanghai ENV JAR_FILE=${JAR_FILE} ENV PORT=${PORT} ENV ENV=${ENV} ENV NACOS_SERVER=${NACOS_SERVER} ENV NACOS_NS=${NACOS_NS} ENV JAVA_OPT=${JAVA_OPT} ENV SKYWALKING_SERVER=${SKYWALKING_SERVER} ENV SERVER_NAME=${SERVER_NAME} WORKDIR /app COPY ${JAR_FILE} . COPY down_nacos.sh . EXPOSE ${PORT} ENTRYPOINT exec java ${JAVA_OPT} -javaagent:/opt/apache-skywalking-apm-bin/agent/skywalking-agent.jar -Dskywalking.agent.service_name=${SERVER_NAME} -Dskywalking.collector.backend_service=${SKYWALKING_SERVER} -jar ${JAR_FILE} --server.port=${PORT} --spring.profiles.active=${ENV} --spring.cloud.nacos.config.server-addr=${NACOS_SERVER} --spring.cloud.nacos.config.namespace=${NACOS_NS}
deploy/down_nacos.sh
ip=`ifconfig | grep inet | awk '{print $2}' | grep -v "127.0.0.1"` serviceName=`ls /app | grep *.jar | awk -F '.' '{print $1}'` port=`ps -ef | grep java | grep -v grep | awk -F '--server.port=' '{print $2}' | awk '{print $1}'` namespacesId=`ps -ef | grep java | grep -v grep | awk -F '--spring.cloud.nacos.config.namespace=' '{print $2}' | awk '{print $1}'` nacosServer=`ps -ef | grep java | grep -v grep | awk -F '--spring.cloud.nacos.config.server-addr=' '{print $2}' | awk '{print $1}'` curl -X PUT -d "serviceName=$serviceName" -d "ip=$ip" -d "port=$port" -d "enabled=false" -d "namespacesId=$namespacesId" $nacosServer/nacos/v1/ns/instance\?serviceName\=$serviceName\&ip\=$ip\&port\=$port\&enabled\=false\&namespaceId\=$namespacesId sleep 30
deploy/helm/Chart.yaml
apiVersion: v1 description: service-name service name: service-name-helm version: 1.0.0
deploy/helm/values.yaml
name: cici-gateway
replicas: 2
service:
port: 9999
check: /health
deployment:
image:
registry: registry.cn-beijing.aliyuncs.com/cici-dev/cici-gateway
version: latest
pullSecretName: cici-dev-registry
deploy/helm/templates/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
cici.apps: {{.Values.name}}
name: {{.Values.name}}
namespace: {{.Values.namespace}}
spec:
replicas: {{.Values.replicas}}
selector:
matchLabels:
cici.apps: {{.Values.name}}
template:
metadata:
labels:
cici.apps: {{.Values.name}}
spec:
{{- with .Values.deployment.image}}
imagePullSecrets:
- name: {{.pullSecretName}}
containers:
- image: {{ .registry}}:{{.version}}
imagePullPolicy: {{ .pullPolicy | default "Always" | quote }}
{{- end}}
name: {{.Values.name}}
ports:
- containerPort: {{.Values.service.port}}
readinessProbe:
httpGet:
path: {{.Values.service.check}}
port: {{.Values.service.port}}
scheme: HTTP
initialDelaySeconds: 30
periodSeconds: 20
timeoutSeconds: 10
failureThreshold: 2
successThreshold: 1
livenessProbe:
httpGet:
path: {{.Values.service.check}}
port: {{.Values.service.port}}
scheme: HTTP
initialDelaySeconds: 30
periodSeconds: 20
timeoutSeconds: 10
failureThreshold: 1
successThreshold: 1
lifecycle:
preStop:
exec:
command: ["/bin/bash", "/app/down_nacos.sh"]
resources:
requests:
memory: "2000Mi"
restartPolicy: Always
deploy/helm/templates/service.yaml
apiVersion: v1
kind: Service
metadata:
labels:
cici.apps: {{.Values.name}}
name: {{.Values.name}}
namespace: {{.Values.namespace}}
spec:
selector:
cici.apps: {{.Values.name}}
ports:
- name: tcp
port: {{.Values.service.port}}
targetPort: {{.Values.service.port}}
jenkins2
jenkins 页面构建通过 kubernetes插件 发布到 k8s 集群中
脚本式流水线与声明式流水线等价
声明式(前端)
pipeline {
environment {
COMMIT_ID = ""
imgurl = "1111222.dkr.ecr.us-west-2.amazonaws.com/overseas/${JOB_BASE_NAME}" //生成镜像存放镜像的仓库地址
GIT_ADDR = "[email protected]/we/r.git" //代码仓库地址
BRANCH = 'develop' //master develop
ENV = 'dev'
TAG = "" //镜像tag,会在下面生成,这里只是定义全局变量
}
//全局配置
options {
timestamps() //所有输出每行都会打印时间戳
buildDiscarder(logRotator(numToKeepStr: '20')) //保留20个历史构建版本
}
agent {
kubernetes {
cloud 'aws-overseas'
yaml '''
apiVersion: v1
kind: Pod
spec:
#imagePullSecrets:
#- name: abcd-hub
containers:
- name: jnlp
image: 1111222.dkr.ecr.us-west-2.amazonaws.com/overseas/jnlp-slave:latest
imagePullPolicy: IfNotPresent
args: [\'$(JENKINS_SECRET)\', \'$(JENKINS_NAME)\']
workingDir: /home/jenkins/agent
tty: true
volumeMounts:
- mountPath: "/root/.m2/repository"
name: "volume-0"
subPath: maven-repo-us-west-2-dev
readOnly: false
- mountPath: "/home/jenkins/agent"
name: "workspace-volume"
readOnly: false
workingDir: "/home/jenkins/agent"
- name: nodejs
image: node:13.14.0
imagePullPolicy: IfNotPresent
command: ['cat']
tty: true
volumeMounts:
- mountPath: "/home/jenkins/agent"
name: "workspace-volume"
readOnly: false
workingDir: "/home/jenkins/agent"
- name: dind
image: 1111222.dkr.ecr.us-west-2.amazonaws.com/overseas/docker-curl:latest
imagePullPolicy: IfNotPresent
command: ['cat']
tty: true
volumeMounts:
- mountPath: "/home/jenkins/agent"
name: "workspace-volume"
readOnly: false
workingDir: "/home/jenkins/agent"
- name: aws-cli
image: public.ecr.aws/aws-cli/aws-cli:latest
imagePullPolicy: IfNotPresent
command: ['cat']
tty: true
restartPolicy: "Never"
volumes:
- name: "volume-0"
emptyDir: {}
# nfs:
# path: "/mnt/nfs/maven-repo-singapore-dev"
# readOnly: false
# server: "10.2.0.1"
- name: "workspace-volume"
emptyDir:
medium: ""
'''
}
}
stages {
stage('Set Build Name') {
steps {
script {
currentBuild.displayName = "#${BUILD_NUMBER} ${BRANCH} by ${env.BUILD_USER}"
}
}
}
stage('Checkout Git') {
steps {
checkout([$class: 'GitSCM', branches: [[name: "${BRANCH}"]], userRemoteConfigs: [[credentialsId: '87d2e959-5740-4114-b9c7-0c42c8492cca', url: "${GIT_ADDR}"]]])
script {
COMMIT_ID = sh(returnStdout: true, script: "git log -n 1 --pretty=format:'%h'").trim()
TAG = BUILD_NUMBER + '_' + ENV + '_' + COMMIT_ID
println "Current branch is ${BRANCH}, Commit ID is ${COMMIT_ID}, Image TAG is ${TAG}"
}
}
}
stage('Build a Maven project') {
steps {
container('nodejs') {
sh 'echo "nameserver 8.8.8.8" >> /etc/resolv.conf'
sh """
npm install --unsafe-perm
npm run build:seasTest
ls dist/*
"""
}
}
}
stage('Build a Docker Image') {
steps {
container('aws-cli') {
sh """
export AWS_ACCESS_KEY_ID=000111
export AWS_SECRET_ACCESS_KEY=000111
if ! aws ecr describe-repositories --repository-names overseas/${JOB_BASE_NAME} &>/dev/null; then
echo Create repository
aws ecr create-repository --repository-name overseas/${JOB_BASE_NAME}
fi
"""
}
container('dind') {
sh '''
wget https://amazon-ecr-credential-helper-releases.s3.us-east-2.amazonaws.com/0.9.0/linux-amd64/docker-credential-ecr-login
chmod +x docker-credential-ecr-login
mv docker-credential-ecr-login /usr/local/bin/docker-credential-ecr-login
mkdir -pv ~/.docker/
cat > ~/.docker/config.json <<EOF
{
"credHelpers": {
"public.ecr.aws": "ecr-login",
"111112222222.dkr.ecr.us-west-2.amazonaws.com": "ecr-login"
}
}
EOF
mkdir -pv ~/.aws/
cat <<'EOF'> ~/.aws/credentials
[default]
aws_access_key_id = 000111
aws_secret_access_key = 000111
EOF
'''
}
container('dind') {
sh """
docker -H tcp://10.0.11.234:2375 build -t ${imgurl}:${TAG} -f- . <<'EOF'
FROM public.ecr.aws/nginx/nginx:latest
LABEL maintainer=ci
ENV SERVICE_NAME=manage-web SERVICE_PATH=/opt/manage-web
COPY dist \${SERVICE_PATH}/site
EXPOSE 80
WORKDIR \${SERVICE_PATH}
ENV TZ Asia/Shanghai
EOF
"""
sh "docker -H tcp://10.0.11.234:2375 tag ${imgurl}:${TAG} ${imgurl}:latest"
}
container('dind') {
sh """
#docker -H tcp://10.0.11.234:2375 login --username=abc -p 8ql6,yhY registry-vpc.ap-southeast-1.aliyuncs.com
docker -H tcp://10.0.11.234:2375 push ${imgurl}:${TAG}
docker -H tcp://10.0.11.234:2375 push ${imgurl}:latest
docker -H tcp://10.0.11.234:2375 rmi -f ${imgurl}:${TAG} ${imgurl}:latest
"""
}
}
}
stage('Deploy Trigger') {
agent { label 'master' }
steps {
sh "k8s_set_tag_aws_uswest2_dev ${JOB_BASE_NAME} ${TAG}"
}
}
stage("Deploy Status"){
agent { label 'master' }
steps {
sh "k8s_log_aws_uswest2_dev ${JOB_BASE_NAME}"
}
}
}
}
脚本式（后端）
def giturl = "[email protected]/abcd/w/a.git" def imgurl = "1111222.dkr.ecr.us-west-2.amazonaws.com/overseas/${JOB_BASE_NAME}" def pkg = "a/target/ap.jar" def pom = 'pom.xml' podTemplate( cloud: "aws-overseas", yaml: """ apiVersion: v1 kind: Pod spec: #imagePullSecrets: #- name: abcd-hub containers: - name: jnlp image: 1111222.dkr.ecr.us-west-2.amazonaws.com/overseas/jnlp-slave:latest imagePullPolicy: IfNotPresent args: ['\$(JENKINS_SECRET)', '\$(JENKINS_NAME)'] workingDir: /home/jenkins/agent tty: true volumeMounts: - mountPath: "/root/.m2/repository" name: "volume-0" subPath: maven-repo-us-west-2-dev readOnly: false - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false workingDir: "/home/jenkins/agent" - name: maven image: 1111222.dkr.ecr.us-west-2.amazonaws.com/overseas/maven:3.6-jdk-8-v3 imagePullPolicy: IfNotPresent workingDir: /home/jenkins/agent command: ['cat'] tty: true volumeMounts: - mountPath: "/root/.m2/repository" name: "volume-0" subPath: maven-repo-us-west-2-dev readOnly: false - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false workingDir: "/home/jenkins/agent" - name: dind image: 1111222.dkr.ecr.us-west-2.amazonaws.com/overseas/docker-curl:latest imagePullPolicy: IfNotPresent command: ['cat'] tty: true volumeMounts: - mountPath: "/root/.m2/repository" name: "volume-0" subPath: maven-repo-us-west-2-dev readOnly: false - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false workingDir: "/home/jenkins/agent" - name: aws-cli image: public.ecr.aws/aws-cli/aws-cli:latest imagePullPolicy: IfNotPresent command: ['cat'] tty: true restartPolicy: "Never" volumes: - name: "volume-0" #emptyDir: {} persistentVolumeClaim: claimName: efs-claim-maven # nfs: # path: "/mnt/nfs/maven-repo-singapore-dev" # readOnly: false # server: "10.0.11.234" - name: "workspace-volume" emptyDir: medium: "" """ ) { node(POD_LABEL) { stage('Set Build Name') { script { currentBuild.displayName = "#${BUILD_NUMBER} ${branch} by ${env.BUILD_USER}" } } stage('Checkout Git') { checkout([$class: 'GitSCM', branches: [[name: '${branch}']], extensions: [[$class: 'CloneOption', shallow: true]],userRemoteConfigs: [[credentialsId: '87d2e959-5740-4114-b9c7-0c42c8492cca', url: giturl]]]) } stage('Build a Maven project') { container('maven') { sh 'echo "nameserver 8.8.8.8" >> /etc/resolv.conf' sh 'mvn -B -Dmaven.test.skip=true -P overseasdev clean package -f ' + pom } container('aws-cli') { sh """ export AWS_ACCESS_KEY_ID=000111 export AWS_SECRET_ACCESS_KEY=000111 if ! aws ecr describe-repositories --repository-names overseas/${JOB_BASE_NAME} &>/dev/null; then echo Create repository aws ecr create-repository --repository-name overseas/${JOB_BASE_NAME} fi """ } } container('dind') { stage('Build a Docker Image') { sh ''' wget https://amazon-ecr-credential-helper-releases.s3.us-east-2.amazonaws.com/0.9.0/linux-amd64/docker-credential-ecr-login chmod +x docker-credential-ecr-login mv docker-credential-ecr-login /usr/local/bin/docker-credential-ecr-login mkdir -pv ~/.docker/ cat > ~/.docker/config.json <<EOF { "credHelpers": { "public.ecr.aws": "ecr-login", "1111222.dkr.ecr.us-west-2.amazonaws.com": "ecr-login" } } EOF mkdir -pv ~/.aws/ cat <<'EOF'> ~/.aws/credentials [default] aws_access_key_id = 000111 aws_secret_access_key = 000111 EOF ''' sh 'docker -H tcp://10.0.11.234:2375 build -t ' + imgurl + ':${BUILD_NUMBER} -f- .' + """<<EOF FROM 1111222.dkr.ecr.us-west-2.amazonaws.com/overseas/openjdk:8-jdk-arthas-start-v8-dev COPY """ + pkg + """ /ROOT.jar WORKDIR / ENV TZ Asia/Shanghai ENTRYPOINT ["java","-jar"] EOF """ sh 'docker -H tcp://10.0.11.234:2375 tag ' + imgurl + ':${BUILD_NUMBER} ' + imgurl + ':latest' } stage('Push Docker Image') { //sh 'docker -H tcp://10.0.11.234:2375 login --username=abc -p 8ql6,yhY 1111222.dkr.ecr.us-west-2.amazonaws.com' sh 'docker -H tcp://10.0.11.234:2375 push ' + imgurl + ':${BUILD_NUMBER}' sh 'docker -H tcp://10.0.11.234:2375 push ' + imgurl + ':latest' sh 'docker -H tcp://10.0.11.234:2375 rmi -f ' + imgurl + ':${BUILD_NUMBER} ' + imgurl + ':latest' } } } } node{ stage('Deploy Trigger') { sh 'k8s_set_tag_aws_uswest2_dev ${JOB_BASE_NAME} ${BUILD_NUMBER}' } stage("Deploy Status"){ sh 'k8s_log_aws_uswest2_dev ${JOB_BASE_NAME}' } }
cat <<\EOF> /usr/bin/k8s_set_tag_aws_uswest2_dev job_name=$1 build_num=$2 repo_url=111222.dkr.ecr.us-west-2.amazonaws.com/overseas/ img_url=$repo_url$job_name kubectl --kubeconfig /mnt/jenkins/config_aws_uswest2_dev -n dev1 get deploy ${job_name} if [ $? -eq 0 ]; then kubectl --kubeconfig /mnt/jenkins/config_aws_uswest2_dev -n dev1 set image deploy ${job_name} ${job_name}=$img_url:$build_num else ## 新建资源 端口随机，不好处理。 kubectl --kubeconfig /mnt/jenkins/config_aws_uswest2_dev -n dev1 set image deploy ${job_name} ${job_name}=$img_url:$build_num fi EOF cat <<\EOF> /usr/bin/k8s_log_aws_uswest2_dev job_name=$1 config="-n dev1 --kubeconfig /mnt/jenkins/config_aws_uswest2_dev" pod_name=$(kubectl ${config} get pod --sort-by=.metadata.creationTimestamp -l app=${job_name}|tail -1|awk '{print $1}') # 检测pod启动后再获取log while true; do kubectl ${config} get pod $pod_name 2>/dev/null |grep -q Running if [ $? -eq 0 ];then echo $pod_name "is running" break fi echo $pod_name "waiting to running" sleep 1 done kubectl ${config} logs -f $pod_name & #jenkins无需自己解决子进程问题 #pid=$! while true; do restarts=$(kubectl $config get pod $pod_name 2>/dev/null|grep -v NAME|awk '{print $4}') if [ $restarts -gt 0 ];then #kill $pid exit 1 fi kubectl $config rollout status deployment $job_name --watch=false 2>/dev/null|grep -q successfully if [ $? -eq 0 ];then #kill $pid exit 0 fi sleep 1 done EOF
服务发布更新减少耗时方案
maven构建速度： https://springdoc.cn/maven-fast-build/#google_vignette
构建工具优化-maven
0.构建耗时
pom.xml
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-profiler-plugin</artifactId>
<version>1.7</version>
</plugin>
</plugins>
</build>
执行
#cdp-server mvn -B -Dmaven.test.skip=true -Dprofile -P <env> clean package -f pom.xml #cat profiler cat .profiler/xxx.html
构建耗时：1m09s
1.参数优化(无效果)
#export MAVEN_OPTS=-Xms4096m -Xmx4096m export MAVEN_OPTS='-XX:-TieredCompilation -XX:TieredStopAtLevel=1' mvn -T 2C -B -Dmaven.test.skip=true -P <xxx> clean package -f pom.xml
- export MAVEN_OPTS='-XX:-TieredCompilation -XX:TieredStopAtLevel=1' 禁用默认分层编译
- -B 以批处理(batch)模式运行
- -T 2C 每一个CPU核使用2个线程，逻辑CPU核数. 模块化编译效果更佳 -pl module1,module2 -am
- -Dmaven.test.skip = true 跳过单元测试。使用 -DskipTests 选项跳过测试执行，但仍会编译 test 文件夹
- -Dmaven.compile.fork=true 指明多线程进行编译
- clean 通常情况下，我们建议用户在构建时使用 clean 参数保证构建的正确性。clean 可以删除旧的构建产物，但其实我们大多数时间可能不需要这个参数，只有在某些情况下（比如，更改了类名，或者删除了一些类）才必须使用这个参数，所以，如果某次变更只是修改了一些方法，或者增加了一些类，那么就不需要强制执行 clean 了
参数选项参考：
- 官方 https://maven.apache.org/ref/3.9.8/maven-embedder/cli.html
- 中文 https://maven.org.cn/ref/3.6.2/maven-embedder/cli.html
2.maven子项目mvnd构建（无效果）
wget https://downloads.apache.org/maven/mvnd/1.0.1/maven-mvnd-1.0.1-linux-amd64.tar.gz export PATH=/home/jenkins/agent/maven-mvnd-1.0.1-linux-amd64/bin:$PATH #构建 mvnd -T 2C -B -Dmaven.test.skip=true -P <xxx> clean package -f pom.xml
3.增量编译(无效果)
参考：https://stackoverflow.com/questions/8918165/does-maven-support-incremental-builds
4.缩小jar包体积
结论：不推荐。 不好实时跟进各组件依赖关系，维护成本高
参考文档：
deployment 滚动更新
shrink=' spec: strategy: rollingUpdate: maxSurge: 20% maxUnavailable: 0 type: RollingUpdate template: spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: type operator: NotIn values: - virtual-kubelet ' expand=' spec: strategy: rollingUpdate: maxSurge: 100% maxUnavailable: 0 type: RollingUpdate template: spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: type operator: In values: - virtual-kubelet ' kubectl patch deploy server-tmp-eci -o yaml --dry-run='client' --type merge --patch-file <(echo "$expand") kubectl patch deploy server-tmp-eci --type merge --patch-file <(echo "$expand") kubectl patch deploy server-tmp-eci --type merge --patch-file <(echo "$shrink")