Kubernetes: Gitlab

gitlab:
  ## https://docs.gitlab.com/charts/charts/gitlab/task-runner
  - task-runner
  Task Runner Pod 用于在 GitLab 应用程序中执行定期内务管理任务。这些任务包括备份、Sidekiq 维护和 Rake 任务
  镜像registry.gitlab.com/gitlab-org/build/cng/gitlab-toolbox-ee:v14.2.0
  ## https://docs.gitlab.com/charts/charts/gitlab/migrations
  - migrations:
  数据迁移，与task-runner使用同一个镜像

  ## https://docs.gitlab.com/charts/charts/gitlab/webservice
  - webservice:
  Puma(GitLab Rails)的Web服务器由2个容器组成
  镜像registry.gitlab.com/gitlab-org/build/cng/gitlab-webservice-ee:v14.2.0
  镜像registry.gitlab.com/gitlab-org/build/cng/gitlab-workhorse-ee:v14.2.0

  ## https://docs.gitlab.com/charts/charts/gitlab/sidekiq
  - sidekiq:
  后台服务，从redis队列中提取作业来处理
  镜像registry.gitlab.com/gitlab-org/build/cng/gitlab-sidekiq-ee:v14.2.0

  ## https://docs.gitlab.com/charts/charts/gitlab/gitaly
  - gitaly:
  Git RPC服务，用于处理GitLab发出的所有Git调用
  镜像registry.gitlab.com/gitlab-org/build/cng/gitaly:v14.2.0

  ## https://docs.gitlab.com/charts/charts/gitlab/gitlab-shell
  - gitlab-shell:
  git ssh支持
  image.repository=registry.gitlab.com/gitlab-org/build/cng/gitlab-shell:v13.19.1

  ## https://docs.gitlab.com/charts/charts/gitlab/gitlab-grafana
  - gitlab-grafana:

#mkdir -p /data/gitlab-test-xcw/{gitaly{,2,3},postgresql,prometheus,redis}
#Gitaly存储 200Gi
#3k用户集群需要创建至少3个gitaly服务供praefect代理，默认不启动praefect，创建一个gitaly pv就好。
cat <<\EOF> gitlab-pv-sc.yaml
---
#gitaly1
apiVersion: v1
kind: PersistentVolume
metadata:
  name: gitlab-gitaly-test-xcw
  labels:
    storage: gitaly-data-test-xcw
spec:
  capacity:
    storage: "200Gi"
  accessModes:
    - "ReadWriteMany"
  volumeMode: Filesystem
  local:
    path: /data/gitlab-test-xcw/gitaly
  storageClassName: "gitlab-storageclass-test-xcw"
  nodeAffinity:
    required:
      nodeSelectorTerms:
      - matchExpressions:
        - key: kubernetes.io/hostname
          operator: In
          values:
          - 10.22.0.18
---
#gitaly2
apiVersion: v1
kind: PersistentVolume
metadata:
  name: gitlab-gitaly2-test-xcw
  labels:
    storage: gitaly2-data-test-xcw
spec:
  capacity:
    storage: "200Gi"
  accessModes:
    - "ReadWriteMany"
  volumeMode: Filesystem
  local:
    path: /data/gitlab-test-xcw/gitaly2
  storageClassName: "gitlab-storageclass-test-xcw"
  nodeAffinity:
    required:
      nodeSelectorTerms:
      - matchExpressions:
        - key: kubernetes.io/hostname
          operator: In
          values:
          - 10.22.0.18
---
#gitaly3
apiVersion: v1
kind: PersistentVolume
metadata:
  name: gitlab-gitaly3-test-xcw
  labels:
    storage: gitaly3-data-test-xcw
spec:
  capacity:
    storage: "200Gi"
  accessModes:
    - "ReadWriteMany"
  volumeMode: Filesystem
  local:
    path: /data/gitlab-test-xcw/gitaly3
  storageClassName: "gitlab-storageclass-test-xcw"
  nodeAffinity:
    required:
      nodeSelectorTerms:
      - matchExpressions:
        - key: kubernetes.io/hostname
          operator: In
          values:
          - 10.22.0.18
---
#postgresql
apiVersion: v1
kind: PersistentVolume
metadata:
  name: gitlab-postgresql-test-xcw
  labels:
    storage: postgresql-data-test-xcw
spec:
  capacity:
    storage: "50Gi"
  accessModes:
    - "ReadWriteMany"
  volumeMode: Filesystem
  local:
    path: /data/gitlab-test-xcw/postgresql
  storageClassName: "gitlab-storageclass-test-xcw"
  nodeAffinity:
    required:
      nodeSelectorTerms:
      - matchExpressions:
        - key: kubernetes.io/hostname
          operator: In
          values:
          - 10.22.0.18
---
#redis
apiVersion: v1
kind: PersistentVolume
metadata:
  name: gitlab-redis-test-xcw
  labels:
    storage: redis-data-test-xcw
spec:
  capacity:
    storage: "20Gi"
  accessModes:
    - "ReadWriteMany"
  volumeMode: Filesystem
  local:
    path: /data/gitlab-test-xcw/redis
  storageClassName: "gitlab-storageclass-test-xcw"
  nodeAffinity:
    required:
      nodeSelectorTerms:
      - matchExpressions:
        - key: kubernetes.io/hostname
          operator: In
          values:
          - 10.22.0.18
---
#prometheus
apiVersion: v1
kind: PersistentVolume
metadata:
  name: gitlab-prometheus-test-xcw
  labels:
    storage: prometheus-data-test-xcw
spec:
  capacity:
    storage: "10Gi"
  accessModes:
    - "ReadWriteMany"
  volumeMode: Filesystem
  local:
    path: /data/gitlab-test-xcw/prometheus
  storageClassName: "gitlab-storageclass-test-xcw"
  nodeAffinity:
    required:
      nodeSelectorTerms:
      - matchExpressions:
        - key: kubernetes.io/hostname
          operator: In
          values:
          - 10.22.0.18
---
kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
  name: gitlab-storageclass-test-xcw
provisioner: kubernetes.io/no-provisioner
volumeBindingMode: WaitForFirstConsumer
reclaimPolicy: Retain
EOF

kubectl apply -f gitlab-pv-sc.yaml

kubectl --namespace=gitlab-test-xcw create secret tls cici-com --cert=cici.com.crt --key=cici.com.key

主账号ID 100000xxx
用户名 gitlab-test-xcw 
登录密码 - 
SecretId Axxxx
SecretKey oLxxx

{
    "version": "2.0",
    "statement": [
        {
            "effect": "allow",
            "action": [
                "name/cos:*"
            ],
            "resource": [
                "qcs::cos:ap-beijing:uid/1254024480:gitlab-backup-test-xcw-1254024480/",
                "qcs::cos:ap-beijing:uid/1254024480:gitlab-backup-test-xcw-1254024480/*",
                "qcs::cos:ap-beijing:uid/1254024480:gitlab-backup-tmp-test-xcw-1254024480/",
                "qcs::cos:ap-beijing:uid/1254024480:gitlab-backup-tmp-test-xcw-1254024480/*",
                "qcs::cos:ap-beijing:uid/1254024480:gitlab-artifacts-test-xcw-1254024480/",
                "qcs::cos:ap-beijing:uid/1254024480:gitlab-artifacts-test-xcw-1254024480/*",
                "qcs::cos:ap-beijing:uid/1254024480:gitlab-lfs-test-xcw-1254024480/",
                "qcs::cos:ap-beijing:uid/1254024480:gitlab-lfs-test-xcw-1254024480/*",
                "qcs::cos:ap-beijing:uid/1254024480:gitlab-packages-test-xcw-1254024480/",
                "qcs::cos:ap-beijing:uid/1254024480:gitlab-packages-test-xcw-1254024480/*",
                "qcs::cos:ap-beijing:uid/1254024480:gitlab-pseudonymizer-test-xcw-1254024480/",
                "qcs::cos:ap-beijing:uid/1254024480:gitlab-pseudonymizer-test-xcw-1254024480/*",
                "qcs::cos:ap-beijing:uid/1254024480:gitlab-uploads-test-xcw-1254024480/",
                "qcs::cos:ap-beijing:uid/1254024480:gitlab-uploads-test-xcw-1254024480/*",
                "qcs::cos:ap-beijing:uid/1254024480:gitlab-registry-test-xcw-1254024480/",
                "qcs::cos:ap-beijing:uid/1254024480:gitlab-registry-test-xcw-1254024480/*"
            ]
        }
    ]
}

#统一存储
cat <<EOF> object-storage-test.yaml
provider: AWS
region: ap-beijing
aws_access_key_id: AKIDxxx
aws_secret_access_key: oLxxxx
aws_signature_version: 2
host: "cos.ap-beijing.myqcloud.com"
endpoint: "https://cos.ap-beijing.myqcloud.com"
EOF
kubectl --namespace=gitlab-test-xcw create secret generic gitlab-rails-storage --from-file=connection=object-storage-test.yaml


#镜像存储
cat >registry-test.yaml<<EOF
s3:
  bucket: gitlab-registry-test
  accesskey: AKIDMxxxx
  secretkey: oLaZIxxxx
  regionendpoint: "https://cos.ap-beijing.myqcloud.com"
  region: ap-beijing
EOF
kubectl --namespace=gitlab-test-xcw create secret generic gitlab-registry --from-file=config=registry-test.yaml

#备份
cat <<EOF> s3cfg_cos
[default]
access_key = AKxxx
secret_key = oLaZxxxx
bucket_location = ap-beijing
host_base = cos.ap-beijing.myqcloud.com
host_bucket = cos.ap-beijing.myqcloud.com
signature_v2 = True
EOF
kubectl --namespace=gitlab-test-xcw create secret generic task-runenr-s3-config --from-file=config=s3cfg_cos

cat <<\EOF> pg_redis_ldap_smtp.yaml
---
apiVersion: v1
kind: Secret
metadata: 
  namespace: gitlab-test-xcw
  name: gitlab-test-pg
type: Opaque 
data: 
  password: cG9zdGdyZXM=

---
apiVersion: v1
kind: Secret
metadata: 
  namespace: gitlab-test-xcw
  name: gitlab-test-redis
type: Opaque 
data: 
  password: MTIzNDU2

---
apiVersion: v1
kind: Secret
metadata: 
  namespace: gitlab-test-xcw
  name: gitlab-test-ldap
type: Opaque 
data: 
  password: S3hzWWdKYk52TXdGOEtrcThuZUxiRWtRS1VpYTl4Vks=

---
apiVersion: v1
kind: Secret
metadata: 
  namespace: gitlab-test-xcw
  name: gitlab-test-smtp
type: Opaque 
data: 
  password: WiNwczklMGo=
EOF
kubectl apply -f pg_redis_ldap_smtp.yaml

#使用内部生成的redis postgresql，不需要使用这里创建的密钥
#如果内部生成的redis postgresql使用这里的密钥，需要指定下面参数及服务器内部密码参数
#   --set global.psql.password.secret=gitlab-test-pg \
#   --set global.psql.password.key='password' \
#global.postgresql.postgresqlPassword
#global.postgresql.existingSecret

--set nginx-ingress.controller.service.annotations.service\\.kubernetes\\.io\\/qcloud-loadbalancer-internal-subnetid=subnet-fubxophz

--set global.hosts.externalIP=10.22.0.71 \
--set nginx-ingress.controller.service.annotations.service\\.kubernetes\\.io\\/qcloud-loadbalancer-internal-subnetid=subnet-fubxophz \
--set nginx-ingress.controller.service.annotations.kubernetes\\.io\\/ingress\\.existLbId=<loadbalanceid> \

docker push docker.cici.com/library/gitlab/gitlab-container-registry:v3.9.0-gitlab
docker push docker.cici.com/library/gitlab/gitlab-shell:v13.19.1
docker push docker.cici.com/library/gitlab/gitlab-workhorse-ee:v14.2.1
docker push docker.cici.com/library/gitlab/gitlab-toolbox-ee:v14.2.1
docker push docker.cici.com/library/gitlab/gitlab-webservice-ee:v14.2.1
docker push docker.cici.com/library/gitlab/gitlab-sidekiq-ee:v14.2.1
docker push docker.cici.com/library/gitlab/gitaly:v14.2.1
docker push docker.cici.com/library/gitlab/gitlab-exporter:11.2.0
docker push docker.cici.com/library/gitlab/kubectl:1.16.15
docker push docker.cici.com/library/gitlab/postgresql:12.7.0
docker push docker.cici.com/library/gitlab/gitlab-runner:alpine-v14.2.0
docker push docker.cici.com/library/gitlab/alpine-certificates:20191127-r2
docker push docker.cici.com/library/gitlab/nginx-ingress-controller:v0.41.2
docker push docker.cici.com/library/gitlab/redis:6.0.9-debian-10-r0
docker push docker.cici.com/library/gitlab/defaultbackend-amd64:1.5
docker push docker.cici.com/library/gitlab/busybox:latest

cat <<\EOF> create_gitlab_secrets.sh
#!/bin/bash
# --- 创建持久卷
kubectl apply -f gitlab-pv-sc.yaml

# --- 创建域名证书tls secret
kubectl --namespace=gitlab-test-xcw create secret tls cici-com --cert=cici.com.crt --key=cici.com.key

# --- 启动外部对象存储
#统一存储
kubectl --namespace=gitlab-test-xcw create secret generic gitlab-rails-storage --from-file=connection=object-storage-test.yaml
#:<<eof
#5.2.1版本前没有整合配置，需要单独执行
kubectl --namespace=gitlab-test-xcw create secret generic gitlab-lfs --from-file=connection=object-storage-test.yaml
kubectl --namespace=gitlab-test-xcw create secret generic gitlab-artifacts --from-file=connection=object-storage-test.yaml
kubectl --namespace=gitlab-test-xcw create secret generic gitlab-uploads --from-file=connection=object-storage-test.yaml
kubectl --namespace=gitlab-test-xcw create secret generic gitlab-packages --from-file=connection=object-storage-test.yaml
#kubectl --namespace=gitlab-test-xcw create secret generic gitlab-externaldiffs --from-file=connection=object-storage-test.yaml
kubectl --namespace=gitlab-test-xcw create secret generic gitlab-pseudonymizer --from-file=connection=object-storage-test.yaml
#eof

#镜像存储
kubectl --namespace=gitlab-test-xcw create secret generic gitlab-registry --from-file=config=registry-test.yaml
#备份
kubectl --namespace=gitlab-test-xcw create secret generic task-runenr-s3-config --from-file=config=s3cfg_cos

# --- 服务连接密码
kubectl apply -f pg_redis_ldap_smtp.yaml
EOF

helm upgrade --install  -n gitlab-test-xcw  gitlab-test-xcw gitlab/gitlab \
     --timeout 600s \
     --version=5.2.1 \
     --dry-run \
     `#--域名--` \
     `#https://docs.gitlab.com/charts/installation/deployment.html#networking-and-dns` \
     `#主机域名https://docs.gitlab.com/charts/charts/globals#configure-host-settings` \
     --set global.hosts.gitlab.name='git-test-xcw.cici.com' \
     --set global.hosts.gitlab.https=true \
     --set nginx-ingress.controller.service.annotations.service\\.kubernetes\\.io\\/qcloud-loadbalancer-internal-subnetid=subnet-fubxophz \
     `#--持久存储--` \
     `#https://docs.gitlab.com/charts/installation/storage.html` \
     `#动态卷 kubectl apply -f gitlab_storageclass.yaml` \
     --set gitlab.gitaly.persistence.storageClass=gitlab-storageclass-test-xcw \
     --set gitlab.gitaly.persistence.size=200Gi \
     --set gitlab.gitaly.persistence.accessMode=ReadWriteMany \
     --set postgresql.persistence.storageClass=gitlab-storageclass-test-xcw \
     --set postgresql.persistence.size=50Gi \
     --set postgresql.persistence.accessModes={ReadWriteMany} \
     --set redis.master.persistence.storageClass=gitlab-storageclass-test-xcw \
     --set redis.master.persistence.size=20Gi \
     --set redis.master.persistence.accessModes={ReadWriteMany} \
     `#--tls证书管理--` \
     `#https://docs.gitlab.com/charts/installation/tls.html` \
     `#导入通配域名证书 https://docs.gitlab.com/charts/installation/tls.html#option-2-use-your-own-wildcard-certificate` \
     `#kubectl --namespace=gitlab-test-xcw create secret tls cici-com-test-xcw --cert=cici.com.crt --key=cici.com.key` \
     --set certmanager.install=false \
     --set global.ingress.configureCertmanager=false \
     --set global.ingress.tls.secretName=cici-com \
     `#--外部postgresql--` \
     `#https://docs.gitlab.com/charts/installation/deployment.html#postgresql` \
     `#密码可通过secret方式导入` \
     `#kubectl -n gitlab create secret generic gitalb-postgresql-password --from-literal=postgresql-password=$(head -c 512 /dev/urandom | LC_CTYPE=C tr -cd 'a-zA-Z0-9' | head -c 64)` \
     `#--set postgresql.install=false` \
     `#--set global.psql.host=production.postgress.hostname.local` \
     `#--set global.psql.username='postgres'` \
     `#--set global.psql.password.secret=gitlab-test-pg` \
     `#--set global.psql.password.key='password'` \
     `#--set global.psql.database='gitlabtest'` \
     `#--外部redis--` \
     `#https://docs.gitlab.com/charts/charts/globals.html#configure-redis-settings` \
     `#可选单点、哨兵、集群方式部署` \
     `#kubectl create secret generic gitlab-redis-secret --from-literal=password=$(head -c 512 /dev/urandom | LC_CTYPE=C tr -cd 'a-zA-Z0-9' | head -c 64)` \
     `#--set redis.install=false` \
     `#--set global.redis.host='redis.example.com'` \
     `#--set global.redis.password.secret=gitlab-test-redis` \
     `#--set global.redis.password.key='password'` \
     `#--启动外部对象存储，不使用内部自建的minio分布式存储，需要提前生成连接密钥--` \
     `#https://docs.gitlab.com/charts/advanced/external-object-storage/index.html` \
     `#完整样例 https://gitlab.com/gitlab-org/charts/gitlab/blob/master/examples/values-external-objectstorage.yaml` \
     `#统一存储，https://docs.gitlab.com/charts/charts/globals.html#consolidated-object-storage` \
     `#存储启动默认值https://docs.gitlab.com/12.10/charts/charts/globals.html#configure-minio-settings` \
     `#gitlab各存储桶连接样例，https://gitlab.com/gitlab-org/charts/gitlab/-/blob/master/examples/objectstorage/rails.s3.yaml` \
     `#kubectl --namespace=gitlab-test-xcw create secret generic gitlab-rails-storage --from-file=connection=rails.yaml` \
     --set global.minio.enabled=false \
     --set global.appConfig.object_store.enabled=true \
     --set global.appConfig.object_store.connection.secret=gitlab-rails-storage \
     --set global.appConfig.object_store.connection.key=connection \
     --set global.appConfig.lfs.bucket=gitlab-lfs-test-xcw-1254024480 \
     --set global.appConfig.artifacts.bucket=gitlab-artifacts-test-xcw-1254024480 \
     --set global.appConfig.uploads.bucket=gitlab-uploads-test-xcw-1254024480 \
     --set global.appConfig.packages.bucket=gitlab-packages-test-xcw-1254024480 \
     --set global.appConfig.pseudonymizer.bucket=gitlab-pseudonymizer-test-xcw-1254024480 \
     `#docker镜像库` \
     `#https://docs.gitlab.com/charts/advanced/external-object-storage/index.html#docker-registry-images` \
     `#docker镜像存储地址样例 https://gitlab.com/gitlab-org/charts/gitlab/-/blob/master/examples/objectstorage/registry.s3.yaml` \
     `#kubectl --namespace=gitlab-test-xcw create secret generic gitlab-registry --from-file=config=registry-test.yaml` \
     --set registry.storage.secret=gitlab-registry \
     --set registry.storage.key=config \
     --set global.registry.bucket=gitlab-registry-test-xcw-1254024480 \
     `#备份` \
     `#https://docs.gitlab.com/charts/advanced/external-object-storage/index.html#backups` \
     `#备份存储配置样例 https://docs.gitlab.com/charts/advanced/external-object-storage/index.html#backups-storage-example` \
     `#kubectl --namespace=gitlab-test-xcw create secret generic task-runenr-s3-config --from-file=config=s3cfg_cos` \
     --set global.appConfig.backups.bucket=gitlab-backup-test-xcw-1254024480 \
     --set global.appConfig.backups.tmpBucket=gitlab-backup-tmp-test-xcw-1254024480 \
     --set gitlab.task-runner.backups.objectStorage.config.secret=task-runenr-s3-config \
     --set gitlab.task-runner.backups.objectStorage.config.key=config \
     `#--监控--` \
     `#https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus#configuration` \
     --set prometheus.server.persistentVolume.storageClass=gitlab-storageclass-test-xcw \
     --set prometheus.server.persistentVolume.accessModes={ReadWriteMany} \
     --set prometheus.server.persistentVolume.size=10Gi \
     `#--外发电子邮件--` \
     `#https://docs.gitlab.com/charts/installation/command-line-options.html#outgoing-email-configuration` \
     `#kubectl --namespace=gitlab create secret generic gitlab-smtp-password --from-literal=password=yourpasswordhere` \
     --set [email protected] \
     --set global.email.display_name='GitLab' \
     --set global.smtp.enabled=true \
     --set global.smtp.address=smtp.exmail.qq.com \
     --set global.smtp.tls=true \
     --set global.smtp.port=456 \
     --set global.smtp.user_name="[email protected]" \
     --set global.smtp.password.secret="gitlab-test-smtp" \
     --set global.smtp.password.key=password \
     --set global.smtp.authentication="login" \
     --set global.smtp.starttls_auto=true \
     --set global.smtp.pool=true \
     `#--rbac--` \
     `#https://docs.gitlab.com/charts/installation/deployment.html#rbac` \
     --set certmanager.rbac.create=false \
     `#--cpu和内存配置--` \
     `#最小化配置 https://docs.gitlab.com/charts/installation/deployment.html#cpu-and-ram-resource-requirements` \
     --set gitlab-runner.install=false \
     --set nginx-ingress.controller.replicaCount=3 \
     --set nginx-ingress.controller.minAvailable=2 \
     --set nginx-ingress.defaultBackend.replicaCount=2 \
     `#时区` \
     --set global.time_zone=Asia/Shanghai \
     `#--ldap配置--` \
     `#https://docs.gitlab.com/charts/charts/globals.html#ldap` \
     `#https://docs.gitlab.com/ee/administration/auth/ldap/` \
     `#kubectl --namespace=gitlab create secret generic gitlab-ldap-password --from-literal=password=yourpasswordhere` \
     --set global.appConfig.ldap.servers.main.label='LDAP' \
     --set global.appConfig.ldap.servers.main.host='ldap.cici.com' \
     --set global.appConfig.ldap.servers.main.port='389' \
     --set global.appConfig.ldap.servers.main.uid='cn' \
     --set global.appConfig.ldap.servers.main.bind_dn='cn=gitlab_admin\,ou=sys_admins\,dc=cici\,dc=com' \
     --set global.appConfig.ldap.servers.main.base='ou=staff\,dc=cici\,dc=com' \
     --set global.appConfig.ldap.servers.main.encryption='plain' \
     --set global.appConfig.ldap.servers.main.password.secret='gitlab-test-ldap' \
     --set global.appConfig.ldap.servers.main.password.key='password' \
     --set global.appConfig.ldap.servers.main.user_filter='(&(memberOf=cn=rds\,ou=groups\,dc=cici\,dc=com))' \
     --set global.appConfig.ldap.servers.main.attributes.username='[cn]' \
     --set global.appConfig.ldap.servers.main.attributes.email='[mail\, email]' \
     --set global.appConfig.ldap.servers.main.attributes.name='displayName' \
     --set global.appConfig.ldap.servers.main.attributes.first_name='givenName' \
     --set global.appConfig.ldap.servers.main.attributes.last_name='sn' \
     --set global.appConfig.ldap.servers.main.allow_username_or_email_login=true \
     `#--omniauth认证--` \
     `#https://docs.gitlab.com/charts/charts/globals#omniauth` \
     --set global.appConfig.omniauth.enabled=true \
     `#--自动备份--` \
     `#https://docs.gitlab.com/charts/backup-restore/index.html` \
     --set gitlab.task-runner.backups.cron.enabled=true \
     --set gitlab.task-runner.backups.cron.schedule='0 2 * * *' \
     `#--上传限制--` \
     --set global.ingress.proxyBodySize=10Gi \
     `#--qos服务质量--` \
     --set nginx-ingress.controller.resources.requests.cpu='100m' \
     --set nginx-ingress.controller.resources.requests.memory='100Mi' \
     `#--3k用户--` \
     `#开启gitaly代理pracefect，启动gitaly集群` \
     `#praefect启动限制，安装后创建对应库https://docs.gitlab.com/charts/charts/gitlab/praefect/index.html` \
     --set global.praefect.enabled=true \
     `#--自定义镜像--` \
     --set global.kubectl.image.repository=docker.cici.com/library/gitlab/kubectl \
     --set global.kubectl.image.tag="1.16.15" \
     --set global.certificates.image.repository=docker.cici.com/library/gitlab/alpine-certificates \
     --set global.certificates.image.tag="20191127-r2" \
     --set nginx-ingress.controller.image.repository=docker.cici.com/library/gitlab/nginx-ingress-controller \
     --set nginx-ingress.controller.image.tag="v0.41.2" \
     --set nginx-ingress.controller.image.digest="" \
     --set nginx-ingress.defaultBackend.image.repository=docker.cici.com/library/gitlab/defaultbackend-amd64 \
     --set nginx-ingress.defaultBackend.image.tag="1.5" \
     --set gitlab.task-runner.image.repository=docker.cici.com/library/gitlab/gitlab-toolbox-ee \
     --set gitlab.task-runner.image.tag="v14.2.1" \
     --set gitlab.migrations.image.repository=docker.cici.com/library/gitlab/gitlab-toolbox-ee \
     --set gitlab.migrations.image.tag="v14.2.1" \
     --set gitlab.gitaly.image.repository=docker.cici.com/library/gitlab/gitaly \
     --set gitlab.gitaly.image.tag="v14.2.1" \
     --set gitlab.praefect.image.repository=docker.cici.com/library/gitlab/gitaly \
     --set gitlab.praefect.image.tag="v14.2.1" \
     --set gitlab.sidekiq.image.repository=docker.cici.com/library/gitlab/gitlab-sidekiq-ee \
     --set gitlab.sidekiq.image.tag="v14.2.1" \
     --set gitlab.gitlab-shell.image.repository=docker.cici.com/library/gitlab/gitlab-shell \
     --set gitlab.gitlab-shell.image.tag="v13.19.1" \
     --set gitlab.webservice.image.repository=docker.cici.com/library/gitlab/gitlab-webservice-ee \
     --set gitlab.webservice.image.tag="v14.2.1" \
     --set gitlab.webservice.workhorse.image=docker.cici.com/library/gitlab/gitlab-workhorse-ee \
     --set gitlab.webservice.workhorse.tag="v14.2.1" \
     --set registry.image.repository=docker.cici.com/library/gitlab/gitlab-container-registry \
     --set registry.image.tag="v3.9.0-gitlab" \
     --set postgresql.image.repository=docker.cici.com/library/gitlab/postgresql \
     --set postgresql.image.tag="12.7.0" \
     --set redis.image.repository=docker.cici.com/library/gitlab/redis \
     --set redis.image.tag="6.0.9-debian-10-r0" \
     --set gitlab-runner.image=docker.cici.com/library/gitlab/gitlab-runner:alpine-v14.2.0 \
     --set gitlab.gitlab-exporter.image.repository=docker.cici.com/library/gitlab/gitlab-exporter \
     --set gitlab.gitlab-exporter.image.tag="11.2.0" \
     --set global.busybox.image.repository=docker.cici.com/library/gitlab/busybox \
     --set global.busybox.image.tag=latest

#初次登录
#kubectl -n gitlab-test-xcw  get secret gitlab-test-xcw-gitlab-initial-root-password -ojsonpath='{.data.password}' | base64 --decode ; echo

#1登录到您的数据库实例
kubectl -n gitlab-test-xcw  exec -it $(kubectl -n gitlab-test-xcw get pods -l app=postgresql -o custom-columns=NAME:.metadata.name --no-headers) -- bash
PGPASSWORD=$(cat $POSTGRES_POSTGRES_PASSWORD_FILE) psql -U postgres -d template1
#2创建数据库用户
CREATE ROLE praefect WITH LOGIN;
#3设置数据库用户密码
#获取密码
kubectl -n gitlab-test-xcw get secret  gitlab-test-xcw-praefect-dbsecret -o jsonpath="{.data.secret}" | base64 --decode
#在psql提示中设置密码
\password praefect
#4创建数据库：
CREATE DATABASE praefect WITH OWNER praefect;

helm uninstall  -n gitlab-test-xcw  gitlab-test-xcw

cat <<\EOF> delete_gitlab_secrets.sh
#!/bin/bash

NS=gitlab-test-xcw
RELEASE_NAME=gitlab-test-xcw
kubectl -n $NS delete secrets \
        gitlab-rails-storage \
        gitlab-registry \
        task-runenr-s3-config \
        gitlab-test-pg \
        gitlab-test-redis \
        gitlab-test-ldap \
        gitlab-test-smtp \
        cici-com \
        gitlab-artifacts \
        gitlab-lfs \
        gitlab-packages \
        gitlab-uploads \
        gitlab-pseudonymizer \
        `#自动创建的` \
        ${RELEASE_NAME}-gitaly-secret \
        ${RELEASE_NAME}-gitlab-initial-root-password \
        ${RELEASE_NAME}-gitlab-runner-secret \
        ${RELEASE_NAME}-gitlab-shell-host-keys \
        ${RELEASE_NAME}-gitlab-shell-secret \
        ${RELEASE_NAME}-gitlab-workhorse-secret \
        ${RELEASE_NAME}-rails-secret \
        ${RELEASE_NAME}-registry-httpsecret \
        ${RELEASE_NAME}-registry-notification \
        ${RELEASE_NAME}-registry-secret \
        ${RELEASE_NAME}-postgresql-password \
        ${RELEASE_NAME}-redis-secret

kubectl -n $NS delete cm ingress-controller-leader-gitlab-test-xcw-nginx
echo "delete secrets Complete"

echo "请求查看对应PVPVC绑定情况，可删除"
kubectl get pv |grep test-xcw
kubectl -n gitlab-test-xcw get pvc
kubectl get sc |grep gitlab
EOF

helm get values -n gitlab-test-xcw  gitlab-test-xcw  >customize_conf/values-test.yaml

cat <<\EOF> deploy-gitlab-test.sh
helm upgrade --install -n gitlab-test-xcw gitlab-test-xcw gitlab/gitlab \
     --timeout 600s \
     --version=5.2.1 \
     --dry-run \
     -f customize_conf/values-test.yaml \
     --set global.hosts.gitlab.name='git-test-xcw.cici.com' \
     `#--外部postgresql--` \
     `#--set postgresql.install=false` \
     `#--set global.psql.host=production.postgress.hostname.local` \
     `#--set global.psql.password.secret=gitlab-test-pg` \
     `#--set global.psql.password.key='password'` \
     `#--外部redis--` \
     `#--set redis.enabled=false` \
     `#--set global.redis.host='redis.example.com'` \
     `#--set global.redis.password.secret=gitlab-test-redis` \
     `#--set global.redis.password.key='password'` \
     --set gitlab.migrations.enabled=true 
EOF

mkdir /data/gitlab-gpt/{results,config/{environments,projects}} -p
cd  /data/gitlab-gpt/config/environments
cat <<\EOF> 10k.json
{
  "environment": {
    "name": "10k",
    "url": "https://git-test-xcw.cici.com",
    "user": "gpt-admin",
    "config": {
      "latency": "0"
    },
    "storage_nodes": ["default"]
  },
  "gpt_data": {
    "root_group": "gpt",
    "large_projects": {
      "group": "large_projects",
      "project": "gitlabhq"
    },
    "many_groups_and_projects": {
      "group": "many_groups_and_projects",
      "subgroups": 250,
      "subgroup_prefix": "gpt-subgroup-",
      "projects": 10,
      "project_prefix": "gpt-project-"
    }
  }
}
EOF

: <<eof
通常只需要改变：name，url，user和storage_nodes
name- 环境的名称。主要用于输出和结果
url- 环境的完整 URL，用于所有测试和其他区域。
user- 作为创建用户步骤的一部分准备的用户名称。
storage_nodes-目标 GitLab 环境中的存储库存储阵列
eof

# 大项目比较大，ingress开放上传限制
docker run --rm -it \
  -e ACCESS_TOKEN=MnLzZsfPNF-Bgtmomgsm \
  -v /data/gitlab-gpt/config:/config \
  -v /data/gitlab-gpt/results:/results \
  --add-host git-test-xcw.cici.com:10.22.0.207 \
  gitlab/gpt-data-generator \
  --environment 10k.json

#--no-horizontal #不导入水平数据
#--no-vertical   #不导入垂直数据

#导入自定义项目docker run --rm -it   -e ACCESS_TOKEN=MnLzZsfPNF-Bgtmomgsm   -v /data/gitlab-gpt/config:/config   -v /data/gitlab-gpt/results:/results   --add-host git-test-xcw.cici.com:10.22.0.207   gitlab/gpt-data-generator   --environment 10k.json  --no-horizontal  --large-project-tarball=/config/gitlabhq_export_13.0.0.tar.gz

:<<EOF
目标环境用户数使用以下选项文件：
1千 - 60s_20rps.json
2k - 60s_40rps.json
3k - 60s_60rps.json
5k - 60s_100rps.json
10k - 60s_200rps.json
EOF

mkdir /data/gitlab-gpt/{results,tests,config/{environments,options,projects}} -p
cd /data/gitlab-gpt/
cat <<\EOF> config/options/60s_40rps.json
{
  "stages": [
    { "duration": "5s", "target": 40 },
    { "duration": "50s", "target": 40 },
    { "duration": "5s", "target": 0 }
  ],
  "rps": 40,
  "batchPerHost": 0
}
EOF

cat <<\EOF> config/options/60s_60rps.json
{
  "stages": [
    { "duration": "5s", "target": 60 },
    { "duration": "50s", "target": 60 },
    { "duration": "5s", "target": 0 }
  ],
  "rps": 60,
  "batchPerHost": 0
}
EOF

cat <<\EOF> tests/api_v4_projects_project.js
import http from "k6/http";
import { group } from "k6";
import { Rate } from "k6/metrics";
import { logError, getRpsThresholds, getTtfbThreshold, getLargeProjects, selectRandom } from "../../lib/gpt_k6_modules.js";

export let rpsThresholds = getRpsThresholds()
export let ttfbThreshold = getTtfbThreshold()
export let successRate = new Rate("successful_requests")
export let options = {
  thresholds: {
    "successful_requests": [`rate>${__ENV.SUCCESS_RATE_THRESHOLD}`],
    "http_req_waiting": [`p(90)<${ttfbThreshold}`],
    "http_reqs": [`count>=${rpsThresholds['count']}`]
  }
};

export let projects = getLargeProjects(['encoded_path']);

export function setup() {
  console.log('')
  console.log(`RPS Threshold: ${rpsThresholds['mean']}/s (${rpsThresholds['count']})`)
  console.log(`TTFB P90 Threshold: ${ttfbThreshold}ms`)
  console.log(`Success Rate Threshold: ${parseFloat(__ENV.SUCCESS_RATE_THRESHOLD)*100}%`)
}

export default function() {
  group("API - Project Overview", function() {
    let project = selectRandom(projects);

    let params = { headers: { "Accept": "application/json", "PRIVATE-TOKEN": `${__ENV.ACCESS_TOKEN}` } };
    let res = http.get(`${__ENV.ENVIRONMENT_URL}/api/v4/projects/${project['encoded_path']}`, params);
    /20(0|1)/.test(res.status) ? successRate.add(true) : (successRate.add(false), logError(res));
  });
}
EOF

#整个跑测试耗时久，可以添加--test参数单独一块测试
docker run --rm -it \
  -e ACCESS_TOKEN=MnLzZsfPNF-Bgtmomgsm \
  -v /data/gitlab-gpt/config:/config \
  -v /data/gitlab-gpt/tests:/tests \
  -v /data/gitlab-gpt/results:/results \
  --add-host git-test-xcw.cici.com:10.22.0.207 \
  gitlab/gitlab-performance-tool \
  --environment 10k.json \
  --options 60s_40rps.json --tests api_v4_groups_projects.js

#3k用户
docker run --rm -it \
  -e ACCESS_TOKEN=MnLzZsfPNF-Bgtmomgsm \
  -v /data/gitlab-gpt/config:/config \
  -v /data/gitlab-gpt/tests:/tests \
  -v /data/gitlab-gpt/results:/results \
  --add-host git-test-xcw.cici.com:10.22.0.207 \
  gitlab/gitlab-performance-tool \
  --environment 10k.json \
  --options 60s_60rps.json

#备份
#https://docs.gitlab.com/charts/backup-restore/backup.html
kubectl -n gitlab-test-xcw exec -it <gitlab task-runner pod> -- backup-utility

#备份密钥
#https://docs.gitlab.com/charts/backup-restore/backup.html#backup-the-secrets
kubectl -n gitlab get secrets | grep rails-secret
kubectl -n gitlab get secrets <rails-secret-name> -o jsonpath="{.data['secrets\.yml']}" | base64 --decode > secrets.yaml

#cat docker-compose.yml
version: '3.9'
services:
  postgresql:
    image: 'docker.io/bitnami/postgresql:10'
    restart: always
    ports:
      - 15432:5432
      #- POSTGRESQL_USERNAME=cmn_git_web_rw
    environment:
      - TZ=Asia/Shanghai
      - ALLOW_EMPTY_PASSWORD=yes
      - POSTGRESQL_PASSWORD=postgres
      - POSTGRESQL_DATABASE=cici_cmn_git
    volumes:
      - 'postgresql_data:/bitnami/postgresql'
    networks:
    - app-pr

  redis:
    image: 'docker.io/bitnami/redis:6.0-debian-10'
    restart: always
    ports:
      - 16379:6379
    environment:
      - TZ=Asia/Shanghai
      - REDIS_PASSWORD=123456
      - REDIS_DISABLE_COMMANDS=FLUSHDB,FLUSHALL
    volumes:
      - 'redis_data:/bitnami'
    networks:
      - app-pr
volumes:
  postgresql_data:
    driver: local
  redis_data:
    driver: local
networks:
  app-pr:
    driver: bridge

#docker-compose up -d
#卸载    
#docker-compose stop
#docker-compose rm -v    
#docker-compose down -v #删除数据卷

#cat my-up-down.sh
#!/bin/bash
#
FILE=$1
#DOWN_FILE=1631823001_2021_09_16_12.3.1-ee_gitlab_backup.tar
#down  avg 8分26秒
#\cp ~/.cos.conf.prod  ~/.cos.conf
#coscmd download  ${FILE} ./

#up  avg 13分57秒
\cp ~/.cos.conf.bedin  ~/.cos.conf
coscmd upload  ${FILE}

#查找版本
helm search repo -l gitlab/gitlab
#下载（可选，用来对比差异）
helm  pull  gitlab/gitlab --version=2.3.2


# 安装升级12.3.1版本，如是新环境需导入备份数据
cat <<\EOF> up-gitlab-2.3.2-test.sh
helm upgrade --install --force --namespace=gitlab-test-xcw gitlab-test-xcw  gitlab/gitlab \
     --timeout 600s \
     --version=2.3.2 \
     `#--dry-run` \
     --debug \
     -f values-customize-2.3.2-test.yaml \
     --set global.hosts.gitlab.name='git-test-xcw.cici.com' \
     `#--外部postgresql--` \
     --set postgresql.install=false \
     --set global.psql.host='10.0.0.59' \
     --set global.psql.port=15432 \
     --set global.psql.password.secret=gitlab-test-pg \
     --set global.psql.password.key='password' \
     `#--外部redis--` \
     --set redis.enabled=false \
     --set global.redis.host='10.0.0.59' \
     --set global.redis.port=16379 \
     --set global.redis.password.secret=gitlab-test-redis \
     --set global.redis.password.key='password' \
     `#--set gitlab.migrations.enabled=true`
EOF

#upgrade to GitLab Helm Chart version 2.6.0 before upgrading to 3.3.13
helm upgrade --install --force --namespace=gitlab-test-xcw gitlab-test-xcw  gitlab/gitlab \
     --timeout 600s \
     --version=3.3.13 \
     `#--dry-run` \
     --debug \
     -f values-customize-3.3.13-test.yaml \
     --set global.hosts.gitlab.name='git-test-xcw.cici.com' \
     `#--外部postgresql--` \
     --set postgresql.install=false \
     --set global.psql.host='10.0.0.59' \
     --set global.psql.port=35432 \
     --set global.psql.password.secret=gitlab-test-pg \
     --set global.psql.password.key='password' \
     `#--外部redis--` \
     --set redis.install=false \
     --set global.redis.host='10.0.0.59' \
     --set global.redis.port=16379 \
     --set global.redis.password.secret=gitlab-test-redis \
     --set global.redis.password.key='password' \
     `#--set gitlab.migrations.enabled=true`

kubectl -n gitlab get secrets | grep rails-secret
kubectl -n gitlab get secrets <rails-secret-name> -o jsonpath="{.data['secrets\.yml']}" | base64 --decode > secrets.yaml

#切到升级的k8s环境
kubectl config use-context k8s-test-context
#查找 rails secrets 的对象名称
kubectl -n gitlab-test-xcw get secrets | grep rails-secret
#删除现有的秘密
kubectl -n gitlab-test-xcw delete secret gitlab-test-xcw-rails-secret
#使用与旧密钥相同的名称创建新密钥，并传入您的本地 YAML 文件
#local-yaml-filepath为备份时导出的rails的secrets文件 
kubectl -n gitlab-test-xcw create secret generic gitlab-test-xcw-rails-secret --from-file=secrets.yml=<local-yaml-filepath>

#如果找不到原来的秘钥文件了，可以重置秘钥
#gitlab-rails console
#>ApplicationSetting.current.reset_runners_registration_token!
#=>true

helm_release_name=gitlab-test-xcw
kubectl -n gitlab-test-xcw delete pods -lapp=sidekiq,release=${helm_release_name}
#v5.2.1 unicorn 改为webservice
kubectl -n gitlab-test-xcw delete pods -lapp=unicorn,release=${helm_release_name}
kubectl -n gitlab-test-xcw delete pods -lapp=task-runner,release=${helm_release_name}

#恢复方式1：bucket桶中读取tarball文件，默认
kubectl exec <Task Runner pod name> -it -- backup-utility --restore -t <timestamp>_<version>

#恢复方式2：url
#您可以提供本地路径作为 URL，只要它采用以下格式： file://<path>
kubectl exec <Task Runner pod name> -it -- backup-utility --restore -f <URL>