Linux: AWS使用

#角色名
role_name=AmazonEKS_EBS_CSI_DriverRole-ap-south-1-prod
#EKS集群名
c_n=prod-eks

eksctl create iamserviceaccount \
        --name ebs-csi-controller-sa \
        --namespace kube-system \
        --cluster ${c_n} \
        --role-name ${role_name} \
        --role-only \
        --attach-policy-arn arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy \
        --approve

mkdir gp3
cd gp3

cat <<\EOF> gp3.yaml
kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
  name: gp3
  annotations:
    storageclass.kubernetes.io/is-default-class: "true"
allowVolumeExpansion: true
provisioner: ebs.csi.aws.com
volumeBindingMode: WaitForFirstConsumer
parameters:
  type: gp3
EOF

# https://kubernetes.io/docs/tasks/administer-cluster/change-default-storage-class/
kubectl apply -f gp3.yaml
kubectl patch storageclass gp2 -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"false"}}}'

args:
controller
--endpoint=$(CSI_ENDPOINT)
--logtostderr
--v=2
--extra-tags=Techteam=RC,Environment=ops,BusinessUnit=RC,IgnoreCostAdvisor=true,Ignoreoldgen=true,Application=ops,Cluster=ops-eks,Name=ops-ebs

mkdir awsloadbalance
cd  awsloadbalance

#https://docs.aws.amazon.com/zh_cn/eks/latest/userguide/lbc-manifest.html

#步骤 1：配置 IAM

curl -O https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.7.2/docs/install/iam_policy.json

##已经存在

aws iam create-policy \
    --policy-name AWSLoadBalancerControllerIAMPolicy \
    --policy-document file://iam_policy.json

##使用 eksctl 创建 IAM 角色
#修改角色名，集群名，策略arn
cluster_name=prod-eks
role_name=AmazonEKSLoadBalancerControllerRole-prod-eks
account_id=<ACCOUNT_ID>

eksctl create iamserviceaccount \
    --cluster=<cluster-name> \
    --namespace=kube-system \
    --name=aws-load-balancer-controller \
    --attach-policy-arn=arn:aws:iam::<AWS_ACCOUNT_ID>:policy/AWSLoadBalancerControllerIAMPolicy \
    --override-existing-serviceaccounts \
    --region <aws-region-code> \
    --approve

#步骤 2：安装 cert-manager

kubectl apply \
    --validate=false \
    -f https://github.com/jetstack/cert-manager/releases/download/v1.13.5/cert-manager.yaml

#步骤 3：安装 AWS Load Balancer Controller

curl -Lo v2_7_2_full.yaml https://github.com/kubernetes-sigs/aws-load-balancer-controller/releases/download/v2.7.2/v2_7_2_full.yaml

sed -i.bak -e '612,620d' ./v2_7_2_full.yaml  #按官方文档操作，

sed -i.bak -e 's|your-cluster-name|my-cluster|' ./v2_7_2_full.yaml # my-cluster 改成集群名
#添加--aws-vpc-id=vpc-xxxxxxxx

kubectl apply -f v2_7_2_full.yaml

curl -Lo v2_7_2_ingclass.yaml https://github.com/kubernetes-sigs/aws-load-balancer-controller/releases/download/v2.7.2/v2_7_2_ingclass.yaml

kubectl apply -f v2_7_2_ingclass.yaml

#或者helm安装
helm upgrade --install aws-load-balancer-controller eks/aws-load-balancer-controller \
  -n kube-system \
  --set clusterName=${cluster_name} \
  --set serviceAccount.create=false \
  --set serviceAccount.name=aws-load-balancer-controller \
  --set vpcId=vpc-07dbd3d1e6db2f452 \
  --version 1.13.0

#步骤 4：验证控制器是否已安装

kubectl get deployment -n kube-system aws-load-balancer-controller

export cluster_name=prod-eks
export region_name=ap-south-1

vpc_id=$(aws eks describe-cluster \
    --name $cluster_name \
    --query "cluster.resourcesVpcConfig.vpcId" \
    --output text)

echo $vpc_id

cidr_range=$(aws ec2 describe-vpcs \
    --vpc-ids $vpc_id \
    --query "Vpcs[].CidrBlock" \
    --output text \
    --region $region_name)

echo $cidr_range

security_group_id=$(aws ec2 create-security-group \
    --group-name MyEfsSecurityGroup \
    --description "My EFS security group" \
    --vpc-id $vpc_id \
    --output text)

echo $security_group_id
security_group_id=`echo $security_group_id |awk '{print $1}'`
echo $security_group_id

aws ec2 authorize-security-group-ingress \
    --group-id $security_group_id \
    --protocol tcp \
    --port 2049 \
    --cidr $cidr_range

#创建EFS实例
file_system_id=$(aws efs create-file-system \
    --region ${region_name} \
    --performance-mode generalPurpose \
    --query 'FileSystemId' \
    --output text)

echo $file_system_id
echo $security_group_id

#配置挂载点网络关联
subnet1=subnet-0789f35e7ff7e10da
subnet2=subnet-0ddfcc08a775ffc3c
subnet3=subnet-0d8f55ad4d534b641

aws efs create-mount-target \
    --file-system-id $file_system_id \
    --subnet-id $subnet1 \
    --security-groups $security_group_id

aws efs create-mount-target \
    --file-system-id $file_system_id \
    --subnet-id $subnet2 \
    --security-groups $security_group_id

aws efs create-mount-target \
    --file-system-id $file_system_id \
    --subnet-id $subnet3 \
    --security-groups $security_group_id

#EFS控制台创建接入点， 默认路径/ ，可设置 /{业务名}

#获取文件系统id
efs_id=`aws efs describe-file-systems --query "FileSystems[*].FileSystemId" --output text`

mkdir efs ; cd efs

cat <<\EOF> storageclass.yaml 
kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
  name: efs-sc
provisioner: efs.csi.aws.com
EOF

cat <<EOF> pv.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
  name: efs-pv-logs
spec:
  capacity:
    storage: 5Gi
  volumeMode: Filesystem
  claimRef:
    apiVersion: v1
    kind: PersistentVolumeClaim
    name: efs-claim-logs
    namespace: prod
  accessModes:
    - ReadWriteMany
  persistentVolumeReclaimPolicy: Retain
  storageClassName: efs-sc
  csi:
    driver: efs.csi.aws.com
    volumeHandle: ${efs_id}
EOF

cat <<\EOF> claim.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: efs-claim-logs
  namespace: prod
spec:
  accessModes:
    - ReadWriteMany
  storageClassName: efs-sc
  volumeName: efs-pv-logs
  resources:
    requests:
      storage: 5Gi
EOF

#挂载子路径
#https://github.com/kubernetes-sigs/aws-efs-csi-driver/blob/master/examples/kubernetes/volume_path/README.md
cat <<EOF> pv.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
  name: efs-pv-logs
spec:
  capacity:
    storage: 5Gi
  volumeMode: Filesystem
  claimRef:
    apiVersion: v1
    kind: PersistentVolumeClaim
    name: efs-claim-logs
    namespace: prod
  accessModes:
    - ReadWriteMany
  persistentVolumeReclaimPolicy: Retain
  storageClassName: efs-sc
  csi:
    driver: efs.csi.aws.com
    volumeHandle: ${efs_id}:/xxxxxx
EOF

kubectl  apply -f ./


#业务deployment.yaml
......
spec:
  template:
    spec:
      containers:
      - env:
      .....
        volumeMounts:
        - mountPath: /app/server/logs
          name: logs
          subPath: game-server
      volumes:
      - hostPath:
          path: /usr/share/zoneinfo/Africa/Lagos
          type: ""
        name: localtime
#      - emptyDir: {}
#        name: logs
      - name: logs
        persistentVolumeClaim:
          claimName: efs-claim-logs

#https://github.com/kubernetes-sigs/aws-efs-csi-driver/blob/master/docs/README.md#examples
#git clone https://github.com/kubernetes-sigs/aws-efs-csi-driver.git
#cd aws-efs-csi-driver/examples/kubernetes/multiple_pods/
#cat specs/*

#Amazon Linux主机挂载efs， efs中创建接入点，按提示本地执行命令
sudo yum install -y amazon-efs-utils
mkdir /mnt/efs
#控制台页面点击连接获取挂载命令
sudo mount -t efs -o tls,accesspoint=fsap-05625xxxxxxxxxx fs-0cxxxx2xxx:/ /mnt/efs
cd /mnt/efs  && mkdir rummy


mkdir /efs/rummy -pv
sudo mount -t efs -o tls,accesspoint=fsap-0f225xxx fs-00a7bxxx:/rummy /efs/rummy

#centos 主机挂载efs
$ sudo yum -y install git rpm-build make rust cargo openssl-devel
$ git clone https://github.com/aws/efs-utils
$ cd efs-utils
$ make rpm
$ sudo yum -y install build/amazon-efs-utils*rpm
mkdir /mnt/efs
mount -t efs -o tls,accesspoint=fsap-03xxxx fs-0xxx:/ /mnt/efs

curl -o deploy.yaml https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.12.3/deploy/static/provider/aws/deploy.yaml
#修改为多实例

[root@jump ingress-nginx]# cat ingress-alb.yaml 
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: ingress-prod
  namespace: ingress-nginx-prod
  annotations:
    #kubernetes.io/ingress.class: "alb"
    alb.ingress.kubernetes.io/group.name: "ingress-prod"
    alb.ingress.kubernetes.io/group.order: "900"
    alb.ingress.kubernetes.io/load-balancer-name: ingress-prod-alb
    alb.ingress.kubernetes.io/scheme: internet-facing
    alb.ingress.kubernetes.io/security-groups: sg-xxxx,sg-xxx
    alb.ingress.kubernetes.io/subnets: subnet-0xxxx, subnet-0xxx, subnet-0d8xxx  #公网子网
    alb.ingress.kubernetes.io/success-codes: '200-499'
    alb.ingress.kubernetes.io/target-type: ip
    alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80},{"HTTPS": 443}]'
    alb.ingress.kubernetes.io/ssl-redirect: '443'
    alb.ingress.kubernetes.io/ssl-policy: ELBSecurityPolicy-TLS13-1-2-2021-06
    alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:ap-south-1:xxxxx:certificate/xxxxx
    alb.ingress.kubernetes.io/tags: Techteam=AN, Application=R, Name=ingress-prod-alb, Environment=Production, Author=jasper.xu
spec:
  ingressClassName: alb
  rules:
    - host: "*.xxxx.in"
      http:
        paths:
          - pathType: ImplementationSpecific
            path: /*
            backend:
              service:
                name: ingress-nginx-controller
                port:
                  number: 80

# Add kubernetes-dashboard repository
helm repo add kubernetes-dashboard https://kubernetes.github.io/dashboard/
# Deploy a Helm Release named "kubernetes-dashboard" using the kubernetes-dashboard chart
helm upgrade --install kubernetes-dashboard kubernetes-dashboard/kubernetes-dashboard --create-namespace --namespace kubernetes-dashboard
helm list -A

cat <<\EOF> ingress.yaml 
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  annotations:
    nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
    nginx.ingress.kubernetes.io/ssl-redirect: "true"
    nginx.ingress.kubernetes.io/server-snippet: |
        location ~* /api/actuator {
            deny all;
        }
  name: dashboard-ingress
  namespace: kubernetes-dashboard
spec:
  ingressClassName: nginx-prod
  rules:
  - host: kubedashboard.xx.com
    http:
      paths:
      - backend:
          service:
            name: kubernetes-dashboard-kong-proxy
            port:
              number: 443
        path: /
        pathType: ImplementationSpecific
EOF


#管理员
cat<<\EOF> admin.yaml 
apiVersion: v1
kind: ServiceAccount
metadata:
  name: admin-user
  namespace: kubernetes-dashboard
---
apiVersion: v1
kind: Secret
metadata:
  name: admin-user
  namespace: kubernetes-dashboard
  annotations:
    kubernetes.io/service-account.name: "admin-user"   
type: kubernetes.io/service-account-token  
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: admin-user
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: admin-user
  namespace: kubernetes-dashboard
EOF

#普通用户
cat<<\EOF> account.yaml 
apiVersion: v1
kind: ServiceAccount
metadata:
  name: cluster-readonly
  namespace: kube-system
#secrets:
#- name: cluster-readonly
---
#手动为 ServiceAccount 创建长期有效的 API 令牌
apiVersion: v1
kind: Secret
metadata:
  name: cluster-readonly
  namespace: kube-system
  annotations:
    kubernetes.io/service-account.name: "cluster-readonly"
type: kubernetes.io/service-account-token
EOF

cat<<\EOF> readonly.yaml
#apiVersion: v1
#kind: ServiceAccount
#metadata:
#  name: cluster-readonly
#  namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: cluster-readonly
rules:
- apiGroups:
  - ""
  resources:
  - pods
  - pods/attach
  - pods/exec
  - pods/portforward
  - pods/proxy
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - ""
  resources:
  - pods/attach
  - pods/exec
  verbs:
  - create
- apiGroups:
  - ""
  resources:
  - configmaps
  - endpoints
  - persistentvolumeclaims
  - replicationcontrollers
  - replicationcontrollers/scale
  - serviceaccounts
  - services
  - services/proxy
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - ""
  resources:
  - bindings
  - events
  - limitranges
  - namespaces/status
  - pods/log
  - pods/status
  - replicationcontrollers/status
  - resourcequotas
  - resourcequotas/status
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - ""
  resources:
  - namespaces
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - apps
  resources:
  - deployments
  - deployments/rollback
  - deployments/scale
  - statefulsets
  - replicasets
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - autoscaling
  resources:
  - horizontalpodautoscalers
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - batch
  resources:
  - cronjobs
  - jobs
  - scheduledjobs
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - extensions
  resources:
  - daemonsets
  - deployments
  - ingresses
  - replicasets
  verbs:
  - get
  - list
  - watch
- apiGroups: [""]
  resources: ["nodes"]
  verbs: ["get", "list", "watch"]
- apiGroups:
  - networking.k8s.io
  resources:
  - ingresses
  - ingressclasses
  verbs:
  - list
  - watch
  - get
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: cluster-readonly
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-readonly
subjects:
- apiGroup: rbac.authorization.k8s.io
  kind: Group
  name: develop:readonly
- kind: ServiceAccount
  name: cluster-readonly
  namespace: kube-system
EOF


#https://github.com/kubernetes/dashboard/blob/master/docs/user/access-control/creating-sample-user.md
kubectl get secret admin-user -n kubernetes-dashboard -o jsonpath={".data.token"} | base64 -d

cat <<\EOF>> ~/.bashrc
alias gett='kubectl -n kube-system get secret cluster-readonly -o jsonpath={.data.token} | base64 -d'
alias gettt='kubectl  -n kubernetes-dashboard get secret admin-user -o jsonpath={.data.token} | base64 -d'
EOF

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Action": [
                "ecr:BatchCheckLayerAvailability",
                "ecr:BatchGetImage",
                "ecr:GetDownloadUrlForLayer",
                "ecr:GetAuthorizationToken"
            ],
            "Resource": "*"
        }
    ]
}

#!/bin/bash
echo "编译 Docker image 仓库2"
ret=`aws --region  ap-south-1 --profile prod ecr describe-repositories --repository-names ${application}/${service_name} &>/dev/null`
if [ -z $ret ]; then
    echo Create repository
    aws --region  ap-south-1 --profile prod ecr create-repository --repository-name ${application}/${service_name}
fi


#aws ecr get-login-password --region af-south-1 | docker login --username AWS --password-stdin $REGISTRY_ADDRESS_prod
aws --region  ap-south-1 --profile prod ecr get-login-password --region ap-south-1 | docker login --username AWS --password-stdin xxxx.dkr.ecr.ap-south-1.amazonaws.com

export BUILDX_NO_DEFAULT_ATTESTATIONS=1
docker buildx build -t ${release_image_name} --platform=linux/arm64,linux/amd64 --push .

        stage('Build a Docker Image') {
            steps {
              container('aws-cli') {
                sh """
                  export AWS_ACCESS_KEY_ID=xxYx
                  export AWS_SECRET_ACCESS_KEY=xxxx
                  if ! aws ecr describe-repositories --repository-names rxx/${JOB_BASE_NAME} &>/dev/null; then
                      echo Create repository
                      aws ecr create-repository --repository-name rxx/${JOB_BASE_NAME}
                  fi
                """
              }
              container('dind') {
                  sh '''
                    wget https://amazon-ecr-credential-helper-releases.s3.us-east-2.amazonaws.com/0.9.0/linux-amd64/docker-credential-ecr-login
                    chmod +x docker-credential-ecr-login
                    mv docker-credential-ecr-login /usr/local/bin/docker-credential-ecr-login

                    mkdir -pv ~/.docker/
                    cat > ~/.docker/config.json <<EOF
{
    "credHelpers": {
        "public.ecr.aws": "ecr-login",
        "xxxx.dkr.ecr.us-west-2.amazonaws.com": "ecr-login"
    }
}
EOF

                    mkdir -pv ~/.aws/
                    cat <<'EOF'> ~/.aws/credentials
[default]
aws_access_key_id = xxxx
aws_secret_access_key = xxxx
EOF
                  '''
              }
              container('dind') {
                sh """
                docker -H tcp://1.0.0.1:2375 build -t  ${imgurl}:${TAG} -f- . <<'EOF'
FROM public.ecr.aws/nginx/nginx:latest
LABEL maintainer jenkins@
ENV SERVICE_NAME=rl-manage-web SERVICE_PATH=/opt/web
COPY dist \${SERVICE_PATH}/site
EXPOSE 80
WORKDIR \${SERVICE_PATH}
ENV TZ Asia/Shanghai
EOF
"""
                sh "docker -H tcp://10.0.0.1:2375 tag  ${imgurl}:${TAG}  ${imgurl}:latest"
              }
              container('dind') {
                sh """
                  #docker -H tcp://10.0.0.1:2375 login --username=xx -p Wc=xx harbor.xxx.com
                  docker -H tcp://10.0.0.1:2375 push ${imgurl}:${TAG}
                  docker -H tcp://10.0.0.1:2375 push ${imgurl}:latest
                  docker -H tcp://10.0.0.1:2375 rmi -f ${imgurl}:${TAG} ${imgurl}:latest
                """
              }
            }
        }

# 开机 并检查 $i 为ec2实例id
aws ec2 start-instances  --instance-ids $i
aws --region ap-south-1 ec2 describe-instances --filters "Name=instance-id,Values=${i}" --query 'Reservations[*].Instances[*].[PrivateIpAddress]' --output text

# 关机 并检查
aws ec2 stop-instances  --instance-ids $i
aws --region ap-south-1 ec2 describe-instances --filters "Name=instance-id,Values=${i}" --query 'Reservations[*].Instances[*].[PrivateIpAddress]' --output text

      aws ec2 describe-instances --region ap-south-1 \
          --filters Name=tag:Name,Values=ludo-staging-ec2-new-stg-eks-app

      cat ec2-tag.log |jq '.Reservations[].Instances[]|[.PrivateIpAddress, .State.Name, ([(.Tags[]?|[.Key, .Value]|join("="))]|join("@"))] | join("|")'


aws ec2 describe-instances \
    --filters Name=instance-state-name,Values=running \
    --query 'Reservations[*].Instances[*].{Instance:InstanceId,IP:PrivateIpAddress,InstanceType:InstanceType,AZ:Placement.AvailabilityZone,Name:Tags[?Key==`Name`]|[0].Value,_SubModule:Tags[?Key==`_SubModule`]|[0].Value,__Usage:Tags[?Key==`_Usage`]|[0].Value,ClusterName:Tags[?Key==`eks:cluster-name`]|[0].Value,Techteam:Tags[?Key==`Techteam`]|[0].Value}' \
    --output table > a.log
cat a.log  |grep None  >b.log
cat b.log |grep -v RDS  |sort -n -k 11 >c.log

aws ec2 describe-instances --filters Name="network-interface.addresses.private-ip-address",Values="172.21.38.5" \
 --query 'Reservations[*].Instances[*].{Instance:InstanceId,AZ:Placement.AvailabilityZone,Name:Tags[?Key==`Name`]|[0].Value,_SubModule:Tags[?Key==`_SubModule`]|[0].Value,IP:PrivateIpAddress}' \
    --output table

#确定实例是基于 Xen 还是基于 Nitro
instance_type=m7g.2xlarge
aws ec2 describe-instance-types --instance-type $instance_type --query "InstanceTypes[].Hypervisor"

#nitro
#1.调整分区的大小
#检查卷是否有分区。使用 lsblk 命令. 根卷（nvme0n1）有两个分区（nvme0n1p1 和 nvme0n1p128
[ec2-user@ip-172-31-5-34 ~]$ sudo lsblk
NAME          MAJ:MIN RM  SIZE RO TYPE MOUNTPOINTS
nvme0n1       259:0    0  200G  0 disk 
├─nvme0n1p1   259:1    0  100G  0 part /
└─nvme0n1p128 259:2    0   10M  0 part /boot/efi
#扩展分区。使用 growpart 命令并指定设备名称和分区编号。
[ec2-user@ip-172-31-5-34 ~]$ sudo growpart /dev/nvme0n1 1
CHANGED: partition=1 start=22528 old: size=209692639 end=209715167 new: size=419407839 end=419430367
#验证是否已扩展分区
[ec2-user@ip-172-31-5-34 ~]$ sudo lsblk
NAME          MAJ:MIN RM  SIZE RO TYPE MOUNTPOINTS
nvme0n1       259:0    0  200G  0 disk 
├─nvme0n1p1   259:1    0  200G  0 part /
└─nvme0n1p128 259:2    0   10M  0 part /boot/efi

#2.扩展文件系统
#获取需要扩展的文件系统的名称、大小、类型和挂载点。使用 df -hT 命令。
[ec2-user@ip-172-31-5-34 ~]$ df -hT
Filesystem       Type      Size  Used Avail Use% Mounted on
/dev/nvme0n1p1   xfs       100G   92G  8.1G  92% /
#扩展文件系统的命令因文件系统类型而异
#[XFS 文件系统] 使用 xfs_growfs 命令并指定您在上一步中记录的文件系统的挂载点
[ec2-user@ip-172-31-5-34 ~]$ sudo xfs_growfs -d /
#[Ext4 文件系统] 使用 resize2fs 命令并指定您在上一步中记录的文件系统的名称
sudo resize2fs /dev/nvme0n1p1
#验证是否已扩展文件系统。使用 df -hT 命令并确认文件系统大小等于卷大小
[ec2-user@ip-172-31-5-34 ~]$ df -hT
Filesystem       Type      Size  Used Avail Use% Mounted on
/dev/nvme0n1p1   xfs       200G   93G  108G  47% /