Linux: AWS使用

#角色名
role_name=AmazonEKS_EBS_CSI_DriverRole-ap-south-1-prod
#EKS集群名
c_n=prod-eks

eksctl create iamserviceaccount \
        --name ebs-csi-controller-sa \
        --namespace kube-system \
        --cluster ${c_n} \
        --role-name ${role_name} \
        --role-only \
        --attach-policy-arn arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy \
        --approve

mkdir gp3
cd gp3

cat <<\EOF> gp3.yaml
kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
  name: gp3
  annotations:
    storageclass.kubernetes.io/is-default-class: "true"
allowVolumeExpansion: true
provisioner: ebs.csi.aws.com
volumeBindingMode: WaitForFirstConsumer
parameters:
  type: gp3
EOF

# https://kubernetes.io/docs/tasks/administer-cluster/change-default-storage-class/
kubectl apply -f gp3.yaml
kubectl patch storageclass gp2 -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"false"}}}'

args:
controller
--endpoint=$(CSI_ENDPOINT)
--logtostderr
--v=2
--extra-tags=Techteam=RC,Environment=ops,BusinessUnit=RC,IgnoreCostAdvisor=true,Ignoreoldgen=true,Application=ops,Cluster=ops-eks,Name=ops-ebs

mkdir awsloadbalance
cd  awsloadbalance

#https://docs.aws.amazon.com/zh_cn/eks/latest/userguide/lbc-manifest.html

#步骤 1：配置 IAM

curl -O https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.7.2/docs/install/iam_policy.json

##已经存在

aws iam create-policy \
    --policy-name AWSLoadBalancerControllerIAMPolicy \
    --policy-document file://iam_policy.json

##使用 eksctl 创建 IAM 角色
#修改角色名，集群名，策略arn
cluster_name=prod-eks
role_name=AmazonEKSLoadBalancerControllerRole-prod-eks
account_id=<ACCOUNT_ID>

# 请将 my-cluster 替换为您的集群的名称，将 111122223333 替换为您的账户 ID，然后运行命令。
eksctl create iamserviceaccount \
  --cluster=my-cluster \
  --namespace=kube-system \
  --name=aws-load-balancer-controller \
  --role-name AmazonEKSLoadBalancerControllerRole \
  --attach-policy-arn=arn:aws:iam::111122223333:policy/AWSLoadBalancerControllerIAMPolicy \
  --approve

#步骤 2：安装 cert-manager

kubectl apply \
    --validate=false \
    -f https://github.com/jetstack/cert-manager/releases/download/v1.13.5/cert-manager.yaml

#步骤 3：安装 AWS Load Balancer Controller

curl -Lo v2_7_2_full.yaml https://github.com/kubernetes-sigs/aws-load-balancer-controller/releases/download/v2.7.2/v2_7_2_full.yaml

sed -i.bak -e '612,620d' ./v2_7_2_full.yaml  #按官方文档操作，

sed -i.bak -e 's|your-cluster-name|my-cluster|' ./v2_7_2_full.yaml # my-cluster 改成集群名
#添加--aws-vpc-id=vpc-xxxxxxxx

kubectl apply -f v2_7_2_full.yaml

curl -Lo v2_7_2_ingclass.yaml https://github.com/kubernetes-sigs/aws-load-balancer-controller/releases/download/v2.7.2/v2_7_2_ingclass.yaml

kubectl apply -f v2_7_2_ingclass.yaml

#或者helm安装
helm upgrade --install aws-load-balancer-controller eks/aws-load-balancer-controller \
  -n kube-system \
  --set clusterName=${cluster_name} \
  --set serviceAccount.create=false \
  --set serviceAccount.name=aws-load-balancer-controller \
  --set vpcId=vpc-07dbd3d1e6db2f452 \
  --version 1.13.0

#步骤 4：验证控制器是否已安装

kubectl get deployment -n kube-system aws-load-balancer-controller

# 安装标准版 gateway api CRD
kubectl apply --server-side -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.6.1/standard-install.yaml

# 范例
root@ip-172-31-18-198:/docker/nginx/data/site.d# kubectl apply --server-side -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.6.1/standard-install.yaml
customresourcedefinition.apiextensions.k8s.io/backendtlspolicies.gateway.networking.k8s.io serverside-applied
customresourcedefinition.apiextensions.k8s.io/gatewayclasses.gateway.networking.k8s.io serverside-applied
customresourcedefinition.apiextensions.k8s.io/gateways.gateway.networking.k8s.io serverside-applied
customresourcedefinition.apiextensions.k8s.io/grpcroutes.gateway.networking.k8s.io serverside-applied
customresourcedefinition.apiextensions.k8s.io/httproutes.gateway.networking.k8s.io serverside-applied
customresourcedefinition.apiextensions.k8s.io/listenersets.gateway.networking.k8s.io serverside-applied
customresourcedefinition.apiextensions.k8s.io/referencegrants.gateway.networking.k8s.io serverside-applied
customresourcedefinition.apiextensions.k8s.io/tcproutes.gateway.networking.k8s.io serverside-applied
customresourcedefinition.apiextensions.k8s.io/tlsroutes.gateway.networking.k8s.io serverside-applied
customresourcedefinition.apiextensions.k8s.io/udproutes.gateway.networking.k8s.io serverside-applied
validatingadmissionpolicy.admissionregistration.k8s.io/safe-upgrades.gateway.networking.k8s.io serverside-applied
validatingadmissionpolicybinding.admissionregistration.k8s.io/safe-upgrades.gateway.networking.k8s.io serverside-applied

root@ip-172-31-18-198:~# kubectl  get crd |grep gateway.networking.k8s.io
backendtlspolicies.gateway.networking.k8s.io    2026-08-26T09:10:31Z
gatewayclasses.gateway.networking.k8s.io        2026-08-26T09:10:31Z
gateways.gateway.networking.k8s.io              2026-08-26T09:10:31Z
grpcroutes.gateway.networking.k8s.io            2026-08-26T09:10:31Z
httproutes.gateway.networking.k8s.io            2026-08-26T09:10:31Z
listenersets.gateway.networking.k8s.io          2026-08-26T09:10:32Z
referencegrants.gateway.networking.k8s.io       2026-08-26T09:10:32Z
tcproutes.gateway.networking.k8s.io             2026-08-26T09:10:32Z
tlsroutes.gateway.networking.k8s.io             2026-08-26T09:10:33Z
udproutes.gateway.networking.k8s.io             2026-08-26T09:10:33Z

# 安装 NGINX Gateway Fabric CRDs
# https://github.com/nginx/nginx-gateway-fabric/tree/v2.6.7/deploy
kubectl apply --server-side -f https://raw.githubusercontent.com/nginx/nginx-gateway-fabric/refs/tags/v2.6.7/deploy/crds.yaml
# 普通 kubectl apply 是客户端 apply：本地保存一份注解 kubectl.kubernetes.io/last-applied-configuration，用来对比变更。
# --server‑side 是把合并逻辑交给 **k8s APIServer 服务端处理**，不在客户端存大段 last‑applied 注解。

# 范例
root@ip-172-31-18-198:~# kubectl apply --server-side -f https://raw.githubusercontent.com/nginx/nginx-gateway-fabric/refs/tags/v2.6.7/deploy/crds.yaml
customresourcedefinition.apiextensions.k8s.io/authenticationfilters.gateway.nginx.org serverside-applied
customresourcedefinition.apiextensions.k8s.io/clientsettingspolicies.gateway.nginx.org serverside-applied
customresourcedefinition.apiextensions.k8s.io/nginxgateways.gateway.nginx.org serverside-applied
customresourcedefinition.apiextensions.k8s.io/nginxproxies.gateway.nginx.org serverside-applied
customresourcedefinition.apiextensions.k8s.io/observabilitypolicies.gateway.nginx.org serverside-applied
customresourcedefinition.apiextensions.k8s.io/proxysettingspolicies.gateway.nginx.org serverside-applied
customresourcedefinition.apiextensions.k8s.io/ratelimitpolicies.gateway.nginx.org serverside-applied
customresourcedefinition.apiextensions.k8s.io/snippetsfilters.gateway.nginx.org serverside-applied
customresourcedefinition.apiextensions.k8s.io/snippetspolicies.gateway.nginx.org serverside-applied
customresourcedefinition.apiextensions.k8s.io/upstreamsettingspolicies.gateway.nginx.org serverside-applied
customresourcedefinition.apiextensions.k8s.io/wafpolicies.gateway.nginx.org serverside-applied

root@ip-172-31-18-198:~# kubectl  get crd |grep nginx
authenticationfilters.gateway.nginx.org         2026-08-26T09:37:50Z
clientsettingspolicies.gateway.nginx.org        2026-08-26T09:37:50Z
nginxgateways.gateway.nginx.org                 2026-08-26T09:37:50Z
nginxproxies.gateway.nginx.org                  2026-08-26T09:37:50Z
observabilitypolicies.gateway.nginx.org         2026-08-26T09:37:50Z # 可观测规则
proxysettingspolicies.gateway.nginx.org         2026-08-26T09:37:51Z # 代理设置规则
ratelimitpolicies.gateway.nginx.org             2026-08-26T09:37:51Z # 限速规则
snippetsfilters.gateway.nginx.org               2026-08-26T09:37:51Z
snippetspolicies.gateway.nginx.org              2026-08-26T09:37:51Z
upstreamsettingspolicies.gateway.nginx.org      2026-08-26T09:37:51Z
wafpolicies.gateway.nginx.org                   2026-08-26T09:37:51Z # 应用防火墙规则

# 
mkdir ngfdir
cd ngfdir/

# 下载 2.6.7 版本文件。拉取 OCI 格式 Helm chart，解压到本地
helm pull oci://ghcr.io/nginx/charts/nginx-gateway-fabric --version 2.6.7 --untar
cd nginx-gateway-fabric/

helm install ngf . \
  --create-namespace -n nginx-gateway \
  --set nginx.service.type=LoadBalancer \
  --set nginx.service.externalTrafficPolicy=Cluster \
  --set nginxGateway.snippetsFilters.enable=true

#  --set nginx.service.type=LoadBalancer 使用负载均衡器。会自动分配公网 LB
#  --set nginx.service.externalTrafficPolicy=Cluster 外部流量转发策略。
#    Cluster（这里配置的）：源 IP 会被 SNAT 改写为节点 IP；可以做负载均衡跨节点转发。
#    Local：保留真实客户端源 IP，但流量只能落到后端 Pod 所在节点。
#  --set nginxGateway.snippetsFilters.enable=true  NGF 的扩展能力，可以在 Gateway API 资源中嵌入原生 Nginx 配置片段（nginx.conf 片段）。

root@ip-172-31-18-198:~/.jasper/ngfdir/nginx-gateway-fabric# kubectl  get ns |grep gateway
nginx-gateway       Active   16m

root@ip-172-31-18-198:~/.jasper/ngfdir/nginx-gateway-fabric# kubectl  get pods -n nginx-gateway
NAME                                        READY   STATUS    RESTARTS   AGE
ngf-nginx-gateway-fabric-746f6d68fc-hspmf   1/1     Running   0          85s

root@ip-172-31-18-198:~/.jasper/ngfdir/nginx-gateway-fabric# kubectl  get svc -n nginx-gateway
NAME                       TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)   AGE
ngf-nginx-gateway-fabric   ClusterIP   10.210.167.53   <none>        443/TCP   2m7s

# 卸载
# helm uninstall ngf -n nginx-gateway
# kubectl delete ns nginx-gateway --ignore-not-found

root@ip-172-31-18-198:~/.jasper/ngfdir/nginx-gateway-fabric# cat values-dev.yaml
nginx:
  config:
    # 信任 TCP Proxy‑Protocol v2 四层 NLb
    rewriteClientIP:
      mode: ProxyProtocol
      # AWS VPC CIDR，信任NLB的来源网段
      trustedAddresses:
        - type: CIDR
          value: "172.31.0.0/16"
  service:
    type: LoadBalancer
    externalTrafficPolicy: Cluster
    patches:
    - type: StrategicMerge
      value:
        metadata:
          annotations:
            # 创建公网 nlb
            service.beta.kubernetes.io/aws-load-balancer-type: "external"
            service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: "ip"
            service.beta.kubernetes.io/aws-load-balancer-scheme: "internet-facing"
            service.beta.kubernetes.io/aws-load-balancer-subnets: "subnet-0a415b6c7f8914523,subnet-0ef0b0cd6d66d7596"
            service.beta.kubernetes.io/aws-load-balancer-additional-resource-tags: "Env=Dev"
            # 让 AWS NLB 向外发 Proxy‑Protocol v2
            service.beta.kubernetes.io/aws-load-balancer-proxy-protocol: "*"

nginxGateway:
  snippetsFilters:
    enable: true
    
    
# balancer-subnets 为公有子网，pod所在实例安全组允许该网段

helm install ngf . \
  --create-namespace -n nginx-gateway \
  -f ./values-dev.yaml \
  --wait --timeout 5m

# 安装完成后校验 NginxProxy CR 是否渲染正确 patches：
kubectl get nginxproxy ngf-proxy-config -n nginx-gateway -o yaml

# 控制器 nginx
root@ip-172-31-18-198:~/.jasper/ngfdir/nginx-gateway-fabric# kubectl  get gatewayclasses -n nginx-gateway
NAME    CONTROLLER                                   ACCEPTED   AGE
nginx   gateway.nginx.org/nginx-gateway-controller   True       2m55s

# nginxproxy 相当说configmap
root@ip-172-31-18-198:~/.jasper/ngfdir/nginx-gateway-fabric# kubectl  get nginxproxy -n nginx-gateway
NAME               AGE
ngf-proxy-config   4m33s

root@ip-172-31-18-198:~/.jasper/ngfdir/nginx-gateway-fabric# kubectl  get nginxproxies -n nginx-gateway  -oyaml
apiVersion: v1
items:
- apiVersion: gateway.nginx.org/v1alpha2
  kind: NginxProxy
  metadata:
    labels:
      app.kubernetes.io/instance: ngf
      app.kubernetes.io/managed-by: Helm
      app.kubernetes.io/name: nginx-gateway-fabric
      app.kubernetes.io/version: 2.6.7
      helm.sh/chart: nginx-gateway-fabric-2.6.7
    name: ngf-proxy-config
    namespace: nginx-gateway
  spec:
    ipFamily: dual
    kubernetes:
      deployment:
        container:
          image:
            pullPolicy: IfNotPresent
            repository: ghcr.io/nginx/nginx-gateway-fabric/nginx
            tag: 2.6.7
        replicas: 1
      service:
        externalTrafficPolicy: Cluster
        type: LoadBalancer
kind: List
metadata:
  resourceVersion: ""

cat <<\EOF>> 01-gateway-test.yaml
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
  name: nginx-gateway-test
  namespace: default
spec:
  gatewayClassName: nginx
  listeners:
  - name: http
    protocol: HTTP
    port: 80
    allowedRoutes:
      namespaces:
        from: All # 允许所有namespace的HTTPRoute绑定这个Gateway。默认为允许当前名称空间的规则
EOF

kubectl apply -f 01-gateway-test.yaml

# gateway 的地址就是 service 的地址
root@ip-172-31-18-198:~/.jasper/ngfdir# kubectl  get gateway
NAME                 CLASS   ADDRESS                                                                             PROGRAMMED   AGE
nginx-gateway-test   nginx   k8s-d-1.amazonaws.com   True         25m

root@ip-172-31-18-198:~/.jasper/ngfdir# kubectl  get svc
nginx-gateway-test-nginx       LoadBalancer   10.210.170.159   k8s-d-1.amazonaws.com   80:31363/TCP

# 部署 echo ，打印请求信息
cat > 02-deploy-test.yaml <<\EOF
apiVersion: apps/v1
kind: Deployment
metadata:
  name: echo
  namespace: default
spec:
  replicas: 1
  selector:
    matchLabels:
      app: echo
  template:
    metadata:
      labels:
        app: echo
    spec:
      containers:
      - name: echo
        image: ealen/echo-server:latest
        ports:
        - containerPort: 80
---
apiVersion: v1
kind: Service
metadata:
  name: echo-svc
  namespace: default
spec:
  selector:
    app: echo
  ports:
  - port: 80
    targetPort: 80
    name: http
  type: ClusterIP
EOF

kubectl  apply -f 02-deploy-test.yaml

# 创建 gateway 规则
cat <<\EOF>> 03-httproute.yaml
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
  name: ngfapp-httproute
  namespace: default
spec:
  parentRefs:
  - name: nginx-gateway-test
    namespace: default
  hostnames:
  - "www.echo.com"
  rules:
  - backendRefs:
    - name: echo-svc
      port: 80
      kind: Service
    matches:
    - path:
        type: PathPrefix
        value: /
EOF

root@ip-172-31-18-198:~/.jasper/ngfdir# kubectl  apply -f 03-httproute.yaml


root@ip-172-31-18-198:~/.jasper/ngfdir# kubectl  get httproute
NAME         HOSTNAMES          AGE
echo-route   ["www.echo.com"]   11m


# 排查。查看 gateway 对应的 pod 日志。
kubectl logs -f nginx-gateway-test-nginx-79d64d4cc4-kcp5q

# 查看 nginx 配置文件
root@ip-172-31-18-198:~/.jasper/ngfdir# kubectl  exec -it nginx-gateway-test-nginx-79d64d4cc4-kcp5q -c nginx -- nginx -T
nginx: the configuration file /etc/nginx/nginx.conf syntax is ok
nginx: configuration file /etc/nginx/nginx.conf test is successful
# configuration file /etc/nginx/nginx.conf:
load_module modules/ngx_http_js_module.so;
include /etc/nginx/main-includes/*.conf;

worker_processes auto;

pid /var/run/nginx/nginx.pid;

# 真实客户端 ip
    "headers": {
      "host": "www.echo.com",
      "x-forwarded-for": "1.1.162.204",
      "x-real-ip": "1.1.162.204",


# 或在 pod echo 所在主机使用 tcpdump 命令查看
tcpdump -i any -nn -vvv -A dst host 172.31.82.72 and port 80  |grep -C50 echo

export cluster_name=prod-eks
export region_name=ap-south-1

vpc_id=$(aws eks describe-cluster \
    --name $cluster_name \
    --query "cluster.resourcesVpcConfig.vpcId" \
    --output text)

echo $vpc_id

cidr_range=$(aws ec2 describe-vpcs \
    --vpc-ids $vpc_id \
    --query "Vpcs[].CidrBlock" \
    --output text \
    --region $region_name)

echo $cidr_range

security_group_id=$(aws ec2 create-security-group \
    --group-name MyEfsSecurityGroup \
    --description "My EFS security group" \
    --vpc-id $vpc_id \
    --output text)

echo $security_group_id
security_group_id=`echo $security_group_id |awk '{print $1}'`
echo $security_group_id

aws ec2 authorize-security-group-ingress \
    --group-id $security_group_id \
    --protocol tcp \
    --port 2049 \
    --cidr $cidr_range

#创建EFS实例
file_system_id=$(aws efs create-file-system \
    --region ${region_name} \
    --performance-mode generalPurpose \
    --query 'FileSystemId' \
    --output text)

echo $file_system_id
echo $security_group_id

#配置挂载点网络关联
subnet1=subnet-0789f35e7ff7e10da
subnet2=subnet-0ddfcc08a775ffc3c
subnet3=subnet-0d8f55ad4d534b641

aws efs create-mount-target \
    --file-system-id $file_system_id \
    --subnet-id $subnet1 \
    --security-groups $security_group_id

aws efs create-mount-target \
    --file-system-id $file_system_id \
    --subnet-id $subnet2 \
    --security-groups $security_group_id

aws efs create-mount-target \
    --file-system-id $file_system_id \
    --subnet-id $subnet3 \
    --security-groups $security_group_id

#EFS控制台创建接入点， 默认路径/ ，可设置 /{业务名}

#获取文件系统id
efs_id=`aws efs describe-file-systems --query "FileSystems[*].FileSystemId" --output text`

mkdir efs ; cd efs

cat <<\EOF> storageclass.yaml 
kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
  name: efs-sc
provisioner: efs.csi.aws.com
EOF

cat <<EOF> pv.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
  name: efs-pv-logs
spec:
  capacity:
    storage: 5Gi
  volumeMode: Filesystem
  claimRef:
    apiVersion: v1
    kind: PersistentVolumeClaim
    name: efs-claim-logs
    namespace: prod
  accessModes:
    - ReadWriteMany
  persistentVolumeReclaimPolicy: Retain
  storageClassName: efs-sc
  csi:
    driver: efs.csi.aws.com
    volumeHandle: ${efs_id}
EOF

cat <<\EOF> claim.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: efs-claim-logs
  namespace: prod
spec:
  accessModes:
    - ReadWriteMany
  storageClassName: efs-sc
  volumeName: efs-pv-logs
  resources:
    requests:
      storage: 5Gi
EOF

#挂载子路径
#https://github.com/kubernetes-sigs/aws-efs-csi-driver/blob/master/examples/kubernetes/volume_path/README.md
cat <<EOF> pv.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
  name: efs-pv-logs
spec:
  capacity:
    storage: 5Gi
  volumeMode: Filesystem
  claimRef:
    apiVersion: v1
    kind: PersistentVolumeClaim
    name: efs-claim-logs
    namespace: prod
  accessModes:
    - ReadWriteMany
  persistentVolumeReclaimPolicy: Retain
  storageClassName: efs-sc
  csi:
    driver: efs.csi.aws.com
    volumeHandle: ${efs_id}:/xxxxxx
EOF

kubectl  apply -f ./

# 测试
apiVersion: v1
kind: Pod
metadata:
  name: app1
  namespace: prod
spec:
  containers:
  - name: app1
    image: busybox
    command: ["/bin/sh"]
    args: ["-c", "while true; do echo $(date -u) >> /data/out1.txt; sleep 5; done"]
    volumeMounts:
    - name: persistent-storage
      mountPath: /data
      subPath: tmp-test
  volumes:
  - name: persistent-storage
    persistentVolumeClaim:
      claimName: efs-claim-logs


#业务deployment.yaml
......
spec:
  template:
    spec:
      containers:
      - env:
      .....
        volumeMounts:
        - mountPath: /app/server/logs
          name: logs
          subPath: game-server
      volumes:
      - hostPath:
          path: /usr/share/zoneinfo/Africa/Lagos
          type: ""
        name: localtime
#      - emptyDir: {}
#        name: logs
      - name: logs
        persistentVolumeClaim:
          claimName: efs-claim-logs

#https://github.com/kubernetes-sigs/aws-efs-csi-driver/blob/master/docs/README.md#examples
#git clone https://github.com/kubernetes-sigs/aws-efs-csi-driver.git
#cd aws-efs-csi-driver/examples/kubernetes/multiple_pods/
#cat specs/*

#Amazon Linux主机挂载efs， efs中创建接入点，按提示本地执行命令
sudo yum install -y amazon-efs-utils
mkdir /mnt/efs
#控制台页面点击连接获取挂载命令
sudo mount -t efs -o tls,accesspoint=fsap-05625xxxxxxxxxx fs-0cxxxx2xxx:/ /mnt/efs
cd /mnt/efs  && mkdir rummy


mkdir /efs/rummy -pv
sudo mount -t efs -o tls,accesspoint=fsap-0f225xxx fs-00a7bxxx:/rummy /efs/rummy

#centos 主机挂载efs
$ sudo yum -y install git rpm-build make rust cargo openssl-devel
$ git clone https://github.com/aws/efs-utils
$ cd efs-utils
$ make rpm
$ sudo yum -y install build/amazon-efs-utils*rpm
mkdir /mnt/efs
mount -t efs -o tls,accesspoint=fsap-03xxxx fs-0xxx:/ /mnt/efs

curl -o deploy.yaml https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.12.3/deploy/static/provider/aws/deploy.yaml
#修改为多实例

[root@jump ingress-nginx]# cat ingress-alb.yaml 
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: ingress-prod
  namespace: ingress-nginx-prod
  annotations:
    #kubernetes.io/ingress.class: "alb"
    alb.ingress.kubernetes.io/group.name: "ingress-prod"
    alb.ingress.kubernetes.io/group.order: "900"
    alb.ingress.kubernetes.io/load-balancer-name: ingress-prod-alb
    alb.ingress.kubernetes.io/scheme: internet-facing
    alb.ingress.kubernetes.io/security-groups: sg-xxxx,sg-xxx
    alb.ingress.kubernetes.io/subnets: subnet-0xxxx, subnet-0xxx, subnet-0d8xxx  #公网子网
    alb.ingress.kubernetes.io/success-codes: '200-499'
    alb.ingress.kubernetes.io/target-type: ip
    alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80},{"HTTPS": 443}]'
    alb.ingress.kubernetes.io/ssl-redirect: '443'
    alb.ingress.kubernetes.io/ssl-policy: ELBSecurityPolicy-TLS13-1-2-2021-06
    alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:ap-south-1:xxxxx:certificate/xxxxx
    alb.ingress.kubernetes.io/tags: Techteam=AN, Application=R, Name=ingress-prod-alb, Environment=Production, Author=jasper.xu
spec:
  ingressClassName: alb
  rules:
    - host: "*.xxxx.in"
      http:
        paths:
          - pathType: ImplementationSpecific
            path: /*
            backend:
              service:
                name: ingress-nginx-controller
                port:
                  number: 80

# Add kubernetes-dashboard repository
helm repo add kubernetes-dashboard https://kubernetes.github.io/dashboard/
# Deploy a Helm Release named "kubernetes-dashboard" using the kubernetes-dashboard chart
helm upgrade --install kubernetes-dashboard kubernetes-dashboard/kubernetes-dashboard --create-namespace --namespace kubernetes-dashboard
helm list -A

cat <<\EOF> ingress.yaml 
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  annotations:
    nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
    nginx.ingress.kubernetes.io/ssl-redirect: "true"
    nginx.ingress.kubernetes.io/server-snippet: |
        location ~* /api/actuator {
            deny all;
        }
  name: dashboard-ingress
  namespace: kubernetes-dashboard
spec:
  ingressClassName: nginx-prod
  rules:
  - host: kubedashboard.xx.com
    http:
      paths:
      - backend:
          service:
            name: kubernetes-dashboard-kong-proxy
            port:
              number: 443
        path: /
        pathType: ImplementationSpecific
EOF


#管理员
cat<<\EOF> admin.yaml 
apiVersion: v1
kind: ServiceAccount
metadata:
  name: admin-user
  namespace: kubernetes-dashboard
---
apiVersion: v1
kind: Secret
metadata:
  name: admin-user
  namespace: kubernetes-dashboard
  annotations:
    kubernetes.io/service-account.name: "admin-user"   
type: kubernetes.io/service-account-token  
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: admin-user
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: admin-user
  namespace: kubernetes-dashboard
EOF

#普通用户
cat<<\EOF> account.yaml 
apiVersion: v1
kind: ServiceAccount
metadata:
  name: cluster-readonly
  namespace: kube-system
#secrets:
#- name: cluster-readonly
---
#手动为 ServiceAccount 创建长期有效的 API 令牌
apiVersion: v1
kind: Secret
metadata:
  name: cluster-readonly
  namespace: kube-system
  annotations:
    kubernetes.io/service-account.name: "cluster-readonly"
type: kubernetes.io/service-account-token
EOF

cat<<\EOF> readonly.yaml
#apiVersion: v1
#kind: ServiceAccount
#metadata:
#  name: cluster-readonly
#  namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: cluster-readonly
rules:
- apiGroups:
  - ""
  resources:
  - pods
  - pods/attach
  - pods/exec
  - pods/portforward
  - pods/proxy
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - ""
  resources:
  - pods/attach
  - pods/exec
  verbs:
  - create
- apiGroups:
  - ""
  resources:
  - configmaps
  - endpoints
  - persistentvolumeclaims
  - replicationcontrollers
  - replicationcontrollers/scale
  - serviceaccounts
  - services
  - services/proxy
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - ""
  resources:
  - bindings
  - events
  - limitranges
  - namespaces/status
  - pods/log
  - pods/status
  - replicationcontrollers/status
  - resourcequotas
  - resourcequotas/status
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - ""
  resources:
  - namespaces
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - apps
  resources:
  - deployments
  - deployments/rollback
  - deployments/scale
  - statefulsets
  - replicasets
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - autoscaling
  resources:
  - horizontalpodautoscalers
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - batch
  resources:
  - cronjobs
  - jobs
  - scheduledjobs
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - extensions
  resources:
  - daemonsets
  - deployments
  - ingresses
  - replicasets
  verbs:
  - get
  - list
  - watch
- apiGroups: [""]
  resources: ["nodes"]
  verbs: ["get", "list", "watch"]
- apiGroups:
  - networking.k8s.io
  resources:
  - ingresses
  - ingressclasses
  verbs:
  - list
  - watch
  - get
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: cluster-readonly
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-readonly
subjects:
- apiGroup: rbac.authorization.k8s.io
  kind: Group
  name: develop:readonly
- kind: ServiceAccount
  name: cluster-readonly
  namespace: kube-system
EOF


#https://github.com/kubernetes/dashboard/blob/master/docs/user/access-control/creating-sample-user.md
kubectl get secret admin-user -n kubernetes-dashboard -o jsonpath={".data.token"} | base64 -d

cat <<\EOF>> ~/.bashrc
alias gett='kubectl -n kube-system get secret cluster-readonly -o jsonpath={.data.token} | base64 -d'
alias gettt='kubectl  -n kubernetes-dashboard get secret admin-user -o jsonpath={.data.token} | base64 -d'
EOF

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Action": [
                "ecr:BatchCheckLayerAvailability",
                "ecr:BatchGetImage",
                "ecr:GetDownloadUrlForLayer",
                "ecr:GetAuthorizationToken"
            ],
            "Resource": "*"
        }
    ]
}

#!/bin/bash
echo "编译 Docker image 仓库2"
ret=`aws --region  ap-south-1 --profile prod ecr describe-repositories --repository-names ${application}/${service_name} &>/dev/null`
if [ -z $ret ]; then
    echo Create repository
    aws --region  ap-south-1 --profile prod ecr create-repository --repository-name ${application}/${service_name}
fi


#aws ecr get-login-password --region af-south-1 | docker login --username AWS --password-stdin $REGISTRY_ADDRESS_prod
aws --region  ap-south-1 --profile prod ecr get-login-password --region ap-south-1 | docker login --username AWS --password-stdin xxxx.dkr.ecr.ap-south-1.amazonaws.com

export BUILDX_NO_DEFAULT_ATTESTATIONS=1
docker buildx build -t ${release_image_name} --platform=linux/arm64,linux/amd64 --push .

        stage('Build a Docker Image') {
            steps {
              container('aws-cli') {
                sh """
                  export AWS_ACCESS_KEY_ID=xxYx
                  export AWS_SECRET_ACCESS_KEY=xxxx
                  if ! aws ecr describe-repositories --repository-names rxx/${JOB_BASE_NAME} &>/dev/null; then
                      echo Create repository
                      aws ecr create-repository --repository-name rxx/${JOB_BASE_NAME}
                  fi
                """
              }
              container('dind') {
                  sh '''
                    wget https://amazon-ecr-credential-helper-releases.s3.us-east-2.amazonaws.com/0.9.0/linux-amd64/docker-credential-ecr-login
                    chmod +x docker-credential-ecr-login
                    mv docker-credential-ecr-login /usr/local/bin/docker-credential-ecr-login

                    mkdir -pv ~/.docker/
                    cat > ~/.docker/config.json <<EOF
{
    "credHelpers": {
        "public.ecr.aws": "ecr-login",
        "xxxx.dkr.ecr.us-west-2.amazonaws.com": "ecr-login"
    }
}
EOF

                    mkdir -pv ~/.aws/
                    cat <<'EOF'> ~/.aws/credentials
[default]
aws_access_key_id = xxxx
aws_secret_access_key = xxxx
EOF
                  '''
              }
              container('dind') {
                sh """
                docker -H tcp://1.0.0.1:2375 build -t  ${imgurl}:${TAG} -f- . <<'EOF'
FROM public.ecr.aws/nginx/nginx:latest
LABEL maintainer jenkins@
ENV SERVICE_NAME=rl-manage-web SERVICE_PATH=/opt/web
COPY dist \${SERVICE_PATH}/site
EXPOSE 80
WORKDIR \${SERVICE_PATH}
ENV TZ Asia/Shanghai
EOF
"""
                sh "docker -H tcp://10.0.0.1:2375 tag  ${imgurl}:${TAG}  ${imgurl}:latest"
              }
              container('dind') {
                sh """
                  #docker -H tcp://10.0.0.1:2375 login --username=xx -p Wc=xx harbor.xxx.com
                  docker -H tcp://10.0.0.1:2375 push ${imgurl}:${TAG}
                  docker -H tcp://10.0.0.1:2375 push ${imgurl}:latest
                  docker -H tcp://10.0.0.1:2375 rmi -f ${imgurl}:${TAG} ${imgurl}:latest
                """
              }
            }
        }

# 开机 并检查 $i 为ec2实例id
aws ec2 start-instances  --instance-ids $i
aws --region ap-south-1 ec2 describe-instances --filters "Name=instance-id,Values=${i}" --query 'Reservations[*].Instances[*].[PrivateIpAddress]' --output text

# 关机 并检查
aws ec2 stop-instances  --instance-ids $i
aws --region ap-south-1 ec2 describe-instances --filters "Name=instance-id,Values=${i}" --query 'Reservations[*].Instances[*].[PrivateIpAddress]' --output text

      aws ec2 describe-instances --region ap-south-1 \
          --filters Name=tag:Name,Values=ludo-staging-ec2-new-stg-eks-app

      cat ec2-tag.log |jq '.Reservations[].Instances[]|[.PrivateIpAddress, .State.Name, ([(.Tags[]?|[.Key, .Value]|join("="))]|join("@"))] | join("|")'


aws ec2 describe-instances \
    --filters Name=instance-state-name,Values=running \
    --query 'Reservations[*].Instances[*].{Instance:InstanceId,IP:PrivateIpAddress,InstanceType:InstanceType,AZ:Placement.AvailabilityZone,Name:Tags[?Key==`Name`]|[0].Value,_SubModule:Tags[?Key==`_SubModule`]|[0].Value,__Usage:Tags[?Key==`_Usage`]|[0].Value,ClusterName:Tags[?Key==`eks:cluster-name`]|[0].Value,Techteam:Tags[?Key==`Techteam`]|[0].Value}' \
    --output table > a.log
cat a.log  |grep None  >b.log
cat b.log |grep -v RDS  |sort -n -k 11 >c.log

aws ec2 describe-instances --filters Name="network-interface.addresses.private-ip-address",Values="172.21.38.5" \
 --query 'Reservations[*].Instances[*].{Instance:InstanceId,AZ:Placement.AvailabilityZone,Name:Tags[?Key==`Name`]|[0].Value,_SubModule:Tags[?Key==`_SubModule`]|[0].Value,IP:PrivateIpAddress}' \
    --output table

#确定实例是基于 Xen 还是基于 Nitro
instance_type=m7g.2xlarge
aws ec2 describe-instance-types --instance-type $instance_type --query "InstanceTypes[].Hypervisor"

#nitro
#1.调整分区的大小
#检查卷是否有分区。使用 lsblk 命令. 根卷（nvme0n1）有两个分区（nvme0n1p1 和 nvme0n1p128
[ec2-user@ip-172-31-5-34 ~]$ sudo lsblk
NAME          MAJ:MIN RM  SIZE RO TYPE MOUNTPOINTS
nvme0n1       259:0    0  200G  0 disk 
├─nvme0n1p1   259:1    0  100G  0 part /
└─nvme0n1p128 259:2    0   10M  0 part /boot/efi
#扩展分区。使用 growpart 命令并指定设备名称和分区编号。
[ec2-user@ip-172-31-5-34 ~]$ sudo growpart /dev/nvme0n1 1
CHANGED: partition=1 start=22528 old: size=209692639 end=209715167 new: size=419407839 end=419430367
#验证是否已扩展分区
[ec2-user@ip-172-31-5-34 ~]$ sudo lsblk
NAME          MAJ:MIN RM  SIZE RO TYPE MOUNTPOINTS
nvme0n1       259:0    0  200G  0 disk 
├─nvme0n1p1   259:1    0  200G  0 part /
└─nvme0n1p128 259:2    0   10M  0 part /boot/efi

#2.扩展文件系统
#获取需要扩展的文件系统的名称、大小、类型和挂载点。使用 df -hT 命令。
[ec2-user@ip-172-31-5-34 ~]$ df -hT
Filesystem       Type      Size  Used Avail Use% Mounted on
/dev/nvme0n1p1   xfs       100G   92G  8.1G  92% /
#扩展文件系统的命令因文件系统类型而异
#[XFS 文件系统] 使用 xfs_growfs 命令并指定您在上一步中记录的文件系统的挂载点
[ec2-user@ip-172-31-5-34 ~]$ sudo xfs_growfs -d /
#[Ext4 文件系统] 使用 resize2fs 命令并指定您在上一步中记录的文件系统的名称
sudo resize2fs /dev/nvme0n1p1
#验证是否已扩展文件系统。使用 df -hT 命令并确认文件系统大小等于卷大小
[ec2-user@ip-172-31-5-34 ~]$ df -hT
Filesystem       Type      Size  Used Avail Use% Mounted on
/dev/nvme0n1p1   xfs       200G   93G  108G  47% /

# 将保存的 CiphertextBlob (base64 密文 DEK) 传给 KMS 解密，拿回明文 DEK