Kubernetes: k8s 运维篇-Prometheus监控

git clone -b release-0.8 --single-branch https://github.com/prometheus-operator/kube-prometheus

# cd kube-prometheus/manifests
# kubectl create -f setup/

# 查看是否Running
# kubectl get pod -n monitoring
NAME                                   READY   STATUS        RESTARTS   AGE
prometheus-operator-848d669f6d-bz2tc   2/2     Running       0          4m16s

cd kube-prometheus/manifests
kubectl create -f .

# 查看状态
kubectl get pod -n monitoring

containers:
- args:
  - --config.file=/etc/prometheus/prometheus.yml  # 指定配置文件
  - --query.timeout=1m
  - --storage.tsdb.wal-compression  # 数据压缩，默认开启
  - --query.max-samples=20000000
  - --storage.tsdb.path=/prometheus  # 数据存储路
  - --storage.tsdb.retention.time=90d  # 数据保留时间
  - --storage.tsdb.min-block-duration=1h
  - --storage.tsdb.max-block-duration=2h
  - --web.console.libraries=/etc/prometheus/console_libraries
  - --web.console.templates=/etc/prometheus/consoles
  - --web.enable-lifecycle  # 支持热更新，直接执行localhost:9090/-/reload立即生效
  #- --web.external-url=/eks
  - --web.external-url=https://prometheus.gxxx.com/eks  # 对外部地址，alertmanager 发送报警时传递该地址
  - --web.enable-admin-api  #  数据管理，删除数据等。参考 https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-admin-apis
  image: prom/prometheus:v2.34.0

# 创建一下Ingress代理三个service
apiVersion: networking.k8s.io/v1beta1
kind: Ingress
metadata:
  generation: 1
  name: prom-ingresses
  namespace: monitoring
spec:
  rules:
  - host: alert.test.com
    http:
      paths:
      - backend:
          serviceName: alertmanager-main
          servicePort: 9093
        path: /
  - host: grafana.test.com
    http:
      paths:
      - backend:
          serviceName: grafana
          servicePort: 3000
        path: /
  - host: prome.test.com
    http:
      paths:
      - backend:
          serviceName: prometheus-k8s
          servicePort: 9090
        path: /

# 在你Windows的hosts文件添加主机映射，浏览器访问即可
10.4.7.107 alert.test.com grafana.test.com prome.test.com

kubectl create deploy mysql-tmp --image=mysql:5.7.23

kubectl set env deploy/mysql-tmp MYSQL_ROOT_PASSWORD=mysql

kubectl expose deploy mysql-tmp --port 3306

kubectl get svc -l app=mysql-tmp
NAME        TYPE        CLUSTER-IP     EXTERNAL-IP   PORT(S)    AGE
mysql-tmp   ClusterIP   172.20.138.2   <none>        3306/TCP   19s

telnet 172.20.138.2 3306

$ kubectl  exec -it mysql-tmp-79d4dd8d8-jnbrg -- bash
root@mysql-tmp-79d4dd8d8-jnbrg:/# mysql -uroot -pmysql
mysql> create user 'exporter'@'%'  identified by 'exporter' with MAX_USER_CONNECTIONS 3;
mysql> grant process, replication client, select on *.* to 'exporter'@'%';

vim mysql-exporter.yaml

kind: Secret
metadata:
  name: prometheus-mysql-exporter
  namespace: monitoring
type: Opaque
data:
  password: "bXlzcWw="
---
apiVersion: v1
kind: Service
metadata:
  name: prometheus-mysql-exporter
  namespace: monitoring
  labels:
    k8s-app: prometheus-mysql-exporter
spec:
  type: ClusterIP
  ports:
  - port: 9104
    targetPort: 9104
    protocol: TCP
    name: mysql-exporter
  selector:
    k8s-app: prometheus-mysql-exporter
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: prometheus-mysql-exporter
  namespace: monitoring
spec:
  replicas: 1
  selector:
    matchLabels:
      k8s-app: prometheus-mysql-exporter
  template:
    metadata:
      labels:
        k8s-app: prometheus-mysql-exporter
      #annotations: # 动态发现
      #  prometheus.io/path: /metrics
      #  prometheus.io/port: "9104"
      #  prometheus.io/scrape: "true"
    spec:
      containers:
      - name: prometheus-mysql-exporter
        image: "prom/mysqld-exporter:v0.14.0"
        imagePullPolicy: IfNotPresent
        env:
        - name: DB_PASSWORD
          valueFrom:
            secretKeyRef:
              name: prometheus-mysql-exporter
              key: password
        - name: DATA_SOURCE_NAME
          value: "exporter:$(DB_PASSWORD)@(mysql-tmp.default:3306)/"
        ports:
        - containerPort: 9104
        livenessProbe:
          httpGet:
            path: /
            port: 9104
        readinessProbe:
          httpGet:
            path: /
            port: 9104
        resources:
          limits:
            cpu: "100m"
            memory: 100Mi
          requests:
            cpu: "10m"
            memory: 10Mi

$ kubectl  get svc -l k8s-app=prometheus-mysql-exporter -n monitoring
NAME                        TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)    AGE
prometheus-mysql-exporter   ClusterIP   172.20.227.60   <none>        9104/TCP   4m53s

$ curl 172.20.227.60:9104/metrics

$ cat mysql-sm.yaml
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  labels:
    k8s-app: prometheus-mysql-exporter
  name: prometheus-mysql-exporter
  namespace: monitoring
spec:
  endpoints:
  - interval: 30s
    path: /metrics
    port: api
    scheme: http
  selector:
    matchLabels:
      k8s-app: prometheus-mysql-exporter
    namepsaceSelector:
      matchNames:
      - monitoring

kubectl apply -f mysql-sm.yaml

- job_name: 'mysql_master'
  scrape_interval: 20s
  honor_labels: true
  static_configs:
  - targets:
    - "172.21.89.124:9104"
    - "172.21.89.243:9104"
    labels:
      business: App
      dbtype: db-mysql-db
      team: platform

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: prom-blackbox-exporter-config
  namespace: monitoring
  labels:
    app: prom-blackbox-exporter
data:
  blackbox-exporter.yaml: |-
    modules:
      http_2xx:
        prober: http
        timeout: 10s
        http:
          valid_http_versions: ["HTTP/1.1", "HTTP/2"]
          # [], Defaults to 2xx, 200,301,302,401,404
          valid_status_codes: [200]
          method: GET
          preferred_ip_protocol: "ip4"
      http_post_2xx:
        prober: http
        http:
          method: POST
      tcp_connect:
        prober: tcp
      pop3s_banner:
        prober: tcp
        tcp:
          query_response:
          - expect: "^+OK"
          tls: true
          tls_config:
            insecure_skip_verify: false
      ssh_banner:
        prober: tcp
        tcp:
          query_response:
          - expect: "^SSH-2.0-"
      irc_banner:
        prober: tcp
        tcp:
          query_response:
          - send: "NICK prober"
          - send: "USER prober prober prober :prober"
          - expect: "PING :([^ ]+)"
            send: "PONG ${1}"
          - expect: "^:[^ ]+ 001"
      icmp:
        prober: icmp
        timeout: 3s
        icmp:
          preferred_ip_protocol: "ip4"
---
apiVersion: v1
kind: Service
metadata:
  name: prom-blackbox-exporter
  namespace: monitoring
  annotations:
    prometheus.io/scrape: "true"
  labels:
    app: prom-blackbox-exporter
spec:
  selector:
    app: prom-blackbox-exporter
  ports:
    - name: tcp-9115
      protocol: TCP
      port: 9115
      targetPort: 9115
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: prom-blackbox-exporter
  namespace: monitoring
spec:
  replicas: 1
  selector:
    matchLabels:
      app: prom-blackbox-exporter
  template:
    metadata:
      labels:
        app: prom-blackbox-exporter
    spec:
      nodeSelector:
        type: pfgc-ops-prod-prometheus
      containers:
      - name: blackbox-exporter
        image: prom/blackbox-exporter:v0.22.0
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 9115
        args:
        - --config.file=/etc/blackbox_exporter/config.yaml
        - --log.level=info
        - --web.listen-address=:9115
        readinessProbe:
          tcpSocket:
            port: 9115
          initialDelaySeconds: 5
          timeoutSeconds: 5
        resources:
          limits:
            cpu: 200m
            memory: 128Mi
          requests:
            cpu: 10m
            memory: 20Mi
        volumeMounts:
        - mountPath: /etc/blackbox_exporter
          name: config
          readOnly: true
      - args:
        - --webhook-url=http://localhost:9115/-/reload
        - --volume-dir=/etc/blackbox_exporter/
        image: jimmidyson/configmap-reload:v0.5.0
        name: module-configmap-reloader
        resources:
          limits:
            cpu: 20m
            memory: 40Mi
          requests:
            cpu: 10m
            memory: 20Mi
        securityContext:
          allowPrivilegeEscalation: false
          capabilities:
            drop:
            - ALL
          readOnlyRootFilesystem: true
          runAsNonRoot: true
          runAsUser: 65534
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: FallbackToLogsOnError
        volumeMounts:
        - mountPath: /etc/blackbox_exporter/
          name: config
          readOnly: true
      volumes:
      - name: config
        configMap:
          name: prom-blackbox-exporter-config

# 测试exporter是否正常
# 查看svc的IP
[root@k8s-master01 ~]# kubectl get svc -n monitoring  blackbox-exporter
NAME                TYPE        CLUSTER-IP    EXTERNAL-IP   PORT(S)    AGE
blackbox-exporter   ClusterIP   10.100.9.18   <none>        9115/TCP   30m

# curl一下exporter的svc
[root@k8s-master01 ~]# curl "http://10.100.9.18:9115/probe?target=baidu.com&module=http_2xx"

touch prometheus-additional.yaml
kubectl create secret generic additional-configs --from-file=prometheus-additional.yaml

#  kubectl edit prometheus -n monitoring k8s
apiVersion: monitoring.coreos.com/v1
kind: Prometheus
metadata:
  name: k8s
spec:
  replicas: 2
... 加上下面3行
  additionalScrapeConfigs:
    name: additional-configs
    key: prometheus-additional.yaml
    optional: true
...

- source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]  # 172.16.202.106:53;9153
  separator: ;
  regex: ([^:]+)(?::\d+)?;(\d+)
  target_label: __address__
  replacement: $1:$2
  action: replace

- job_name: 'node-exporter-discovery'
  kubernetes_sd_configs:
  - role: node
  relabel_configs:
  - separator: ;
    regex: __meta_kubernetes_node_label_(.+)
    replacement: $1
    action: labelmap
  - source_labels: [__address__]
    regex: '(.*):10250'
    replacement: '${1}:9100'
    target_label: __address__
    action: replace
  metric_relabel_configs:
  - source_labels: [ __name__ ]
    regex: (node_namespace_pod_container:container_memory_rss|node_namespace_pod_container:container_memory_working_set_bytes|node_namespace_pod_container:container_memory_cache|node_scrape_collector_duration_seconds|node_scrape_collector_success|node_cpu_core_throttles_total|node_cpu_guest_seconds_total|node_ipvs_backend_connections_active|node_ipvs_backend_weight|node_ipvs_backend_connections_inactive|node_filesystem_device_error|node_cpu_core_throttles_total|node_nfs_requests_total|node_scrape_collector_success|node_scrape_collector_duration_seconds|node_cpu_scaling_frequency_min_hrts|node_cpu_scaling_frequency_hertz|node_cpu_frequency_max_hertz|node_cpu_scaling_frequency_max_hrts|node_cpu_frequency_min_hertz)
    action: drop

- job_name: 'prometheus-k8s'
  scrape_interval: 60s
  scrape_timeout: 10s
  honor_labels: true
  metrics_path: '/k8s/federate'
  params:
    'match[]':
      - '{job="node-exporter"}'
      - '{job=~"kube-.*"}'
  static_configs:
    - targets:
      - 'prometheus.xxx.com'


- consul_sd_configs:
  - server: consul-server.consul:8500
  job_name: consul-exporter
  relabel_configs:
  - action: replace
    source_labels:
    - __meta_consul_service_metadata_Application
    target_label: application
  - action: replace
    source_labels:
    - __meta_consul_service_metadata_Environment
    target_label: environment
  - action: replace
    source_labels:
    - __meta_consul_service_metadata_SubModule
    target_label: subModule
  - action: replace
    source_labels:
    - __meta_consul_service_metadata_Usage
    target_label: usage
  - action: replace
    source_labels:
    - __meta_consul_service_metadata_instanceID
    target_label: instanceID

- job_name: node-exporter
  honor_timestamps: true
  scrape_interval: 15s
  scrape_timeout: 10s
  metrics_path: /metrics
  scheme: http
  follow_redirects: true
  kubernetes_sd_configs:
  - role: node
    kubeconfig_file: ""
    follow_redirects: true
  relabel_configs:
  - source_labels: [__address__] # node 的 __address__  是`节点ip:10250`
    separator: ;
    regex: (.*):10250
    target_label: __address__
    replacement: $1:9100
    action: replace

- job_name: kubernetes-kubelet
  kubernetes_sd_configs:
  - role: node
  relabel_configs:
  - separator: ;
    regex: __meta_kubernetes_node_label_(.+)
    replacement: $1
    action: labelmap
  - separator: ;
    regex: '(.*):10250'
    target_label: __address__
    replacement: '${1}:10250'
    action: replace

# deploymnet 中 pod 标签
spec:
  template:
    metadata:
      annotations:
        prometheus.io/pathjvm: /actuator/prometheus
        prometheus.io/portjvm: "8099"
        prometheus.io/scrapejvm: "true"

# prometheus 配置
      - source_labels: [__meta_kubernetes_pod_container_name]
        regex: "filebeat.*"
        action: drop   # 去掉不需要容器
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrapejvm]
        action: keep
        regex: true  # 找到 prometheus.io/scrapejvm: "true" 的 pod
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_pathjvm]
        action: replace
        target_label: __metrics_path__
        regex: (.+)    # 找到 prometheus.io/pathjvm: /actuator/prometheus 的路径作为 metrics
      - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_portjvm]
        action: replace
        regex: ([^:]+)(?::\d+)?;(\d+)
        replacement: $1:$2
        target_label: __address__  # 找到 prometheus.io/portjvm: "8099" 所设置的端口，拼接到 __address__
      - action: labelmap
        regex: __meta_kubernetes_pod_label_(.+) #  将 pod 的标签映射到 metrcis 中

# summary类型
[root@ip-172-21-39-120 prometheus]# curl -s 172.21.39.111:9090/111/metrics |grep go_gc_duration_seconds -C5
# HELP go_gc_duration_seconds A summary of the pause duration of garbage collection cycles.
# TYPE go_gc_duration_seconds summary
go_gc_duration_seconds{quantile="0"} 1.421e-05
go_gc_duration_seconds{quantile="0.25"} 5.5171e-05
go_gc_duration_seconds{quantile="0.5"} 6.2903e-05
go_gc_duration_seconds{quantile="0.75"} 8.665e-05
go_gc_duration_seconds{quantile="1"} 0.004545852
go_gc_duration_seconds_sum 88.949090905
go_gc_duration_seconds_count 1.062595e+06

# 每秒增长率 = 1 小时增长率，首尾值 / 3600
increase(http_request_total{endpoint="http",handler="/datasources/proxy/:id/*",instance="10.244.58.200:3000",job="grafana",method="get",namespace="monitoring",pod="grafana-86b55cb79f-fn4ss",service="grafana",statuscode="200"}[1h]) / 3600

# 用 rate 同样效果
rate(http_request_total{endpoint="http",handler="/datasources/proxy/:id/*",instance="10.244.58.200:3000",job="grafana",method="get",namespace="monitoring",pod="grafana-86b55cb79f-fn4ss",service="grafana",statuscode="200"}[1h])

# 根据一天的数据，预测4个小时之后，磁盘分区的空间会不会小于0
predict_linear(node_filesystem_files_free{mountpoint="/"}[1d], 4*3600) < 0

# 1 小时的磁盘差
delta(node_filesystem_files_free[1h])

sort(node_filesystem_files_free)

# 将 instance, mountpoint 赋给新 label new_label，用 , 逗号来做分隔符
label_join(node_filesystem_files_free, "new_label", ",",  "instance", "mountpoint")

# 将 instance 中第 2 个值给 host 标签
label_replace(node_filesystem_files_free, "host","$2", "instance", "(.*)-(.*)")

groups:
- name: cardgames
  rules:
  - alert: rummy_user_thread_pending_task
    expr: thread_pending_task{service="rummy-user"} > 100
    #expr: 1 - avg by (instance, job, mode) (irate(node_cpu_seconds_total{mode="idle", instance="172.21.89.124:9100"}[5m])) > 0
    for: 1m
    labels:
      severity: warning
      team: cardgames-rummy
    annotations:
      runbook_url: https://paytmfirstgames.atlassian.net/l/c/xxxx
      summary: "Rummy-user thread_pending_task is too high."
      description: "warning | The thread_pending_task value of {{ $labels.service }} {{ $labels.instance }} thread \"{{ $labels.thread }}\" is {{ $value }} > 100  [Time: 1m]."

global:
...
  smtp_smarthost: 'smtp.qq.com:465'
  smtp_from: '[email protected]'
  smtp_auth_username: '[email protected]'
  smtp_auth_password: 'xxx'
  smtp_require_tls: false  

receivers:
- name: 'team-Y-mails'
  email_configs:
  - to: '[email protected]'
    send_resolved: true  # 是否发恢复的通知

global:
...
  # wechat
  wechat_api_url: 'https://qyapi.weixin.qq.com/cgi-bin/'
  wechat_api_corp_id: 'wwef86a30130f04f2b'

receivers:
- name: 'wechat'
  wechat_configs:
  - send_resolved: true
    to_party: 3
    to_user: '@all'
    agent_id: '1000003'
    api_secret: 'ZZQt0Ue9mtplH9u1g8PhxR_RxEnRu512CQtmBn6R2x0'

receivers:
    slack_configs:
      - api_url: "此处配置机器人地址"
        channel: "#fchannel_name"
        send_resolved: false
        title: '[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }}'
        text: |-
          {{ range .Alerts }}
          *Alert:* {{ .Annotations.summary }} - `{{ .Labels.severity }}`
          *Description:* {{ .Annotations.description }}
          *Details:*
            {{ range .Labels.SortedPairs }} *{{ .Name }}:* `{{ .Value }}`
            {{ end }}
          {{ end }}
        short_fields: true

receivers:
- name: "app-webhook-critical"
  webhook_configs:
  - url: 'http://localhost:8060/dingtalk/app-webhook-critical/send'
    send_resolved: true
- name: "wallet-webhook-critical"
  webhook_configs:
  - url: 'http://localhost:8060/dingtalk/wallet-webhook-critical/send'
    send_resolved: true
  slack_configs:
  - api_url: "https://hooks.slack.com/services/T026NT2D4/B0jxxxxxxxxxxx"
    channel: "#pfg-alert-wallet"
    text: "{{ range .Alerts }}{{ .Annotations.summary }}\n Value: {{ .Annotations.description }}\n{{ end }}"
    send_resolved: true

curl --location --request POST 'https://hooks.slack.com/services/T5gZxxx' \
--header 'Content-Type: application/json' \
--data-raw '{
    “text”: “Hello, world.”
}'

# prometheus 报警规则文件
      - alert: poker_new_games_dropped_critical
        expr: *
        for: 10m
        labels:
          severity: critical
          team: cardgames-poker
          duty: cardgames_po|infra_op
        annotations:
          runbook_url: https://xxx
          summary:  'x'
          description: 'x'

# alertmanager 配置文件
    receivers:
    - name: "cardgames-business-webhook-critical"
      webhook_configs:
      - url: 'http://prometheus-webhook-dingtalk:8060/dingtalk/cardgames-business-webhook-critical/send'
        send_resolved: true
      - send_resolved: false
        url: "http://ops.xxx.com/api/v1/monitoringWebhook"
      slack_configs:
      - api_url: "https://hooks.slack.com/services/T026NT2D4/B033SJ1DFT6/xx"
        channel: "#pfg-alert-cardgames"
        text: "{{ range .Alerts }}{{ .Annotations.summary }}\n Value: {{ .Annotations.description }}\n{{ end }}"
        send_resolved: true

# 创建alertmanager.yaml的secret
kubectl create secret generic  alertmanager-main --from-file=alertmanager.yaml --from-file=wechat.tmpl   -n monitoring
# 之后更改alertmanager.yaml可以使用热加载去更新k8s的secret
kubectl create secret generic  alertmanager-main --from-file=alertmanager.yaml -n monitoring --dry-run -o yaml | kubectl replace -f -

{{ define "wechat.default.message" }}
{{ if gt (len .Alerts.Firing) 0 -}}
Alerts Firing:
{{ range .Alerts }}
告警级别：{{ .Labels.severity }}
告警类型：{{ .Labels.alertname }}
故障主机: {{ .Labels.instance }}
Job名称: {{.Labels.job}}
告警详情: {{ .Annotations.message }}
触发时间: {{ .StartsAt.Format "2006-01-02 15:04:05" }}
{{- end }}
{{- end }}
{{ if gt (len .Alerts.Resolved) 0 -}}
Alerts Resolved:
{{ range .Alerts }}
告警级别：{{ .Labels.severity }}
告警类型：{{ .Labels.alertname }}
故障主机: {{ .Labels.instance }}
Job名称: {{.Labels.job}}
告警主题: {{ .Annotations.message }}
触发时间: {{ .StartsAt.Format "2006-01-02 15:04:05" }}
恢复时间: {{ .EndsAt.Format "2006-01-02 15:04:05" }}
{{- end }}
{{- end }}
告警链接:
{{ template "__alertmanagerURL" . }}  # 替换成域名
{{- end }}

templates:
- '/etc/alertmanager/config/*.tmpl'

- name: 'wechat'          # 微信告警
  wechat_configs:
  - send_resolved: true   # 报警解决的通过是否发送
    corp_id: 'ww'
    api_secret: 'JJ'
    to_tag: '1'
    agent_id: '1000002'
    api_url: 'https://qyapi.weixin.qq.com/cgi-bin/'
    message: '{{ template "wechat.default.message" . }}'  # 和模板中 define 名称一致

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  labels:
    prometheus: k8s
    role: alert-rules
  name: node-exporter-rules
  namespace: monitoring
spec:
  groups:
  - name: node-exporter
    rules:
    - alert: NodeFilesystemSpaceFillingUp
      annotations:
        description: Filesystem on {{ $labels.device }} at {{ $labels.instance }}
          has only {{ printf "%.2f" $value }}% available space left and is filling
          up.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemspacefillingup
        summary: Filesystem is predicted to run out of space within the next 24 hours.
      expr: |
        (
          node_filesystem_avail_bytes{job="node-exporter",fstype!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!=""} * 100 < 15
        and
          predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!=""}[6h], 24*60*60) < 0
        and
          node_filesystem_readonly{job="node-exporter",fstype!=""} == 0
        )
      for: 1h
      Labels:
        severity: warning

node.rules.yml: |
  groups:
  - name: ssl_cert_expiry # 证书到期告警
    interval: 1d  # 检查间隔
    rules:
    - alert: taskcenter_ssl_cert_expiry_less_than_30days
      expr: (probe_ssl_earliest_cert_expiry{job="blackbox-http"} - time()) /60/60/24  < 30
      labels:
        severity: warning
        type: blackbox
      annotations:
        summary: "ssl cert expiry less than 30 days"
        description: "warning | Current value = {{ $value }}%. | [Duration Time: 1d]"

  - name: url_not_available # 网站不可用告警
    interval: 1m
    rules:
    - alert: url_not_available
      expr: probe_http_status_code{job="blackbox-http"} != 200
      labels:
        severity: warning
        type: blackbox
      annotations:
        summary: "url not available"

  - name: url_delay # 网站延迟
    rules:
    - alert: DomainAccessDelayExceedsls
      expr: sum by (instance) (probe_http_duration_seconds{job="blackbox-http"}) > 1
      for: 1m
      labels:
        severity: warning
        type: blackbox
      annotations:
        summary: "Domain is delayed by more than 1 second"
        description: "The domain {{$labels.instance}} is delayed by more than 1 second, current value is {{ $value | printf "%.2f" }} second"

#----- 自动发现 添加以下内容
- job_name: 'auto_discovery'
  metrics_path: /probe
  params:
    module: [http_2xx]  
  kubernetes_sd_configs:  # 使用 k8s 的服务发现
  - role: ingress  # 服务发现可以基于 ingress, pod, svc, ... ...
  relabel_configs:
  - source_labels: [__meta_kubernetes_ingress_annotation_prometheus_io_http_probe]
    action: keep
    regex: true  # ingress 配置了上面的标签值为true的会加入自动发现
  - source_labels: [__meta_kubernetes_ingress_scheme,__address__,__meta_kubernetes_ingress_path]
    regex: (.+);(.+);(.+)
    replacement: ${1}://${2}${3} # 用上面匹配到的标签拼成 url
    target_label: __param_target
  - source_labels: [__meta_kubernetes_ingress_scheme,__address__,__meta_kubernetes_ingress_path]
    regex: (.+);(.+);(.+)
    replacement: ${1}://${2}${3}
    target_label: target
  - target_label: __address__
    replacement: blackbox-exporter:9115
  - source_labels: [__param_target]
    target_label: instance
  - action: labelmap   # ingress 的标签赋值给 metric 的lable
    regex: __meta_kubernetes_ingress_label_(.+)
  - source_labels: [__meta_kubernetes_namespace]
    target_label: kubernetes_namespace
  - source_labels: [__meta_kubernetes_ingress_name]
    target_label: kubernetes_name

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: prometheus-discovery 
  namespace: monitoring
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: ratel-resource-readonly
subjects:
- namespace: monitoring 
  kind: ServiceAccount
  name: prometheus-k8s

# 监控域名，ingress 中添加如下注释
  annotations:
    prometheus.io/http_probe: true

<dependecies> # 添加如下内容
     <!-- Micrometer Prometheus registry  -->
     <dependency>
     <groupId>org.springframework.boot</groupId>
     <artifactId>spring-boot-starter-actuator</artifactId>
       </dependency>
      <dependency>
             <groupId>io.micrometer</groupId>
              <artifactId>micrometer-core</artifactId>
      </dependency>
      <dependency>
             <groupId>io.micrometer</groupId>
             <artifactId>micrometer-registry-prometheus</artifactId>
      </dependency>
     <!-- finished -->

cd /opt
git clone https://github.com/gongchangwangpi/spring-cloud-demo2
cd spring-cloud-demo2

# 编译
docker run -it --rm -v /opt/m2:/root/.m2 -v `pwd`:/opt -p 18761:8761 maven:3.5.3 bash
> cd /opt
> cd spring-cloud-eureka
> vim pom.xml  # 添加以下内容
   <dependecies> # 添加如下内容
        <!-- Micrometer Prometheus registry  -->
        <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-actuator</artifactId>
          </dependency>
         <dependency>
                <groupId>io.micrometer</groupId>
                 <artifactId>micrometer-core</artifactId>
         </dependency>
         <dependency>
                <groupId>io.micrometer</groupId>
                <artifactId>micrometer-registry-prometheus</artifactId>
         </dependency>
        <!-- finished -->
# 将 micrometer 的接口暴露出来
> vim src/main/resources/application.yml # 新增以下内容
spring:
  application:
    name: cloud-eureka
management:
  endpoints:
    web:
      exposure:
        include: '*'
    shutdown:
      enable: false
  metrics:
    tags:
      application: "${spring.application.name}"

# maven编译命令：
> mvn clean package -DskipTests      
# 启动 euraka 服务
> java -jar target/spring-cloud-eureka-0.0.1-SNAPSHOT.jar

# 新增如下内容
[root@k8s-master01 prometheus-down]# vim prometheus-additional.yaml 
- job_name: 'jvm-prometheus'
  scheme: http
  metrics_path: '/actuator/prometheus'
  static_configs:
  - targets: ['xxx:8080']

vim pom.xml 
<dependency>
    <groupId>at.twinformatics</groupId>
    <artifactId>eureka-consul-adapter</artifactId>
    <version>1.1.0</version>
</dependency>

apiVersion: v1
items:
- apiVersion: v1
  kind: Service
  metadata:
    creationTimestamp: "2020-04-25T14:42:04Z"
    labels:
      k8s-app: kube-controller-manager
    name: kube-controller-manage-monitor
    namespace: kube-system
    resourceVersion: "10081547"
    selfLink: /api/v1/namespaces/kube-system/services/kube-controller-manage-monitor
    uid: d82d9170-9335-49b8-9aae-48630eb6efd4
  spec:
    clusterIP: 10.96.23.157
    ports:
    - name: http-metrics
      port: 10252
      protocol: TCP
      targetPort: 10252
    sessionAffinity: None
    type: ClusterIP
  status:
    loadBalancer: {}
- apiVersion: v1
  kind: Endpoints
  metadata:
    creationTimestamp: "2020-04-25T14:41:16Z"
    labels:
      k8s-app: kube-controller-manager
    name: kube-controller-manage-monitor
    namespace: kube-system
    resourceVersion: "10081388"
    selfLink: /api/v1/namespaces/kube-system/endpoints/kube-controller-manage-monitor
    uid: c7d0214b-58a2-4d05-8cfe-673e914e06b4
  subsets:
  - addresses:
    - ip: 192.168.1.19
    ports:
    - name: http-metrics
      port: 10252
      protocol: TCP
kind: List
metadata:
  resourceVersion: ""
  selfLink: ""

container_cpu_cfs_periods_total：container生命周期中度过的cpu周期总数
container_cpu_cfs_throttled_periods_total：container生命周期中度过的受限的cpu周期总数

sum by(container, pod, namespace) (increase(container_cpu_cfs_throttled_periods_total{container!=""}[5m])) / sum by(container, pod, namespace) (increase(container_cpu_cfs_periods_total[5m])) > (25 / 100)

String to interpolate: '${My:value}'
Interpolation result: '{aaa|AAA,bbb|BBB,ccc|CCC}'


#不适合多选
表达式：
sum(sum by(url, game, code) (rate(Fb_Net_Code_count{game=~".*",url=~".*(${My:value})",url!~".*\\.(png|jpg|jpeg|json)",prom_cluster="prod-eks-client"}[30s])))by (url)
解析后： 可以看到|分隔成 {aaa、AAA,bbb、ccc|CCCC}
sum(sum by(url, game, code) (rate(Fb_Net_Code_count{game=~".*",url=~".*({aaa|AAA,bbb|BBB,ccc|CCCC})",url!~".*\\.(png|jpg|jpeg|json)",prom_cluster="prod-eks-client"}[30s])))by (url)

String to interpolate: '${My:pipe}'
Interpolation result: 'aaa|AAA|bbb|BBB|ccc|CCC'

表达式：
sum(sum by(url, game, code) (rate(Fb_Net_Code_count{game=~".*",url=~".*(${My:pipe})",url!~".*\\.(png|jpg|jpeg|json)",prom_cluster="prod-eks-client"}[30s])))by (url)
解析后：
sum(sum by(url, game, code) (rate(Fb_Net_Code_count{game=~".*",url=~".*(aaa|AAA|bbb|BBB|ccc|CCCC)",url!~".*\\.(png|jpg|jpeg|json)",prom_cluster="prod-eks-client"}[30s])))by (url)

--web.listen-address=":9091"      监听Web界面，API和遥测的地址。
--web.telemetry-path="/metrics"     公开metrics的路径。
--web.external-url=             可从外部访问Pushgateway的URL.
--web.route-prefix=""           Web端点内部路由的前缀。 默认为--web.external-url的路径.
--persistence.file=""           归档以保留metrics。 如果为空，则metrics仅保留在内存中.
--persistence.interval=5m       写入持久性文件的最小间隔。
--log.level="info"              仅记录具有给定严重性或更高严重性的消息。 有效级别：[debug, info, warn, error, fatal]
--log.format="logger:stderr"      设置日志目标和格式。 示例：“ logger：syslog？appname = bob＆local = 7”或“ logger：stdout？json = true”
--version                       显示应用程序版本。

curl -X DELETE http://pushgateway_ip:9091/metrics/job/some_job/instance/some_instance

curl -X PUT http://192.168.0.106:30893/api/v1/admin/wipe

cat service_status.conf
[program:service_monitor]
command =  /bin/bash /gamepind/devops/alivecheck/service_monitor.sh
user = root
autostart = true
autorestart = true
redirect_stderr=true
stdout_logfile_backups = 5
stdout_logfile = /gamepind/devops/alivecheck/service_monitor.log


[root@ip-172-21-89-103 conf.d]# cat /gamepind/devops/alivecheck/service_monitor.sh
#!/bin/bash
#

source /etc/profile

SpringBoot=$1
hostname1=$(hostname)
hostname=${hostname1%%.*}
service_list=("rummy-center" "rummy-game" "rummy-gateway" "rummy-room" "rummy-user")

while true; do
  sv1=""
  let n=0
  let m=${#service_list[*]}
  for i in ${service_list[*]}; do
        va=$(jps -l |grep  " $i.jar"|awk '{print $1}')
        if [ ! -z "$va" ]; then
                port=$(sudo netstat -tnlp|grep $va|awk '{print $4}'|awk -F: '{print $NF}')
                echo "podrt"$port
                if [ -z $port ]; then
                        cr=0
                else
                        cr=$(curl -G -m 10 --connect-timeout 60 -o /dev/null -s -w %{http_code} http://127.0.0.1:$port/api/health/check)
                fi
      else
          cr=0
      fi
      echo "$i-cr"$cr
        sv1=$sv1"service_status{service_name=\"$i\"} $cr"
        if [ $n -lt $m ]; then
          sv1=$sv1"\n"
        fi
  done
echo -e $sv1 | curl -s --data-binary @- http://172.21.39.120:9091/metrics/job/rummy/hostname/$hostname/business/Rummy
      #echo -e $sv1
  sleep 10
done

#!/bin/bash
instance_name=`ifconfig |grep 172.21|awk '{print $2}'`
while (( 0 < 1))
do
    count_mysql_long_tran_list=`/usr/local/mysql/bin/mysql -u'username' -p'******' -h172.21.89.151 -N -e"
select count(*) from information_schema.innodb_trx where TIMESTAMPDIFF( SECOND, trx_started, now() ) >= 0.5 and TIMESTAMPDIFF( SECOND, trx_started, now() ) < 10 and trx_query not like 'alter%'
UNION ALL
select count(*) from information_schema.innodb_trx where TIMESTAMPDIFF( SECOND, trx_started, now() ) >= 10 and TIMESTAMPDIFF( SECOND, trx_started, now() ) < 30 and trx_query not like 'alter%'
UNION ALL
select count(*) from information_schema.innodb_trx where TIMESTAMPDIFF( SECOND, trx_started, now() ) >= 30 and TIMESTAMPDIFF( SECOND, trx_started, now() ) < 60 and trx_query not like 'alter%'
UNION ALL
select count(*) from information_schema.innodb_trx where TIMESTAMPDIFF( SECOND, trx_started, now() ) >= 60 and trx_query not like 'alter%';"`
    echo $count_mysql_long_tran_list|awk -F " " '{print}'|while read count_mysql_long_tran_05 count_mysql_long_tran_10 count_mysql_long_tran_30 count_mysql_long_tran_60
    do
        echo "count_mysql_long_tran_05 $count_mysql_long_tran_05"|curl --data-binary @- http://172.21.39.120:9091/metrics/job/pushgatway/instance/$instance_name/business/Wallet
        echo "count_mysql_long_tran_10 $count_mysql_long_tran_10"|curl --data-binary @- http://172.21.39.120:9091/metrics/job/pushgatway/instance/$instance_name/business/Wallet
        echo "count_mysql_long_tran_30 $count_mysql_long_tran_30"|curl --data-binary @- http://172.21.39.120:9091/metrics/job/pushgatway/instance/$instance_name/business/Wallet
        echo "count_mysql_long_tran_60 $count_mysql_long_tran_60"|curl --data-binary @- http://172.21.39.120:9091/metrics/job/pushgatway/instance/$instance_name/business/Wallet
    done
sleep 1
done

[root@ip-172-21-39-120 conf.d]# cat  /data/devops/wallet_sql_exception_check/check_index.py
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
'''
@File    :   1.py
@Time    :   2021/01/19 16:18:46
@Author  :   jack fu
@Version :   1.0
@Contact :   [email protected]
'''
import datetime
import time
import requests
base_url = "http://172.21.38.147:9200"


def send_dingtalk(data):
    #token = "0a2e4d6f12ee3c3fdcfe04bde8ca60ff3562990d649d8682cb2167f5d1032aa4"
    #token = "bf37ece0c907d0c9113fd08a6d2ff36ef7ee3b1652bc021083443583d6f613b3"
    #token = "d6eaf9c83159ff195df59d4dda9bae9da18adf1c2908a7f1facd688cb2b57645"
    #token = "bf37ece0c907d0c9113fd08a6d2ff36ef7ee3b1652bc021083443583d6f613b3"
    token = "ac3e21684de52a3565a93a735ecd3cdc8e75b356b50dca4bd26ecb80cca5147c"
    if not token:
        print('you must set ROBOT_TOKEN env')
        return
    url = 'http://maven.gamepind.com/robot/send?access_token=%s' % token
    send_data = {"msgtype": "text", "text": {"content": data}}
    req = requests.post(url, json=send_data)
    result = req.json()
    print("error", result)
    if result['errcode'] != 0:
        print('notify dingtalk error: %s' % result['errcode'])


#def get_mysql_gone_away_err(time_start, time_end,error_type='php-mis-err'):
def get_mysql_gone_away_err():
    response = requests.get(
        f'{base_url}/wallet-*/_search?filter_path=aggregations.group.buckets',
        headers={'content-type': 'application/json'},
        json={
            "sort": [{
                "@timestamp": {
                    "order": "desc",
                    "unmapped_type": "boolean"
                }
            }],
            "aggs": {
                "group": {
                    "date_histogram": {
                        "field": "@timestamp",
                        "interval": "1d",
                        "time_zone": "Asia/Shanghai",
                        "min_doc_count": 1
                    }
                }
            },
            "query": {
                "bool": {
                    "must": [{
                        "query_string": {
                            "query": "\"Connection is not available\"",
                            "analyze_wildcard": True,
                            "default_field": "*"
                        }
                    }, {
                        "range": {
                            "@timestamp": {
                                "gt": "now-2m",
                                "lt": "now"
                            }
                        }
                    }]
                }
            }
        })
    print(response.text)
    #get_mysql_gone_away_err()
    buckets = response.json().get('aggregations').get('group').get('buckets')
    #print(buckets)
    date_cnt_arr = []
    for _sub_bucket in buckets:
        date_cnt_arr.append( f'''Alert: wallet sql exception Connection is not avaliable {_sub_bucket.get('key_as_string')[:10]} count: {_sub_bucket.get('doc_count')}''')
    # return int(time.time()) - utc_to_local(end_source['@timestamp'])
    alert_msg = "\n".join(date_cnt_arr)
    print(alert_msg)

    if len(alert_msg) > 5:
        send_dingtalk(alert_msg)
        time.sleep(30)
    else:
        pass


if __name__ == "__main__":
    while True:
      get_mysql_gone_away_err()
      time.sleep(30)

#!/bin/bash
#

source /etc/profile

SpringBoot=$1
hostname1=$(hostname)
hostname=${hostname1%%.*}
service_list=("rummy-center" "rummy-game" "rummy-gateway" "rummy-room" "rummy-user")

while true; do
  sv1=""
  let n=0
  let m=${#service_list[*]}
  for i in ${service_list[*]}; do
        va=$(jps -l |grep  " $i.jar"|awk '{print $1}')
        if [ ! -z "$va" ]; then
                port=$(sudo netstat -tnlp|grep $va|awk '{print $4}'|awk -F: '{print $NF}')
                echo "podrt"$port
                if [ -z $port ]; then
                        cr=0
                else
                        cr=$(curl -G -m 10 --connect-timeout 60 -o /dev/null -s -w %{http_code} http://127.0.0.1:$port/api/health/check)
                fi
      else
          cr=0
      fi
      echo "$i-cr"$cr
        sv1=$sv1"service_status{service_name=\"$i\"} $cr"
        if [ $n -lt $m ]; then
          sv1=$sv1"\n"
        fi
  done
echo -e $sv1 | curl -s --data-binary @- http://172.21.39.120:9091/metrics/job/rummy/hostname/$hostname/business/Rummy
      #echo -e $sv1
  sleep 10
done

cat  /gamepind/devops/alivecheck/process_monitor.sh
#!/bin/bash
#
SpringBoot=$1
hostname1=$(hostname)
hostname=${hostname1%%.*}
service_list=("rummy-center" "rummy-game" "rummy-gateway" "rummy-room" "rummy-user")

while true; do
  sv1=""
  let n=0
  let m=${#service_list[*]}
  for i in ${service_list[*]}; do
      n=$((n+1))
      #z=$(ps -ef |grep java|grep $i.jar|grep -v grep|wc -l)
      # last modtify by song 20200205 add merchantpush svc monitor
      z=$(ps -ef |grep java|grep " $i.jar"|grep -v grep|wc -l)
      sv1=$sv1"alive_check{service_name=\"$i\"} $z"
      if [ $n -lt $m ]; then
        sv1=$sv1"\n"
      fi
  done
echo -e $sv1 | curl -s --data-binary @- http://172.21.39.120:9091/metrics/job/rummy/hostname/$hostname/business/Rummy
      echo -e $sv1
  sleep 10
done
#curl -XPOST -H "Content-Type: text/plain" --data "$var
#" http://172.21.39.69:9091/metrics/job/top/instance/machine

cat /gamepind/devops/alivecheck/redisClusterAliveCheck.sh
#!/bin/bash
#
SpringBoot=$1
hostname1=$(hostname)
hostname=${hostname1%%.*}
#service_list=("mainserver" "eurekaserver")
#service_list=('["push.jar", "usercenter.jar", "scheduler.jar", "mylibrary.jar", "game.jar", "filesystem.jar"]')
server_list=("agt-newrummy-prod.8r1pyf.ng.0001.aps1.cache.amazonaws.com" "agt-newrummy-prod-001.8r1pyf.0001.aps1.cache.amazonaws.com" "agt-newrummy-prod-002.8r1pyf.0001.aps1.cache.amazonaws.com
")
#/usr/bin/redis-cli -h  agt-gpl-game-cache.8r1pyf.0001.aps1.cache.amazonaws.com EXISTS "MMO_SERVER_ADDRESS"
while true; do
  sv2=""
  let n=0
  let m=${#server_list[*]}
  for i in ${server_list[*]}; do
      n=$((n+1))
      #z=$(/usr/bin/redis-cli -h  agt-gpl-game-cache.8r1pyf.0001.aps1.cache.amazonaws.com EXISTS $i)
      echo "***************************"$(date)"*********************************" >> /gamepind/devops/alivecheck/redisali.log
      $(timeout 5 /bin/redis-cli -h $i INFO >> /gamepind/devops/alivecheck/redisali.log)
      #echo "ZZZZ" $z
      sv1=$?
      sv2=$sv2"redisAliveCheck{key_name=\"$i\"} $sv1"

      if [ "$sv1" -ne 0 ]; then
        echo "***************************"${hostname}${i} connected out "*********************************" >> /gamepind/devops/alivecheck/redisali.log
      fi

      #echo "iiiiiiqqq" $?
      if [ $n -lt $m ]; then
        sv2=$sv2"\n"
      fi
   done
      echo -e $sv2 | curl --data-binary @- http://172.21.39.120:9091/metrics/job/redisAliveCheck/hostname/$hostname
      echo -e $sv2
  sleep 5
done
#curl -XPOST -H "Content-Type: text/plain" --data "$var
#" http://172.21.39.69:9091/metrics/job/top/instance/machine

 cat /data/devops/prometheus-webhook-dingtalk/contrib/templates/legacy/template.tmpl
// {{ define "ding.link.title" }}{{ template "legacy.title" . }}{{ end }}
//  {{ define "ding.link.content" }}{{ template "legacy.content" . }}{{ end }}
{{ define "__text_alert_list" }}{{ range . }}
告警主题: {{ .Annotations.summary }}

告警详情: {{ .Annotations.description }}

告警实例: {{ .Labels.instance }}

Pod 名称: {{ .Labels.pod }}

告警级别: {{ .Labels.severity }}

告警状态: {{ .Status }}

告警时间: {{ (.StartsAt.Add 28800e9).Format "2006-01-02 15:04:05" }}

------------------------
{{ end }}{{ end }}

{{ define "__text_resolve_list" }}{{ range .  }}
告警主题: {{ .Annotations.summary }}

告警详情: {{ .Annotations.description }}

告警实例: {{ .Labels.instance }}

Pod 名称: {{ .Labels.pod }}

告警级别: {{ .Labels.severity }}

告警状态: {{ .Status }}

告警时间: {{ (.StartsAti.Add 28800e9).Format "2006-01-02 15:04:05" }}

恢复时间: {{ (.EndsAti.Add 28800e9).Format "2006-01-02 15:04:05" }}

------------------------
{{ end }}{{ end }}

{{ .GeneratorURL }} ：  获取 prometheus external-url，增加 prometheus 启动参数如 --web.external-url=https://prometheus.xxx.com/eks
{{ define "__alertmanagerURL" }}: 函数获取 alertmanager external-url，增加 alertmanager 启动参数如 --web.external-url=https://alertmanager.xxx.com

cat /etc/systemd/system/prometheus-webhook-dingtalk.service
[Unit]
Description=prometheus-webhook-dingtalk
After=network-online.target

[Service]
Restart=on-failure
ExecStart=/data/devops/prometheus-webhook-dingtalk/prometheus-webhook-dingtalk --config.file=/data/devops/prometheus-webhook-dingtalk/config.yml

[Install]
WantedBy=multi-user.target

---
kind: ConfigMap
apiVersion: v1
metadata:
  name: prometheus-webhook-dingtalk-config
  namespace: monitoring
data:
  config.yml: |-
    templates:
      - /config/template.tmpl
    targets:
      default-webhook-warning:
        # 默认告警群
        url: https://oapi.dingtalk.com/robot/send?access_token=xx
        # secret for signature
        secret: SEC4bfd28a4ea2d14836a3db5b2e8101a81d7c72c842b8c05a5b82cb48c5de3c096
  template.tmpl: |-
    {{ define "__subject" }}[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .GroupLabels.SortedPairs.Values | join " " }} {{ if gt (len .CommonLabels) (len .GroupLabels) }}({{ with .CommonLabels.Remove .GroupLabels.Names }}{{ .Values | join " " }}{{ end }}){{ end }}{{ end }}
    {{ define "__alertmanagerURL" }}{{ .ExternalURL }}/#/alerts?receiver={{ .Receiver }}{{ end }}

    {{ define "default.__text_alert_list" }}{{ range . }}
    #### \[{{ .Labels.severity | upper }}\] {{ .Annotations.summary }}

    **Description:** {{ .Annotations.description }}

    **Graph:** [📈]({{ .GeneratorURL }})

    **Details:**
    {{ range .Labels.SortedPairs }}{{ if and (ne (.Name) "severity") (ne (.Name) "summary") }}> - {{ .Name }}: {{ .Value | markdown | html }}
    {{ end }}{{ end }}
    {{ end }}{{ end }}

    {{/* Default */}}
    {{ define "default.title" }}{{ template "__subject" . }}{{ end }}
    {{ define "default.content" }}#### \[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}\] **[{{ index .GroupLabels "alertname" }}]({{ template "__alertmanagerURL" . }})**
    {{ if gt (len .Alerts.Firing) 0 -}}
    **Alerts Firing**
    {{ template "default.__text_alert_list" .Alerts.Firing }}
    {{- end }}
    {{ if gt (len .Alerts.Resolved) 0 -}}
    **Alerts Resolved**
    {{ template "default.__text_alert_list" .Alerts.Resolved }}
    {{- end }}
    {{- end }}

    {{/* Customizable templates by jasper */}}
    {{ define "__prometheusURL" }}https://pfgc-prometheus-prod.xxx.com/eks/alerts{{ end }}
    {{ define "__text_alert_list" }}{{ range . }}
    Summary: {{ .Annotations.summary }}

    Description: {{ .Annotations.description }}

    Graph: [📈]({{ .GeneratorURL }})

    Details:
    {{ range .Labels.SortedPairs }}{{ if and (ne (.Name) "severity") (ne (.Name) "summary") }}> - {{ .Name }}: {{ .Value | markdown | html }}
    {{ end }}{{ end }}

    Severity: {{ .Labels.severity }}

    Status: {{ .Status }}

    StartAt: {{ (.StartsAt.Add 28800e9).Format "2006-01-02 15:04:05" }}

    Solution: {{ .Annotations.runbook_url }}

    ------------------------
    {{ end }}{{ end }}

    {{ define "__text_alertresovle_list" }}{{ range .  }}
    Summary: {{ .Annotations.summary }}

    Description: {{ .Annotations.description }}

    Graph: [📈]({{ .GeneratorURL }})

    StartsAt: {{ (.StartsAt.Add 28800e9).Format "2006-01-02 15:04:05" }}

    EndsAt: {{ (.EndsAt.Add 28800e9).Format "2006-01-02 15:04:05" }}

    Solution: {{ .Annotations.runbook_url }}

    ------------------------
    {{ end }}{{ end }}

    {{/* Legacy */}}
    {{ define "legacy.title" }}{{ template "__subject" . }}{{ end }}
    {{ define "legacy.content" }}#### \[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}\] **[{{ index .CommonLabels "alertname" }}]({{ template "__alertmanagerURL" . }})**
    {{ if gt (len .Alerts.Firing) 0 -}}
    {{ template "__text_alert_list" .Alerts.Firing }}
    {{- end }}
    {{ if gt (len .Alerts.Resolved) 0 -}}
    {{ template "__text_alertresovle_list" .Alerts.Resolved }}
    {{- end }}
    {{- end }}

    {{/* Following names for compatibility */}}
    {{ define "ding.link.title" }}{{ template "legacy.title" . }}{{ end }}
    {{ define "ding.link.content" }}{{ template "legacy.content" . }}{{ end }}

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: dingtalk-nginx-ldap
  namespace: monitoring
data:
  app.conf: |-
    upstream dingtalk {
        server prometheus-webhook-dingtalk:8060 weight=1;
    }
    server {
        listen       80;
        server_name  localhost;

        location ~ \.(gif|jpg|png|css|js|svg|woff2|html|json|query)$ {
            proxy_pass http://dingtalk;
        }

        location / {
            proxy_pass http://dingtalk/;
            auth_ldap    "Forbidden";
            auth_ldap_servers   localdir;
        }
        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   /usr/share/nginx/html;
        }
    }
  nginx.conf: |-
    user  nginx;
    worker_processes  auto;

    error_log  /var/log/nginx/error.log warn;
    pid        /var/run/nginx.pid;

    events {
        worker_connections  1024;
    }

    http {
        include       /etc/nginx/mime.types;
        default_type  application/octet-stream;

        log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                          '$status $body_bytes_sent "$http_referer" '
                          '"$http_user_agent" "$http_x_forwarded_for"';

        access_log  /var/log/nginx/access.log  main;

        sendfile        on;

        keepalive_timeout  65;

        proxy_ignore_client_abort on;


        ldap_server localdir {
          url ldap://ops-freeipa.xxx.com:389/cn=users,cn=accounts,dc=xxx,dc=com?uid?sub?(&(objectClass=*));
          binddn "uid=admin,cn=users,cn=accounts,dc=xxx,dc=com";
          binddn_passwd "rdIfXD6ZoZ7aBNvY";
          #group_attribute memberuid;
          #group_attribute_is_dn on;
          #require valid_user;
          #max_down_retries 3;
        }

        include /etc/nginx/conf.d/*.conf;
    }
    #include /etc/nginx/tcp.conf;

apiVersion: v1
kind: Service
metadata:
  name: prometheus-webhook-dingtalk
  namespace: monitoring
  labels:
    app: prometheus-webhook-dingtalk
spec:
  ports:
  - name: http
    protocol: TCP
    port: 8060
    targetPort: 8060
  - name: http-80
    protocol: TCP
    port: 80
    targetPort: 80
  selector:
    app: prometheus-webhook-dingtalk

apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: prometheus-webhook-dingtalk
  name: prometheus-webhook-dingtalk
  namespace: monitoring
spec:
  replicas: 1
  selector:
    matchLabels:
      app: prometheus-webhook-dingtalk
  template:
    metadata:
      labels:
        app: prometheus-webhook-dingtalk
    spec:
      nodeSelector:
        type: pfgc-ops-prod-prometheus
      containers:
      - args:
        - --web.listen-address=:8060
        - --config.file=/config/config.yml
        - --web.enable-ui
        - --web.enable-lifecycle
        image: timonwong/prometheus-webhook-dingtalk:v2.1.0
        imagePullPolicy: IfNotPresent
        name: prometheus-webhook-dingtalk
        ports:
        - containerPort: 8060
          name: http
          protocol: TCP
        resources:
          limits:
            cpu: 1
            memory: 512Mi
          requests:
            cpu: 10m
            memory: 50Mi
        volumeMounts:
        - mountPath: /config
          name: conf
      - args:
        - --webhook-url=http://localhost:8060/-/reload
        - --volume-dir=/config/
        image: jimmidyson/configmap-reload:v0.5.0
        name: module-configmap-reloader
        resources:
          limits:
            cpu: 20m
            memory: 40Mi
          requests:
            cpu: 10m
            memory: 20Mi
        securityContext:
          allowPrivilegeEscalation: false
          capabilities:
            drop:
            - ALL
          readOnlyRootFilesystem: true
          runAsNonRoot: true
          runAsUser: 65534
        volumeMounts:
        - mountPath: /config/
          name: conf
          readOnly: true
      - image: ops-harbor.xxx.com/ops/nginx-ldap
        imagePullPolicy: Always
        name: nginx-ldap
        resources:
          limits:
            cpu: 200m
            memory: 200Mi
          requests:
            cpu: 10m
            memory: 10Mi
        stdin: true
        tty: true
        volumeMounts:
        - mountPath: /etc/nginx/nginx.conf
          name: nginx-conf
          subPath: nginx.conf
        - mountPath: /etc/nginx/conf.d/
          name: nginx-app-conf
      imagePullSecrets:
      - name: docker-secret
      volumes:
      - configMap:
          defaultMode: 420
          name: prometheus-webhook-dingtalk-config
        name: conf
      - configMap:
          defaultMode: 420
          items:
          - key: nginx.conf
            path: nginx.conf
          name: dingtalk-nginx-ldap
        name: nginx-conf
      - configMap:
          defaultMode: 420
          items:
          - key: app.conf
            path: app.conf
          name: dingtalk-nginx-ldap
        name: nginx-app-conf

#------------
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: kafka-exporter
  name: kafka-exporter
  namespace: monitoring
spec:
  replicas: 1
  selector:
    matchLabels:
      app: kafka-exporter
  strategy:
    rollingUpdate:
      maxSurge: 1
      maxUnavailable: 0
    type: RollingUpdate
  template:
    metadata:
      labels:
        app: kafka-exporter
    spec:
      containers:
      - args:
        - --kafka.server=kafka-0.kafka-headless.public-service:9092
        env:
        - name: TZ
          value: Asia/Shanghai
        - name: LANG
          value: C.UTF-8
        image: danielqsj/kafka-exporter:latest
        imagePullPolicy: IfNotPresent
        name: kafka-exporter
        ports:
        - containerPort: 9308
          name: web
          protocol: TCP
        resources:
          limits:
            cpu: 249m
            memory: 318Mi
          requests:
            cpu: 10m
            memory: 10Mi
        securityContext:
          allowPrivilegeEscalation: false
          privileged: false
          readOnlyRootFilesystem: false
          runAsNonRoot: false
        volumeMounts:
        - mountPath: /usr/share/zoneinfo/Asia/Shanghai
          name: tz-config
        - mountPath: /etc/localtime
          name: tz-config
        - mountPath: /etc/timezone
          name: timezone
      volumes:
      - hostPath:
          path: /usr/share/zoneinfo/Asia/Shanghai
          type: ""
        name: tz-config
      - hostPath:
          path: /etc/timezone
          type: ""
        name: timezone


---
apiVersion: v1
kind: Service
metadata:
  labels:
    app: kafka-exporter
  name: kafka-exporter
  namespace: monitoring
spec:
  ports:
  - name: container-1-web-1
    port: 9308
    protocol: TCP
    targetPort: 9308
  selector:
    app: kafka-exporter
  type: ClusterIP


---

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  labels:
    k8s-app: kafka-exporter
  name: kafka-exporter
  namespace: monitoring
spec:
  endpoints:
  - interval: 30s
    port: container-1-web-1
  namespaceSelector:
    matchNames:
    - monitoring
  selector:
    matchLabels:
      app: kafka-exporter

cat cas-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: kafka-exporter-cas
  namespace: monitoring
spec:
  replicas: 1
  selector:
    matchLabels:
      app: kafka-exporter-cas
  template:
    metadata:
      annotations:
        prometheus.io/path: /metrics
        prometheus.io/port: "9308"
        prometheus.io/scrape: "true"
      labels:
        app: kafka-exporter-cas
        instance: CAS
    spec:
      containers:
      - command:
        - /bin/sh
        - -c
        - kafka_exporter --kafka.server=172.21.38.219:9092 --kafka.server=172.21.38.233:9092
        image: danielqsj/kafka-exporter:latest
        imagePullPolicy: IfNotPresent
        name: kafka-exporter-cas
        env:
        - name: TZ
          value: Asia/Shanghai
        - name: LANG
          value: C.UTF-8
        ports:
        - containerPort: 9308
          protocol: TCP
        readinessProbe:
          failureThreshold: 3
          initialDelaySeconds: 5
          periodSeconds: 10
          successThreshold: 1
          tcpSocket:
            port: 9308
          timeoutSeconds: 5
        resources:
          limits:
            cpu: 500m
            memory: 1Gi
          requests:
            cpu: 200m
            memory: 10Mi
      imagePullSecrets:
      - name: docker-pull
      nodeSelector:
        eks.amazonaws.com/nodegroup: PFGC-EKS-DEVOPS-Prometheus
      restartPolicy: Always

$ ll
node-exporter-clusterRoleBinding.yaml
node-exporter-clusterRole.yaml
node-exporter-daemonset.yaml
node-exporter-serviceAccount.yaml

cat redis-exporter.yaml
apiVersion: v1
kind: Service
metadata:
  name: redis-exporter
  namespace: monitoring
  #annotations:
  #  prometheus.io/scrape: "true"
  labels:
    app: svc-redis-exporter
spec:
  type: NodePort
  selector:
    app: redis-exporter
  ports:
    - name: tcp-9121
      protocol: TCP
      port: 9121
      targetPort: 9121
---
apiVersion: apps/v1
kind: Deployment
metadata:
  namespace: monitoring
  name: redis-exporter
spec:
  replicas: 1
  selector:
    matchLabels:
      app: redis-exporter
  template:
    metadata:
      #annotations:
      #  prometheus.io/scrape: "true"
      #  prometheus.io/port: "9121"
      labels:
        app: redis-exporter
    spec:
      nodeSelector:
        eks.amazonaws.com/nodegroup: PFGC-EKS-DEVOPS-Prometheus
      containers:
      - name: redis-exporter
        image: oliver006/redis_exporter:latest
        resources:
          limits:
            cpu: 300m
            memory: 300Mi
          requests:
            cpu: 10m
            memory: 10Mi
        ports:
        - containerPort: 9121

- job_name: 'redis_exporter_targets'
  static_configs:
  - targets:
    - agt-.amazonaws.com:6379
    - redis://agt-necache.amazonaws.com:6379
    - redis://172.21.89.135:6391
    - redis://172.21.89.135:6392
    labels:
      dbtype: db-mysql-db
  metrics_path: /scrape
  relabel_configs:
  - source_labels: [__address__]
    target_label: __param_target
  - source_labels: [__param_target]
    target_label: instance
  - target_label: __address__
    replacement: redis-exporter:9121

cat gpl-rocketmq-exporter-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: gpl-rocketmq-exporter
  namespace: monitoring
spec:
  replicas: 1
  selector:
    matchLabels:
      app: gpl-rocketmq-exporter
  template:
    metadata:
      annotations:
        prometheus.io/path: "/metrics"
        prometheus.io/port: "5557"
        prometheus.io/scrape: "true"
      labels:
        app: gpl-rocketmq-exporter
    spec:
      nodeSelector:
        type: pfgc-ops-prod-prometheus
      imagePullSecrets:
      - name: docker-secret
      containers:
      - name: rocketmq-exporter
        image: harbor.xxx.com/devops/rocketmq-exporter:v0.0.2
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 5557
        command:
        - /bin/sh
        - -c
        - 'java -jar -Xms2g -Xmx2g -Xmn1024m -Xss256k /opt/rocketmq-exporter/rocketmq-exporter-0.0.2-SNAPSHOT.jar
            --server.port=5557
            --rocketmq.config.namesrvAddr=172.21.89.192:9876'
        readinessProbe:
          tcpSocket:
            port: 5557
          initialDelaySeconds: 5
          timeoutSeconds: 5
        resources:
          limits:
            cpu: 1
            memory: 1Gi
          requests:
            cpu: 10m
            memory: 10Mi

cat wallet-rocketmq-exporter-deployment.yaml
apiVersion: apps/v1
kind: Deployment
        command:
        - /bin/sh
        - -c
        - 'java -jar -Xms2g -Xmx2g -Xmn1024m -Xss256k /opt/rocketmq-exporter/rocketmq-exporter-0.0.2-SNAPSHOT.jar
            --server.port=5557
            --rocketmq.config.namesrvAddr=172.21.89.85:9876
            --rocketmq.config.enableACL=true
            --rocketmq.config.accessKey=RocketMQ
            --rocketmq.config.secretKey=12345678'

cat cloudwatch-exporter.yaml
---
# Source: prometheus-cloudwatch-exporter/templates/serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
automountServiceAccountToken: true
metadata:
  name: cloudwatch-exporter-prometheus-cloudwatch-exporter
  namespace: monitoring
  labels:
    app: prometheus-cloudwatch-exporter
    chart: prometheus-cloudwatch-exporter-0.19.2
    release: "cloudwatch-exporter"
    heritage: "Helm"
---
# Source: prometheus-cloudwatch-exporter/templates/secrets.yaml
apiVersion: v1
kind: Secret
metadata:
  name: cloudwatch-exporter-prometheus-cloudwatch-exporter
  namespace: monitoring
  labels:
    app: prometheus-cloudwatch-exporter
    chart: prometheus-cloudwatch-exporter-0.19.2
    heritage: Helm
    release: cloudwatch-exporter
type: Opaque
data:

  aws_access_key_id: "QsUtJQs="


  aws_secret_access_key: "dlJS9Beg=="
---
# Source: prometheus-cloudwatch-exporter/templates/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: cloudwatch-exporter-prometheus-cloudwatch-exporter
  namespace: monitoring
  labels:
    app: prometheus-cloudwatch-exporter
    chart: prometheus-cloudwatch-exporter-0.19.2
    release: cloudwatch-exporter
    heritage: Helm
data:
  config.yml: |-

    # This is the default configuration for prometheus-cloudwatch-exporter
    region: ap-south-1
    period_seconds: 30
    delay_seconds: 60
    metrics:
    #- aws_namespace: AWS/ELB
    #  aws_metric_name: HealthyHostCount
    #  aws_dimensions: [AvailabilityZone, LoadBalancerName]
    #  aws_statistics: [Average]
    #
    #- aws_namespace: AWS/ELB
    #  aws_metric_name: UnHealthyHostCount
    #  aws_dimensions: [AvailabilityZone, LoadBalancerName]
    #  aws_statistics: [Average]
    #
    #- aws_namespace: AWS/ELB
    #  aws_metric_name: RequestCount
    #  aws_dimensions: [AvailabilityZone, LoadBalancerName]
    #  aws_statistics: [Sum]
    #
    #- aws_namespace: AWS/ELB
    #  aws_metric_name: Latency
    #  aws_dimensions: [AvailabilityZone, LoadBalancerName]
    #  aws_statistics: [Average]
    #
    #- aws_namespace: AWS/ELB
    #  aws_metric_name: SurgeQueueLength
    #  aws_dimensions: [AvailabilityZone, LoadBalancerName]
    #  aws_statistics: [Maximum, Sum]

    # 添加
    - aws_namespace: AWS/ElastiCache
      aws_metric_name: CPUUtilization
      aws_dimensions: [CacheClusterId]
      aws_tag_select:
        tag_selections:
          Owner: ["Klaus.ma"]
        resource_type_selection: elasticache
        resource_id_dimension: Tags
      aws_statistics: [Minimum,Maximum,Average]

    - aws_namespace: AWS/ElastiCache
      aws_metric_name: EngineCPUUtilization
      aws_dimensions: [CacheClusterId]
      aws_dimension_select_regex:
         #LoadBalancerName: ["app/AGT-APP-Prod-Game-ALB02/b6eb4661b489a15c", "app/AGT-APP-Prod-Game-ALB/9f62266ea9b31940"]
        cache_cluster_id: ["pfg.*"]
      aws_statistics: [Minimum,Maximum,Average]

    - aws_namespace: AWS/ElastiCache
      aws_metric_name: BytesUsedForCache
      aws_dimensions: [CacheClusterId]
      aws_statistics: [Minimum,Maximum,Average]

    - aws_namespace: AWS/ElastiCache
      aws_metric_name: CurrConnections
      aws_dimensions: [CacheClusterId]
      aws_statistics: [Minimum,Maximum,Average]

    - aws_namespace: AWS/ElastiCache
      aws_metric_name: NewConnections
      aws_dimensions: [CacheClusterId]
      aws_statistics: [Minimum,Maximum,Average]

    - aws_namespace: AWS/ElastiCache
      aws_metric_name: DatabaseMemoryUsagePercentage
      aws_dimensions: [CacheClusterId]
      aws_statistics: [Minimum,Maximum,Average]

    - aws_namespace: AWS/ElastiCache
      aws_metric_name: NetworkBytesIn
      aws_dimensions: [CacheClusterId]
      aws_statistics: [Minimum,Maximum,Average]

    - aws_namespace: AWS/ElastiCache
      aws_metric_name: NetworkBytesOut
      aws_dimensions: [CacheClusterId]
      aws_statistics: [Minimum,Maximum,Average]

    - aws_namespace: AWS/ElastiCache
      aws_metric_name: FreeableMemory
      aws_dimensions: [CacheClusterId]
      aws_statistics: [Minimum,Maximum,Average]

    - aws_namespace: AWS/ElastiCache
      aws_metric_name: CacheHits
      aws_dimensions: [CacheClusterId]
      aws_statistics: [Minimum,Maximum,Average]

    - aws_namespace: AWS/ElastiCache
      aws_metric_name: CacheMisses
      aws_dimensions: [CacheClusterId]
      aws_statistics: [Minimum,Maximum,Average]

    - aws_namespace: AWS/ElastiCache
      aws_metric_name: ReplicationLag
      aws_dimensions: [CacheClusterId]
      aws_statistics: [Minimum,Maximum,Average]

    - aws_namespace: AWS/ElastiCache
      aws_metric_name: MemoryFragmentationRatio
      aws_dimensions: [CacheClusterId]
      aws_statistics: [Minimum,Maximum,Average]

    - aws_namespace: AWS/ElastiCache
      aws_metric_name: IsMaster
      aws_dimensions: [CacheClusterId]
      aws_statistics: [Minimum,Maximum,Average]

    - aws_namespace: AWS/ElastiCache
      aws_metric_name: CacheHitRate
      aws_dimensions: [CacheClusterId]
      aws_statistics: [Minimum,Maximum,Average]
    ##新增
    - aws_namespace: AWS/ElastiCache
      aws_metric_name: StringBasedCmdsLatency
      aws_dimensions: [CacheClusterId]
      aws_statistics: [Minimum,Maximum,Average]

    - aws_namespace: AWS/ElastiCache
      aws_metric_name: SortedSetBasedCmdsLatency
      aws_dimensions: [CacheClusterId]
      aws_statistics: [Minimum,Maximum,Average]

    - aws_namespace: AWS/ElastiCache
      aws_metric_name: HashBasedCmdsLatency
      aws_dimensions: [CacheClusterId]
      aws_statistics: [Minimum,Maximum,Average]

    - aws_namespace: AWS/ElastiCache
      aws_metric_name: SetBasedCmdsLatency
      aws_dimensions: [CacheClusterId]
      aws_statistics: [Minimum,Maximum,Average]

    - aws_namespace: AWS/ElastiCache
      aws_metric_name: GetTypeCmdsLatency
      aws_dimensions: [CacheClusterId]
      aws_statistics: [Minimum,Maximum,Average]

    - aws_namespace: AWS/ElastiCache
      aws_metric_name: KeyBasedCmdsLatency
      aws_dimensions: [CacheClusterId]
      aws_statistics: [Minimum,Maximum,Average]

    - aws_namespace: AWS/ElastiCache
      aws_metric_name: ListBasedCmdsLatency
      aws_dimensions: [CacheClusterId]
      aws_statistics: [Minimum,Maximum,Average]

    - aws_namespace: AWS/ElastiCache
      aws_metric_name: SetTypeCmdsLatency
      aws_dimensions: [CacheClusterId]
      aws_statistics: [Minimum,Maximum,Average]

    - aws_namespace: AWS/ElastiCache
      aws_metric_name: StringBasedCmds
      aws_dimensions: [CacheClusterId]
      aws_statistics: [Minimum,Maximum,Average]

    - aws_namespace: AWS/ElastiCache
      aws_metric_name: SetBasedCmds
      aws_dimensions: [CacheClusterId]
      aws_statistics: [Minimum,Maximum,Average]

    - aws_namespace: AWS/ElastiCache
      aws_metric_name: SetTypeCmds
      aws_dimensions: [CacheClusterId]
      aws_statistics: [Minimum,Maximum,Average]

    - aws_namespace: AWS/ElastiCache
      aws_metric_name: GetTypeCmds
      aws_dimensions: [CacheClusterId]
      aws_statistics: [Minimum,Maximum,Average]

    - aws_namespace: AWS/ElastiCache
      aws_metric_name: ListBasedCmds
      aws_dimensions: [CacheClusterId]
      aws_statistics: [Minimum,Maximum,Average]

    - aws_namespace: AWS/ElastiCache
      aws_metric_name: SortedSetBasedCmds
      aws_dimensions: [CacheClusterId]
      aws_statistics: [Minimum,Maximum,Average]

    - aws_namespace: AWS/ElastiCache
      aws_metric_name: EvalBasedCmds
      aws_dimensions: [CacheClusterId]
      aws_statistics: [Minimum,Maximum,Average]

    - aws_namespace: AWS/ElastiCache
      aws_metric_name: HashBasedCmds
      aws_dimensions: [CacheClusterId]
      aws_statistics: [Minimum,Maximum,Average]

    - aws_namespace: AWS/ElastiCache
      aws_metric_name: KeyBasedCmds
      aws_dimensions: [CacheClusterId]
      aws_statistics: [Minimum,Maximum,Average]


    - aws_namespace: AWS/ElastiCache
      aws_metric_name: CurrItems
      aws_dimensions: [CacheClusterId]
      aws_statistics: [Minimum,Maximum,Average]


    - aws_namespace: AWS/ApplicationELB
      aws_metric_name: RequestCount
      aws_dimensions: [AvailabilityZone, LoadBalancer, TargetGroup]
      #aws_dimension_select:
      aws_dimension_select_regex:
         #LoadBalancerName: ["app/AGT-APP-Prod-Game-ALB02/b6eb4661b489a15c", "app/AGT-APP-Prod-Game-ALB/9f62266ea9b31940"]
         LoadBalancerName: ["agt*|pg*"]
      aws_statistics: [Sum]
---
# Source: prometheus-cloudwatch-exporter/templates/clusterrole.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: cloudwatch-exporter-prometheus-cloudwatch-exporter
  labels:
    app: prometheus-cloudwatch-exporter
    chart: prometheus-cloudwatch-exporter-0.19.2
    release: cloudwatch-exporter
    heritage: Helm
rules:
  - apiGroups: [""]
    resources: ["secrets","configmap"]
    resourceNames: ["cloudwatch-exporter-prometheus-cloudwatch-exporter"]
    verbs: ["get"]
---
# Source: prometheus-cloudwatch-exporter/templates/clusterrolebinding.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: cloudwatch-exporter-prometheus-cloudwatch-exporter
  labels:
    app: prometheus-cloudwatch-exporter
    chart: prometheus-cloudwatch-exporter-0.19.2
    release: cloudwatch-exporter
    heritage: Helm
subjects:
  - kind: ServiceAccount
    name: cloudwatch-exporter-prometheus-cloudwatch-exporter
    namespace: monitoring
roleRef:
  kind: ClusterRole
  name: cloudwatch-exporter-prometheus-cloudwatch-exporter
  apiGroup: rbac.authorization.k8s.io
---
# Source: prometheus-cloudwatch-exporter/templates/service.yaml
apiVersion: v1
kind: Service
metadata:
  name: cloudwatch-exporter-prometheus-cloudwatch-exporter
  namespace: monitoring
  annotations:
    {}
  labels:
    app: prometheus-cloudwatch-exporter
    chart: prometheus-cloudwatch-exporter-0.19.2
    release: cloudwatch-exporter
    heritage: Helm
spec:
  type: ClusterIP
  ports:
    - port: 9106
      targetPort: container-port
      protocol: TCP
      name: http
  selector:
    app: prometheus-cloudwatch-exporter
    release: cloudwatch-exporter
---
# Source: prometheus-cloudwatch-exporter/templates/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: cloudwatch-exporter-prometheus-cloudwatch-exporter
  namespace: monitoring
  labels:
    app: prometheus-cloudwatch-exporter
    chart: prometheus-cloudwatch-exporter-0.19.2
    release: cloudwatch-exporter
    heritage: Helm
spec:
  replicas: 1
  selector:
    matchLabels:
      app: prometheus-cloudwatch-exporter
      release: cloudwatch-exporter
  template:
    metadata:
      labels:
        app: prometheus-cloudwatch-exporter
        release: cloudwatch-exporter
      annotations:

        checksum/config: 98ede3b76a51e70fce11f9621056dc54497e4bb2a4b6f7237615798cd837965b
        checksum/secrets: 0b69318c1ff017e8f8fb567de33438a87cabb073edb1ec3a33247b8101683bb6
    spec:
      containers:
        - name: prometheus-cloudwatch-exporter
          env:
            - name: AWS_ACCESS_KEY_ID
              valueFrom:
                secretKeyRef:
                  key: aws_access_key_id
                  name: cloudwatch-exporter-prometheus-cloudwatch-exporter
            - name: AWS_SECRET_ACCESS_KEY
              valueFrom:
                secretKeyRef:
                  key: aws_secret_access_key
                  name: cloudwatch-exporter-prometheus-cloudwatch-exporter
          image: "prom/cloudwatch-exporter:v0.14.3"
          imagePullPolicy: IfNotPresent
          ports:
            - name: container-port
              containerPort: 9106
              protocol: TCP
          livenessProbe:
            httpGet:
              path: /-/healthy
              port: container-port
            initialDelaySeconds: 30
            periodSeconds: 5
            timeoutSeconds: 5
            successThreshold: 1
            failureThreshold: 3
          readinessProbe:
            httpGet:
              path: /-/ready
              port: container-port
            initialDelaySeconds: 30
            periodSeconds: 5
            timeoutSeconds: 5
            successThreshold: 1
            failureThreshold: 3
          resources:
            {}
          volumeMounts:
            - name: vol-prometheus-cloudwatch-exporter
              mountPath: /config
      nodeSelector:
        eks.amazonaws.com/nodegroup: PFGC-EKS-DEVOPS-Prometheus
      securityContext:
        fsGroup: 65534
        runAsUser: 65534
      serviceAccount: cloudwatch-exporter-prometheus-cloudwatch-exporter
      serviceAccountName: cloudwatch-exporter-prometheus-cloudwatch-exporter
      volumes:
      - configMap:
          defaultMode: 420
          name: cloudwatch-exporter-prometheus-cloudwatch-exporter
        name: vol-prometheus-cloudwatch-exporter

- job_name: 'aws_cloudwatch'
 metrics_path: /metrics
 scrape_interval: 50s
 scrape_timeout: 45s
 static_configs:
 - targets:
   - "cloudwatch-exporter-prometheus-cloudwatch-exporter:9106"