Kubernetes: k8s 进阶篇-高级调度

apiVersion: batch/v1
kind: CronJob
metadata:
  name: hello
  labels:
    run: hell
  namespace: default
spec:
  concurrencyPolicy: Allow
  failedJobsHistoryLimit: 1
  jobTemplate:
    spec:
      ttlSecondsAfterFinished: 64800
      template:
        metadata:
          labels:
            run: hello
        spec:
          containers:
          - name: hello
            image: busybox:1.28
            imagePullPolicy: IfNotPresent
            args:
            - /bin/sh
            - -c
            - date; echo Hello from the Kubernetes cluster
            resources: {}
          restartPolicy: OnFailure
  schedule: "*/1 * * * *"
  successfulJobsHistoryLimit: 3
  suspend: false

apiVersion: batch/v1
kind: CronJob
metadata:
  name: ludo-xxl-job-clean
  namespace: ludo
spec:
  jobTemplate:
    spec:
      ttlSecondsAfterFinished: 64800
      template:
        spec:
          containers:
          - command:
            - /bin/sh
            - -c
            - echo 操作前总数为：;mysql -h$HOST -P 6033 -u$USER -p$PASSWORD ludo_xxl_job -e 'select count(*)
              from xxl_job_log';mysql -h$HOST -P 6033 -u$USER -p$PASSWORD ludo_xxl_job
              -e 'delete from xxl_job_log';  echo 操作后总数为：;mysql -h$HOST -P 6033 -u$USER -p$PASSWORD
              ludo_xxl_job -e 'select count(*) from xxl_job_log'
            env:
            - name: HOST
              value: nlbxxx.elb.ap-south-1.amazonaws.com
            - name: USER
              value: xxl_job
            - name: PASSWORD
              value: FfVFSj/fdUCRYZ8Y
            image: mysql:5.7
            imagePullPolicy: Always
            name: ludo-xxl-job-clean
          nodeSelector:
            type: pfgc-ludo-prod-app
          restartPolicy: OnFailure
  schedule: "58 1 * * *"

[root@k8s-master01 app]# kubectl get cj hello
NAME    SCHEDULE      SUSPEND   ACTIVE   LAST SCHEDULE   AGE
hello   */1 * * * *   False     6        4s              8m20s

# 删除
[root@k8s-master01 app]# kubectl delete cronjob hello
cronjob.batch "hello" deleted

# master 节点
vi /usr/lib/systemd/system/kube-apiserver.service
--feature-gates=EphemeralContainers=true

vi /usr/lib/systemd/system/kube-controller-manager.service
--feature-gates=EphemeralContainers=true

vi /usr/lib/systemd/system/kube-scheduler.service
--feature-gates=EphemeralContainers=true

# node 节点
vi /usr/lib/systemd/system/kube-proxy.service
--feature-gates=EphemeralContainers=true
vi /etc/kubernetes/kubelet-conf.yml
featureGates:
EphemeralContainers: true

# 重启所有服务
systemctl daemon-reload
systemctl restart kube-apiserver kube-scheduler kube-controller-mannager kubelet kube-proxy

# 判断是否生效方法：配置好了，能使用命令插件临时容器说明是可以的

{
    "apiVersion": "v1",
    "kind": "EphemeralContainers",
    "metadata": {
        "name": "example-pod"
    },
    "ephemeralContainers": [{
        "command": [
            "sh"
        ],
        "image": "busybox",
        "imagePullPolicy": "IfNotPresent",
        "name": "debugger",
        "stdin": true,
        "tty": true,
        "terminationMessagePolicy": "File"
    }]
}

kubectl replace --raw /api/v1/namespaces/default/pods/example-pod/ephemeralcontainers  -f ec.json

{
   "kind":"EphemeralContainers",
   "apiVersion":"v1",
   "metadata":{
      "name":"example-pod",
      "namespace":"default",
      "selfLink":"/api/v1/namespaces/default/pods/example-pod/ephemeralcontainers",
      "uid":"a14a6d9b-62f2-4119-9d8e-e2ed6bc3a47c",
      "resourceVersion":"15886",
      "creationTimestamp":"2019-08-29T06:41:42Z"
   },
   "ephemeralContainers":[
      {
         "name":"debugger",
         "image":"busybox",
         "command":[
            "sh"
         ],
         "resources":{

         },
         "terminationMessagePolicy":"File",
         "imagePullPolicy":"IfNotPresent",
         "stdin":true,
         "tty":true
      }
   ]
}

kubectl describe pod example-pod

...
Ephemeral Containers:
  debugger:
    Container ID:  docker://cf81908f149e7e9213d3c3644eda55c72efaff67652a2685c1146f0ce151e80f
    Image:         busybox
    Image ID:      docker-pullable://busybox@sha256:9f1003c480699be56815db0f8146ad2e22efea85129b5b5983d0e0fb52d9ab70
    Port:          <none>
    Host Port:     <none>
    Command:
      sh
    State:          Running
      Started:      Thu, 29 Aug 2019 06:42:21 +0000
    Ready:          False
    Restart Count:  0
    Environment:    <none>
    Mounts:         <none>
...

kubectl attach -it example-pod -c debugger

kubectl taint nodes NODE_NAME TAINT_KEY=TAINT_VALUE:EFFECT
比如：
kubectl taint nodes k8s-node01 ssd=true:PreferNoSchedule

kubectl taint nodes k8s-master02 node-role.kubernetes.io/master:NoSchedule-

]# kubectl describe node k8s-node01  (内容太多不贴全了)
Taints:             <none>   # 关注这个地方即可 ---没有设置过污点的节点属性中的参数是这样的Taints:     <none>

tolerations:
- key: "key"
  operator: "Equal"
  value: "value"
  effect: "NoSchedule"

tolerations:
- key: "key"
  operator: "Exists"
  effect: "NoSchedule"

tolerations:
- key: "key"
  operator: "Exists"

tolerations:
- operator: "Exists"

tolerations:
- key: "key1"
  operator: "Equal"
  value: "value1"
  effect: "NoExecute"
  tolerationSeconds: 3600 # 在 3600 秒后 pod 被驱逐

apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: nginx
  name: nginx
spec:
  replicas: 2
  selector:
    matchLabels:
      app: nginx
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
      - image: nginx:1.15.12
        name: nginx
      nodeSelector:
        ssh: "true"
      tolerations:
      - key: ssd
        operator: Equal
        value: "true"
      - effect: NoExecute
        key: node.kubernetes.io/unreachable
        operator: Exists
        tolerationSeconds: 10
      - effect: NoExecute
        key: node.kubernetes.io/not-ready
        operator: Exists
        tolerationSeconds: 10

# 检查节点是否有污点
kubectl  describe node |grep Taint
# kubectl taint node k8s-node01 ssd-  # 删除污点

# 节点添加标签，--overwrite覆盖已存在的标签信息
kubectl label nodes k8s-node01 type=app-prod --overwrite

apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: must-be-diff-nodes
  name: must-be-diff-nodes
  namespace: default
spec:
  replicas: 3
  selector:
    matchLabels:
      app: must-be-diff-nodes
  template:
    metadata:
      labels:
        app: must-be-diff-nodes
    spec:
      nodeSelector:
        type: app-prod
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchExpressions:
              - key: app
                operator: In
                values:
                - must-be-diff-nodes
            topologyKey: kubernetes.io/hostname
      containers:
      - image: nginx:1.15.12
        name: must-be-diff-nodes
        imagePullPolicy: IfNotPresent

apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: must-be-diff-nodes
  name: must-be-diff-nodes
  namespace: default
spec:
  replicas: 3
  selector:
    matchLabels:
      app: must-be-diff-nodes
      project: multi
  template:
    metadata:
      labels:
        app: must-be-diff-nodes
        project: multi
    spec:
      nodeSelector:
        type: app-prod
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchExpressions:
              - key: project
                operator: In
                values:
                - multi
            topologyKey: kubernetes.io/hostname
      containers:
      - image: nginx:1.15.12
        name: must-be-diff-nodes
        imagePullPolicy: IfNotPresent

apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: web-store
  name: web-store
  namespace: default
spec:
  replicas: 3
  selector:
    matchLabels:
      app: web-store
  template:
    metadata:
      labels:
        app: web-store
    spec:
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
            matchExpressions:
            - key: app
              operator: In
              values:
              - web-store
          # 拓扑域  若多个node节点具有相同的标签信息【标签键值相同】，则表示这些node节点就在同一拓扑域
          topologyKey: kubernetes.io/hostname
        podAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            podAffinityTerm:
              labelSelector:
                matchExpressions:
                - key: app
                  operator: In
                  values:
                  - store
              topologyKey: kubernetes.io/hostname
      containers:
      - image: nginx:1.15.12
        name: web-app

kubectl label nodes k8s-node01 ssd=true
kubectl label nodes k8s-master01 sshd=true 
kubectl label nodes k8s-master01 gpu=true
kubectl label nodes k8s-node02 type=physical

apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: prefer-ssd
  name: prefer-ssd
  namespace: default
spec:
  replicas: 3
  selector:
    matchLabels:
      app: prefer-ssd
  template:
    metadata:
      labels:
        app: prefer-ssd
    spec:
      nodeSelector:
        type: prefer-ssd
      affinity:
        nodeAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            preference:
              matchExpressions:
              - key: ssd
                operator: In
                values:
                - true
              - key: gpu
                operator: NotIn
                values:
                - true
          - weight: 10
            preference:
              matchExpressions:
              - key: type
                operator: In
                values:
                - physical
      containers:
      - image: nginx:1.15.12
        name: prefer-ssd
        imagePullPolicy: IfNotPresent

kubectl label node k8s-master01 k8s-master02 region=daxing
kubectl label node k8s-master03 k8s-node01 region=chaoyang
kubectl label node k8s-node02 region=xxx

apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: must-be-diff-zone
  name: must-be-diff-zone
  namespace: default
spec:
  replicas: 3
  selector:
    matchLabels:
      app: must-be-diff-zone
      project: multi
  template:
    metadata:
      labels:
        app: must-be-diff-zone
        project: multi
    spec:
      nodeSelector:
        type: app-prod
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
            matchExpressions:
            - key: app
              operator: In
              values:
              - must-be-diff-zone
          topologyKey: region
      containers:
      - image: nginx:1.15.12
        name: must-be-diff-zone
        imagePullPolicy: IfNotPresent

apiVersion: v1
kind: Pod
metadata:
  name: with-pod-affinity
spec:
  affinity:
    podAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
      - labelSelector:
          matchExpressions:
          - key: security
            operator: In
            values:
            - S1
        topologyKey: failure-domain.beta.kubernetes.io/zone
    podAntiAffinity:
      preferredDuringSchedulingIgnoredDuringExecution:
      - weight: 100
        podAffinityTerm:
          labelSelector:
            matchExpressions:
            - key: security
              operator: In
              values:
              - S2
          topologyKey: kubernetes.io/hostname
  containers:
  - name: with-pod-affinity
    image: k8s.gcr.io/pause:2.0

spec:
  topologySpreadConstraints:
  - maxSkew: 1
    topologyKey: eks.amazonaws.com/capacityType
    whenUnsatisfiable: ScheduleAnyway
    labelSelector:
      matchLabels:
        <pod label>
  affinity:
    nodeaffinity:
      requiredduringschedulingignoredduringexecution:
        nodeselectorterms:
        - matchexpressions:
          - key: zone
            operator: notin
            values:
            - zonec