Linux: k8s故障处理

$  crictl  ps  | grep  <pod名称>
拿一下容器ID

$ crictl  inspect  ID   | grep  pid
拿一下pid

$  nsenter --target  xxxx  -n
进入这个pods的网络空间

$ kubectl get pod reviews-v1-5bdc544bbd-5qgxj -o jsonpath="{.status.phase}"
Running

$ kubectl describe pod mypod
...
Containers:
  sh:
    Container ID:  docker://3f7a2ee0e7e0e16c22090a25f9b6e42b5c06ec049405bc34d3aa183060eb4906
    Image:         alpine
    Image ID:      docker-pullable://alpine@sha256:7b848083f93822dd21b0a2f14a110bd99f6efb4b838d499df6d04a49d0debf8b
    Port:          <none>
    Host Port:     <none>
    State:          Terminated
      Reason:       OOMKilled
      Exit Code:    2
    Last State:     Terminated
      Reason:       OOMKilled
      Exit Code:    2
    Ready:          False
    Restart Count:  3
    Limits:
      cpu:     1
      memory:  1G
    Requests:
      cpu:        100m
      memory:     500M
...

# kubectl get pod chinese-user-login-server-85dfc64db4-rnqfn -n chinese-old  -o yaml
... 
  containerStatuses:
  - containerID: docker://030bef4fd72f9f1a237e6a7dada0cf41b3fbcc5471b873a5035e05d8124c4bb5
    image: tope365-registry-vpc.cn-beijing.cr.aliyuncs.com/chinese-old-prod/chinese-user-login-server:3
    imageID: docker-pullable://tope365-registry-vpc.cn-beijing.cr.aliyuncs.com/chinese-old-prod/chinese-user-login-server@sha256:6630b71f9b554b2d2b1a8ffa141e849f10c18348df1d7288d114f7840493fa6c
    lastState: {}
    name: chinese-user-login-server
    ready: true
    restartCount: 0
    started: true
    state:
      running:
        startedAt: "2021-06-01T13:13:38Z"
  hostIP: 10.16.30.159
  initContainerStatuses:
  - containerID: docker://62ed6e20278d0e89c8e7d1477508babda0e6718a446b2d92b20ed8a89b9e15a1
    image: registry-vpc.cn-beijing.aliyuncs.com/arms-docker-repo/arms-pilot-init:v1.0.1
    imageID: docker-pullable://registry-vpc.cn-beijing.aliyuncs.com/arms-docker-repo/arms-pilot-init@sha256:a63c9a6d090e3ba7ec83c39c1aa92e8aca71181065a00f54087f3c6dcb0cb364
    lastState: {}
    name: arms-pilot-initcontainer
    ready: true
    restartCount: 0
    state:
      terminated:
        containerID: docker://62ed6e20278d0e89c8e7d1477508babda0e6718a446b2d92b20ed8a89b9e15a1
        exitCode: 0
        finishedAt: "2021-06-01T13:13:37Z"
        reason: Completed
        startedAt: "2021-06-01T13:13:31Z"
  phase: Running
...

# 命令过滤
rinfo2=`$kube_CMD -n $namespace get pods $rname  -o go-template="{{range .status.containerStatuses}}{{.lastState.terminated.finishedAt}}@{{.lastState.terminated.reason}}{{end}}@{{.status.podIP}}@{{.status.hostIP}}"`
r_last_time=`date -d "@$(date -d "$(echo $rinfo2|awk -F@ '{print $1}')" +%s)" +"%F %H:%M:%S"`
rinfo="$rname发生重启，$rinfo，最近$r_last_time，原因：$(echo $rinfo2|awk -F@ '{print $2}')，容器ip/节点ip:$(echo $rinfo2|awk -F@ '{print $3}')/$(echo $rinfo2|awk -F@ '{print $4}')\n"

type：              Pod 状况的名称
status：            表明该状况是否适用，可能的取值有 "True", "False" 或 "Unknown"
lastProbeTime：     上次探测 Pod 状况时的时间戳
lastTransitionTime：Pod 上次从一种状态转换到另一种状态时的时间戳
reason：            机器可读的、驼峰编码（UpperCamelCase）的文字，表述上次状况变化的原因
message：            人类可读的消息，给出上次状态转换的详细信息

# kubectl describe pod chinese-user-login-server-85dfc64db4-rnqfn -n chinese-old
... 
Conditions:
  Type              Status
  Initialized       True
  Ready             True
  ContainersReady   True
  PodScheduled      True
...

# kubectl get pod chinese-user-login-server-85dfc64db4-rnqfn -n chinese-old  -o yaml
...
status:
  conditions:
  - lastProbeTime: null
    lastTransitionTime: "2021-06-01T13:13:38Z"
    status: "True"
    type: Initialized
  - lastProbeTime: null
    lastTransitionTime: "2021-06-01T13:15:23Z"
    status: "True"
    type: Ready
  - lastProbeTime: null
    lastTransitionTime: "2021-06-01T13:15:23Z"
    status: "True"
    type: ContainersReady
  - lastProbeTime: null
    lastTransitionTime: "2021-06-01T13:13:30Z"
    status: "True"
    type: PodScheduled
...

# 查看Pod的事件
# 1.
kubectl describe pod mypod
...
Events:
  Type     Reason            Age                From               Message
  ----     ------            ----               ----               -------
  Warning  FailedScheduling  12s (x6 over 27s)  default-scheduler  0/4 nodes are available: 2 Insufficient cpu.

#2.
kubectl get events --field-selector involvedObject.name=nginx-85cb5867f-bs7pn,involvedObject.kind=Pod

type Event struct {
...
  // This should be a short, machine understandable string that gives the reason
  // for the transition into the object's current status.
  // TODO: provide exact specification for format.
  // +optional
  Reason string `json:"reason,omitempty" protobuf:"bytes,3,opt,name=reason"`

  // A human-readable description of the status of this operation.
  // TODO: decide on maximum length.
  // +optional
  Message string `json:"message,omitempty" protobuf:"bytes,4,opt,name=message"`

  // Type of this event (Normal, Warning), new types could be added in the future
  // +optional
  Type string `json:"type,omitempty" protobuf:"bytes,9,opt,name=type"`
...
}

// Container event reason list
const (
    CreatedContainer        = "Created"
    StartedContainer        = "Started"
    FailedToCreateContainer = "Failed"
    FailedToStartContainer  = "Failed"
    KillingContainer        = "Killing"
    PreemptContainer        = "Preempting"
    BackOffStartContainer   = "BackOff"
    ExceededGracePeriod     = "ExceededGracePeriod"
)

// Pod event reason list
const (
    FailedToKillPod                = "FailedKillPod"
    FailedToCreatePodContainer     = "FailedCreatePodContainer"
    FailedToMakePodDataDirectories = "Failed"
    NetworkNotReady                = "NetworkNotReady"
)

// Image event reason list
const (
    PullingImage            = "Pulling"
    PulledImage             = "Pulled"
    FailedToPullImage       = "Failed"
    FailedToInspectImage    = "InspectFailed"
    ErrImageNeverPullPolicy = "ErrImageNeverPull"
    BackOffPullImage        = "BackOff"
)

// kubelet event reason list
const (
    NodeReady                            = "NodeReady"
    NodeNotReady                         = "NodeNotReady"
    NodeSchedulable                      = "NodeSchedulable"
    NodeNotSchedulable                   = "NodeNotSchedulable"
    StartingKubelet                      = "Starting"
    KubeletSetupFailed                   = "KubeletSetupFailed"
    FailedAttachVolume                   = "FailedAttachVolume"
    FailedMountVolume                    = "FailedMount"
    VolumeResizeFailed                   = "VolumeResizeFailed"
    VolumeResizeSuccess                  = "VolumeResizeSuccessful"
    FileSystemResizeFailed               = "FileSystemResizeFailed"
    FileSystemResizeSuccess              = "FileSystemResizeSuccessful"
    FailedMapVolume                      = "FailedMapVolume"
    WarnAlreadyMountedVolume             = "AlreadyMountedVolume"
    SuccessfulAttachVolume               = "SuccessfulAttachVolume"
    SuccessfulMountVolume                = "SuccessfulMountVolume"
    NodeRebooted                         = "Rebooted"
    ContainerGCFailed                    = "ContainerGCFailed"
    ImageGCFailed                        = "ImageGCFailed"
    FailedNodeAllocatableEnforcement     = "FailedNodeAllocatableEnforcement"
    SuccessfulNodeAllocatableEnforcement = "NodeAllocatableEnforced"
    SandboxChanged                       = "SandboxChanged"
    FailedCreatePodSandBox               = "FailedCreatePodSandBox"
    FailedStatusPodSandBox               = "FailedPodSandBoxStatus"
    FailedMountOnFilesystemMismatch      = "FailedMountOnFilesystemMismatch"
)

// Image manager event reason list
const (
    InvalidDiskCapacity = "InvalidDiskCapacity"
    FreeDiskSpaceFailed = "FreeDiskSpaceFailed"
)

// Probe event reason list
const (
    ContainerUnhealthy    = "Unhealthy"
    ContainerProbeWarning = "ProbeWarning"
)

// Pod worker event reason list
const (
    FailedSync = "FailedSync"
)

// Config event reason list
const (
    FailedValidation = "FailedValidation"
)

[container runtime status check may not have completed yet, PLEG is not healthy: pleg was last seen active 3m28.93894374s ago; threshold is 3m0s]
 skipping pod synchronization - container runtime status check may not have completed yet

1. 保留当时现场诊断信息后   执行的操作是system restart docker 服务来恢复的
https://help.aliyun.com/knowledge_detail/178340.html   CentOS 7.6系统使用的systemd-219-62.el7_6.6.x86_64软件包存在缺陷，引发异常错误
2. yum update -y systemd && systemctl daemon-reexec

# pod设置临时空间
ephemeral-storage:  8Gi
# 资源紧张，Evicted 

如果运行时指定了别的独立的分区，比如修改了docker的镜像层和容器可写层的存储位置(默认是/var/lib/docker)所在的分区，将不再将其计入ephemeral-storage的消耗。
在容器写入超过存储限制时就会被驱逐掉了

[root@iZ2zegdxx2xkt57v45rtl2Z ~]# kubectl apply -f /tmp/deploy.yaml 
service/english-grammar-web created
The Deployment "english-grammar-web" is invalid: spec.selector: Invalid value: v1.LabelSelector{MatchLabels:map[string]string{"app":"english-grammar-web", "release":"qa"}, MatchExpressions:[]v1.LabelSelectorRequirement(nil)}: field is immutabl

# 我这里之前release是pre

pod peding报错：0/92 nodes are available: 1 node(s) didn't match pod affinity/anti-affinity, 1 node(s) didn't satisfy existing pods anti-affinity rules, 14 Insufficient memory, 29 Insufficient cpu, 56 node(s) didn't match node selector.

pod 状态：
NAME                                    READY   STATUS    RESTARTS   AGE
callbreak-room-1-665d446596-hfrhk       0/1     Running   0          20d
callbreak-room-1-6fdb565b8d-5dm2s       0/1     Pending   0          7m55s

dnsPolicy: ClusterFirstWithHostNet
hostNetwork: true

metadata:
  annotations:
    service.beta.kubernetes.io/backend-type: eni

apiVersion: apps/v1
kind: Deployment
metadata:
  annotations:
    component.revision: v5
    component.version: v0.22.0
    deployment.kubernetes.io/revision: '3'
    field.cattle.io/publicEndpoints: >-
  creationTimestamp: '2020-07-18T01:48:36Z'
  generation: 7
  labels:
    app: ingress-nginx
  name: nginx-ingress-controller
  namespace: kube-system
  resourceVersion: '793741697'
  selfLink: /apis/apps/v1/namespaces/kube-system/deployments/nginx-ingress-controller
  uid: b77c8097-2d3b-4c6b-b33a-656b7a62f5eb
spec:
  progressDeadlineSeconds: 600
  replicas: 2
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      app: ingress-nginx
  strategy:
    rollingUpdate:
      maxSurge: 25%
      maxUnavailable: 25%
    type: RollingUpdate
  template:
    metadata:
      annotations:
        cattle.io/timestamp: '2020-09-28T09:38:08Z'
        field.cattle.io/ports: >-
          [[{"containerPort":80,"dnsName":"nginx-ingress-controller","hostPort":0,"kind":"ClusterIP","name":"http","protocol":"TCP","sourcePort":0},{"containerPort":443,"dnsName":"nginx-ingress-controller","hostPort":0,"kind":"ClusterIP","name":"https","protocol":"TCP","sourcePort":0}]]
        prometheus.io/port: '10254'
        prometheus.io/scrape: 'true'
      labels:
        app: ingress-nginx
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  - key: type
                    operator: NotIn
                    values:
                      - virtual-kubelet
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
            - podAffinityTerm:
                labelSelector:
                  matchExpressions:
                    - key: app
                      operator: In
                      values:
                        - ingress-nginx
                topologyKey: kubernetes.io/hostname
              weight: 100
      containers:
        - args:
            - /nginx-ingress-controller
            - '--configmap=$(POD_NAMESPACE)/nginx-configuration'
            - '--tcp-services-configmap=$(POD_NAMESPACE)/tcp-services'
            - '--udp-services-configmap=$(POD_NAMESPACE)/udp-services'
            - '--annotations-prefix=nginx.ingress.kubernetes.io'
            - '--publish-service=$(POD_NAMESPACE)/nginx-ingress-lb'
            - '--v=2'
          env:
            - name: POD_NAME
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: metadata.name
            - name: POD_NAMESPACE
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: metadata.namespace
          image: >-
            registry-vpc.cn-beijing.aliyuncs.com/acs/aliyun-ingress-controller:v0.30.0.2-9597b3685-aliyun
          imagePullPolicy: IfNotPresent
          livenessProbe:
            failureThreshold: 3
            httpGet:
              path: /healthz
              port: 10254
              scheme: HTTP
            initialDelaySeconds: 10
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 10
          name: nginx-ingress-controller
          ports:
            - containerPort: 80
              hostPort: 80
              name: http
              protocol: TCP
            - containerPort: 443
              hostPort: 443
              name: https
              protocol: TCP
          readinessProbe:
            failureThreshold: 3
            httpGet:
              path: /healthz
              port: 10254
              scheme: HTTP
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 10
          resources:
            requests:
              cpu: 100m
              memory: 70Mi
          securityContext:
            capabilities:
              add:
                - NET_BIND_SERVICE
              drop:
                - ALL
            runAsUser: 101
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          volumeMounts:
            - mountPath: /etc/localtime
              name: localtime
              readOnly: true
      dnsPolicy: ClusterFirstWithHostNet
      hostNetwork: true
      initContainers:
        - command:
            - /bin/sh
            - '-c'
            - |-
              mount -o remount rw /proc/sys
              sysctl -w net.core.somaxconn=65535
              sysctl -w net.ipv4.ip_local_port_range="1024 65535"
              sysctl -w fs.file-max=1048576
              sysctl -w fs.inotify.max_user_instances=16384
              sysctl -w fs.inotify.max_user_watches=524288
              sysctl -w fs.inotify.max_queued_events=16384
          image: 'registry-vpc.cn-beijing.aliyuncs.com/acs/busybox:v1.29.2'
          imagePullPolicy: IfNotPresent
          name: init-sysctl
          resources: {}
          securityContext:
            capabilities:
              add:
                - SYS_ADMIN
              drop:
                - ALL
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
      nodeSelector:
        beta.kubernetes.io/os: linux
      priorityClassName: system-node-critical
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext: {}
      serviceAccount: nginx-ingress-controller
      serviceAccountName: nginx-ingress-controller
      terminationGracePeriodSeconds: 30
      volumes:
        - hostPath:
            path: /etc/localtime
            type: File
          name: localtime
status:
  availableReplicas: 2
  conditions:
    - lastTransitionTime: '2020-09-28T09:19:36Z'
      lastUpdateTime: '2020-09-28T09:19:36Z'
      message: Deployment has minimum availability.
      reason: MinimumReplicasAvailable
      status: 'True'
      type: Available
    - lastTransitionTime: '2020-07-18T01:48:36Z'
      lastUpdateTime: '2020-10-19T12:02:09Z'
      message: >-
        ReplicaSet "nginx-ingress-controller-68bd54b66c" has successfully
        progressed.
      reason: NewReplicaSetAvailable
      status: 'True'
      type: Progressing
  observedGeneration: 7
  readyReplicas: 2
  replicas: 2
  updatedReplicas: 2

apiVersion: v1
kind: Service
metadata:
  annotations:
    service.beta.kubernetes.io/alicloud-loadbalancer-address-type: internet
    service.beta.kubernetes.io/backend-type: eni
  creationTimestamp: '2020-07-18T01:48:36Z'
  labels:
    app: nginx-ingress-lb
  name: nginx-ingress-lb
  namespace: kube-system
  resourceVersion: '793911773'
  selfLink: /api/v1/namespaces/kube-system/services/nginx-ingress-lb
  uid: db3fec56-e8ed-40b9-9d5f-cb8b38dba808
spec:
  clusterIP: 172.24.8.22
  externalTrafficPolicy: Cluster
  ports:
    - name: http
      nodePort: 31820
      port: 80
      protocol: TCP
      targetPort: 80
    - name: https
      nodePort: 31114
      port: 443
      protocol: TCP
      targetPort: 443
  selector:
    app: ingress-nginx
  sessionAffinity: None
  type: LoadBalancer
status:
  loadBalancer:
    ingress:
      - ip: 39.105.75.81

# kubectl get ns
kubesphere‐system Terminating 28m

# 修改主机时间，删除crt文件
timedatectl set-ntp off
date -s "2020-10-16 09:03:00.000"
docker restart <rancher_server_id>
docker exec -ti <rancher_server_id> mv /var/lib/rancher/k3s/server/tls /var/lib/rancher/k3s/server/tlsbak
docker stop <rancher_server_id>

# 恢复时间 并让ran轮换证书
systemctl start  ntpd
docker exec -ti <rancher_server_id> mv /var/lib/rancher/k3s/server/tls /var/lib/rancher/k3s/server/tlsbak
# 执行2次重启，第一次用于申请证书，第二次用于加载证书并启动
docker restart <rancher_server_id>