Kubernetes: k8s-安装篇-kueadm-1.37

bash install_kubernetes_containerd.sh

root@ubuntu2604:~# bash install_kubernetes_containerd.sh
1) 初始化新的Kubernetes集群       3) 退出Kubernetes集群
2) 加入Kubernetes集群前的环境准备  4) 退出本程序
请选择编号(1-4): 2

#yum list kubeadm.x86_64 --showduplicates | sort -r
# 阿里源安装方法 https://developer.aliyun.com/mirror/kubernetes
KUBE_VERSION=1.37.1
KUBE_RELEASE=${KUBE_VERSION}-1.1
KUBE_MAJOR_VERSION=`echo ${KUBE_VERSION}| cut -d . -f 1,2`

apt-get update && apt-get install -y apt-transport-https

#curl -fsSL https://pkgs.k8s.io/core:/stable:/v${KUBE_MAJOR_VERSION}/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
#echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v${KUBE_MAJOR_VERSION}/deb/ /' | sudo tee /etc/apt/sources.list.d/kubernetes.list
curl -fsSL https://mirrors.aliyun.com/kubernetes-new/core/stable/v${KUBE_MAJOR_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://mirrors.aliyun.com/kubernetes-new/core/stable/v${KUBE_MAJOR_VERSION}/deb/ /" > /etc/apt/sources.list.d/kubernetes.list

apt-get update
apt-cache madison kubeadm |head # 查看有哪些版本

#安装指定版本
apt install -y  kubeadm=${KUBE_RELEASE} kubelet=${KUBE_RELEASE} kubectl=${KUBE_RELEASE}
#apt install -y  kubeadm kubelet kubectl
#实现kubectl命令自动补全功能
kubectl completion bash > /etc/profile.d/kubectl_completion.sh

kubeadm version

# 准备所需要的镜像(不用做)
KUBE_VERSION=1.37.1
kubeadm config images list --kubernetes-version=v${KUBE_VERSION}

# 可以提前把镜像下载好，这样安装快（可选步骤）
crictl pull registry.aliyuncs.com/google_containers/kube-apiserver:v${KUBE_VERSION}
crictl pull registry.aliyuncs.com/google_containers/kube-controller-manager:v${KUBE_VERSION}
crictl pull registry.aliyuncs.com/google_containers/kube-scheduler:v${KUBE_VERSION}
crictl pull registry.aliyuncs.com/google_containers/kube-proxy:v${KUBE_VERSION}
crictl pull registry.aliyuncs.com/google_containers/pause:3.10.2
crictl pull registry.aliyuncs.com/google_containers/etcd:3.7.0-0
crictl pull registry.aliyuncs.com/google_containers/coredns:v1.14.6

cp /etc/haproxy/haproxy.cfg{,.bak}

cat > /etc/haproxy/haproxy.cfg << EOF
global
  maxconn  2000
  ulimit-n  16384
  log  127.0.0.1 local0 err
  stats timeout 30s

defaults
  log global
  mode  http
  option  httplog
  timeout connect 5000
  timeout client  50000
  timeout server  50000
  timeout http-request 15s
  timeout http-keep-alive 15s

frontend k8s-master
  bind 0.0.0.0:8443
  bind 127.0.0.1:8443
  mode tcp
  option tcplog
  tcp-request inspect-delay 5s
  default_backend k8s-master

backend k8s-master
  mode tcp
  option tcplog
  option tcp-check
  balance roundrobin
  default-server inter 10s downinter 5s rise 2 fall 2 slowstart 60s maxconn 250 maxqueue 256 weight 100
  server master01.jasper.org    10.0.0.101:6443  check
  server master02.jasper.org    10.0.0.102:6443  check
  server master03.jasper.org    10.0.0.103:6443  check
EOF

cat > /etc/keepalived/keepalived.conf << EOF
! Configuration File for keepalived
global_defs {
    router_id LVS_DEVEL
}
vrrp_script chk_apiserver {
    script "/etc/keepalived/check_apiserver.sh"
    interval 5 
    weight -5
    fall 2
    rise 1
}
vrrp_instance VI_1 {
    state MASTER
    interface eth0
    mcast_src_ip 10.0.0.101
    virtual_router_id 61
    priority 101
    nopreempt
    advert_int 2
    authentication {
        auth_type PASS
        auth_pass K8SHA_KA_AUTH
    }
    virtual_ipaddress {
        10.0.0.236
    }
    track_script {
      chk_apiserver 
} }
EOF

cat > /etc/keepalived/keepalived.conf << EOF
! Configuration File for keepalived
global_defs {
    router_id LVS_DEVEL
}
vrrp_script chk_apiserver {
    script "/etc/keepalived/check_apiserver.sh"
    interval 5 
    weight -5
    fall 2
    rise 1
 
}
vrrp_instance VI_1 {
    state BACKUP
    interface eth0
    mcast_src_ip 10.0.0.102
    virtual_router_id 61
    priority 100
    nopreempt
    advert_int 2
    authentication {
        auth_type PASS
        auth_pass K8SHA_KA_AUTH
    }
    virtual_ipaddress {
        10.0.0.236
    }
    track_script {
      chk_apiserver 
} }
EOF

cat > /etc/keepalived/keepalived.conf << EOF
! Configuration File for keepalived
global_defs {
    router_id LVS_DEVEL
}
vrrp_script chk_apiserver {
    script "/etc/keepalived/check_apiserver.sh"
    interval 5
    weight -5
    fall 2  
    rise 1
}
vrrp_instance VI_1 {
    state BACKUP
    interface eth0
    mcast_src_ip 10.0.0.103
    virtual_router_id 61
    priority 100
    nopreempt
    advert_int 2
    authentication {
        auth_type PASS
        auth_pass K8SHA_KA_AUTH
    }
    virtual_ipaddress {
        10.0.0.236
    }
    track_script {
      chk_apiserver 
} }
EOF

cat > /etc/keepalived/check_apiserver.sh <<\EOF
#!/bin/bash

err=0
for k in $(seq 1 3)
do
    check_code=$(pgrep haproxy)
    if [[ $check_code == "" ]]; then
        err=$(expr $err + 1)
        sleep 1
        continue
    else
        err=0
        break
    fi
done

if [[ $err != "0" ]]; then
    echo "systemctl stop keepalived"
    /usr/bin/systemctl stop keepalived
    exit 1
else
    exit 0
fi
EOF

chmod +x /etc/keepalived/check_apiserver.sh

systemctl daemon-reload
systemctl enable --now haproxy
systemctl enable --now keepalived
systemctl restart haproxy keepalived

[[email protected] pki]# ping 10.0.0.236
PING 10.0.0.236 (10.0.0.236) 56(84) bytes of data.
64 bytes from 10.0.0.236: icmp_seq=1 ttl=64 time=1.39 ms
64 bytes from 10.0.0.236: icmp_seq=2 ttl=64 time=2.46 ms
64 bytes from 10.0.0.236: icmp_seq=3 ttl=64 time=1.68 ms
64 bytes from 10.0.0.236: icmp_seq=4 ttl=64 time=1.08 ms

nc -zv 10.0.0.236 8443
Connection to 10.0.0.236 8443 port [tcp/*] succeeded!

Available Commands:
    alpha #kubeadm 处于测试阶段的命令
    completion #bash 命令补全，需要安装 bash-completion
        #mkdir /data/scripts -p
        #kubeadm completion bash > /data/scripts/kubeadm_completion.sh
        #source /data/scripts/kubeadm_completion.sh
        #vim /etc/profile #添加内容
        source /data/scripts/kubeadm_completion.sh
    config #管理 kubeadm 集群的配置，该配置保留在集群的 ConfigMap 中
        #kubeadm config print init-defaults
    help Help about any command
    init    #启动一个 Kubernetes 主节点
    join    #将节点加入到已经存在的 k8s master
    reset   #还原使用 kubeadm init 或者 kubeadm join 对系统产生的环境变化。清空节点数据
    token   #管理 token
    upgrade #升级 k8s 版本
    version #查看版本信息

root@docker-k8s-node1:~# kubeadm init --help
--apiserver-advertise-address string    #K8S API Server 将要监听的监听的本机 IP
--apiserver-bind-port int32             #API Server 绑定的端口,默认为 6443

--apiserver-cert-extra-sans stringSlice #可选的证书额外信息，用于指定 API Server 的服务器证书。可以是 IP 地址也可以是 DNS 名称。
--cert-dir string                       #证书的存储路径，缺省路径为 /etc/kubernetes/pki
--certificate-key string                #定义一个用于加密 kubeadm-certs Secret 中的控制平台证书的密钥
--config string #kubeadm                #配置文件的路径
--control-plane-endpoint string         #为控制平台指定一个稳定的 IP 地址或 DNS 名称，即配置一个可以长期使用切是高可用的 VIP 或者域名，
                                        k8s多master 高可用基于此参数实现
--cri-socket string                     #要连接的 CRI(容器运行时接口，Container Runtime Interface, 简称 CRI)套接字的路径，如果为空，则 kubeadm 
                                        将尝试自动检测此值，"仅当安装了多个 CRI 或具有非标准 CRI 插槽时，才使用此选项"
--dry-run                               #不要应用任何更改，只是输出将要执行的操作，其实就是测试运行。
--experimental-kustomize string         #用于存储 kustomize 为静态 pod 清单所提供的补丁的路径。
--feature-gates string                  #一组用来描述各种功能特性的键值（key=value）对，选项是：IPv6DualStack=true|false (ALPHA - default=false)
--ignore-preflight-errors strings       #可以忽略检查过程 中出现的错误信息，比如忽略 swap，如果为 all 就忽略所有
--image-repository string               #设置一个镜像仓库，默认为 k8s.gcr.io
--kubernetes-version string             #指定安装 k8s 版本，默认为 stable-1
--node-name string                      #指定 node 节点名称
--pod-network-cidr                      #设置 pod ip 地址范围
--service-cidr                          #设置 service 网络地址范围，默认值："10.96.0.0/12"
--service-dns-domain string             #设置 k8s内部域名，默认为 cluster.local，会有相应的 DNS 服务(kube-dns/coredns)解析生成的域名记录。可以换成公司内部使用的域名，如me.local
--skip-certificate-key-print            #不打印用于加密的key 信息
--skip-phases strings                   #要跳过哪些阶段
--skip-token-print                      #跳过打印 token 信息
--token                                 #指定 token
--token-ttl                             #指定 token 过期时间，默认为 24 小时，0 为永不过期
--upload-certs                          #更新证书

#全局可选项：
--add-dir-header                        #如果为 true，在日志头部添加日志目录
--log-file string                       #如果不为空，将使用此日志文件
--log-file-max-size uint                #设置日志文件的最大大小，单位为兆，默认为 1800 兆，0 为没有限制
--rootfs                                #宿主机的根路径，也就是绝对路径
--skip-headers                          #如果为 true，在 log 日志里面不显示标题前缀
--skip-log-headers                      #如果为 true，在 log 日志里里不显示标题

kubeadm version #查看当前 kubeadm 版本

#查看安装指定版本 k8s 需要的镜像有哪些
KUBE_VERSION=1.37.1
kubeadm config images list --kubernetes-version=v${KUBE_VERSION}

k8s.gcr.io/kube-apiserver:
k8s.gcr.io/kube-controller-manager:
k8s.gcr.io/kube-scheduler:
k8s.gcr.io/kube-proxy:
k8s.gcr.io/pause: #为每个pod封装一个底层的网络
k8s.gcr.io/etcd:
k8s.gcr.io/coredns/coredns:

KUBE_VERSION=1.37.1
IMAGES_URL="registry.aliyuncs.com/google_containers"
POD_NETWORK="10.244.0.0/16"
SERVICE_NETWORK="10.96.0.0/12"
DOMAIN=jasper.org
#CONTROL_PLAN="kubeapi.${DOMAIN}:8443"


#kubeadm init --control-plane-endpoint="kubeapi.${DOMAIN}" \
#       --control-plane-endpoint=${CONTROL_PLAN} \
#       --kubernetes-version=v${KUBE_VERSION} \
#       --pod-network-cidr=${POD_NETWORK} \
#       --service-cidr=${SERVICE_NETWORK} \
#       --token-ttl=0 \
#       --upload-certs \
#       --image-repository=${IMAGES_URL} \
#       --skip-phases=addon/kube-proxy
#
kubeadm init --control-plane-endpoint="kubeapi.${DOMAIN}" \
        --kubernetes-version=v${KUBE_VERSION} \
        --pod-network-cidr=${POD_NETWORK} \
        --service-cidr=${SERVICE_NETWORK} \
        --token-ttl=0 \
        --upload-certs \
        --image-repository=${IMAGES_URL} \
        --skip-phases=addon/kube-proxy
#注意:--skip-phases=addon/kube-proxy表示不安装kube-proxy
#有如下输出结果

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

Alternatively, if you are the root user, you can run:

  export KUBECONFIG=/etc/kubernetes/admin.conf

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

You can now join any number of control-plane nodes running the following command on each as root:

  kubeadm join kubeapi.jasper.org:6443 --token dziprk.vq9csjpsvpqthw7l \
        --discovery-token-ca-cert-hash sha256:de98d23307d8d730734a461cc6ecb715f301daf9ecbfeabf266467440e81af0b \
        --control-plane --certificate-key 723fd45230337a0ef7d5dcd990a19815ac3bddbc3e5dc6d45dea1fc6e7402031

Please note that the certificate-key gives access to cluster sensitive data, keep it secret!
As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use
"kubeadm init phase upload-certs --upload-certs" to reload certs afterward.

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join kubeapi.jasper.org:6443 --token dziprk.vq9csjpsvpqthw7l \
        --discovery-token-ca-cert-hash sha256:de98d23307d8d730734a461cc6ecb715f301daf9ecbfeabf266467440e81af0b

kubeadm reset -f ; ipvsadm --clear  ; rm -rf ~/.kube

# 执行授权
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config

# 配置 kubectl 命令补全
apt update && apt -y install bash-completion
echo 'source <(kubeadm completion bash)' >> ~/.bashrc && exit
#source ~/.bashrc

# vim /etc/hosts 监控把 kubeapi 解析到 10.0.0.101 上，并查看以下状态

# 因为没有网络插件，节点处于 NotReady 状态
root@master01:~# kubectl  get node
NAME                  STATUS     ROLES           AGE     VERSION
master01.jasper.org   NotReady   control-plane   7m31s   v1.37.1
# 因为没有网络插件，coredns 的 Pod 处于 NotReady 状态
root@master01:~# kubectl  get pod -A
NAMESPACE     NAME                                          READY   STATUS    RESTARTS   AGE
kube-system   coredns-558f5766d5-c4r5k                      0/1     Pending   0          2m22s
kube-system   coredns-558f5766d5-vh87l                      0/1     Pending   0          2m22s
kube-system   etcd-master01.jasper.org                      1/1     Running   1          2m29s
kube-system   kube-apiserver-master01.jasper.org            1/1     Running   1          2m29s
kube-system   kube-controller-manager-master01.jasper.org   1/1     Running   1          2m29s
kube-system   kube-scheduler-master01.jasper.org            1/1     Running   1          2m29s

# Token过期后生成新的token：
root@master01:~# kubeadm token create --print-join-command
kubeadm join kubeapi.jasper.org:6443 --token an2oq3.7o47k0ukbljgwvz4 --discovery-token-ca-cert-hash sha256:de98d23307d8d730734a461cc6ecb715f301daf9ecbfeabf266467440e81af0b

# Master需要生成 --certificate-key
root@master01:~#  kubeadm init phase upload-certs  --upload-certs
[upload-certs] Storing the certificates in Secret "kubeadm-certs" in the "kube-system" Namespace
[upload-certs] Using certificate key:
cefdae80b20e904ef61ee93a860bea139be3e2df8dae95c13ee62e3841a45471

# 检查初始集群时的配置文件，找到 token: 7t2weq.bjbawausm0jaxury
secret=$(kubectl get secret -n kube-system|grep bootstrap |grep an2oq3 | awk '{print $1}')
kubectl -n kube-system get secret $secret -o json | jq -r '.data["expiration"]' | base64 -d

systemctl stop containerd kubelet
rm -fr /etc/kubernetes
rm -fr /var/lib/containerd /var/lib/kubelet # umount /var/lib/kubelet/pods/xxx
# 重新配置节点 Containerd 的配置文件
systemctl daemon-reload
systemctl restart containerd
systemctl restart kubelet
tail /var/log/message
# kubeadmin join ....

kubectl label nodes node01.jasper.org node-role.kubernetes.io/worker=
kubectl label nodes node02.jasper.org node-role.kubernetes.io/worker=

curl -LO https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml

# 关闭 tls 检验，更换国内镜像。
vim components.yaml
    spec:
      containers:
      - args:
        - --cert-dir=/tmp
        - --secure-port=10250
        - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
        - --kubelet-use-node-status-port
        - --metric-resolution=15s
        - --kubelet-insecure-tls # HTTPS 加密，跳过证书校验；鉴权（token）仍然保留
        image: registry.cn-hangzhou.aliyuncs.com/google_containers/metrics-server:v0.9.0
        imagePullPolicy: IfNotPresent


kubectl  apply -f components.yaml

root@master01:/data/k8s# kubectl  get pod -n kube-system
NAME                                      READY   STATUS    RESTARTS   AGE
metrics-server-74ccf797cb-4zm25           1/1     Running   0          47s

# kubectl  top node
NAME           CPU(cores)   CPU%   MEMORY(bytes)   MEMORY%   
master01.jasper.org   231m         5%     1620Mi          42%
master02.jasper.org   274m         6%     1203Mi          31%
master03.jasper.org   202m         5%     1251Mi          32%
node01.jasper.org     69m          1%     667Mi           17%
node02.jasper.org     73m          1%     650Mi           16%

# 方法1 yaml 文件
kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/headlamp/main/kubernetes-headlamp.yaml

# 方法2 helm
helm repo add headlamp https://kubernetes-sigs.github.io/headlamp/
helm search repo -l  headlamp # 查看版本
helm pull headlamp/headlamp --version 0.45.0

tar xf headlamp-0.45.0.tgz
cd headlamp/

cat > my-values.yaml << EOF
service:
  type: NodePort
  nodePort: 30080

resources:
  requests: { cpu: 100m, memory: 128Mi }
  limits:   { cpu: 500m, memory: 512Mi }
EOF

helm install my-headlamp  . \
  -f my-values.yaml \
  --create-namespace -n kube-system 

root@master01:~/hdp/headlamp# kubectl  -n kube-system get svc  my-headlamp
NAME          TYPE       CLUSTER-IP       EXTERNAL-IP   PORT(S)        AGE
my-headlamp   NodePort   192.168.249.40   <none>        80:30080/TCP   2m51s

cat > admin.yaml <<EOF
apiVersion: v1
kind: ServiceAccount
metadata:
  name: admin-user
  namespace: kube-system
---
apiVersion: v1
kind: Secret
metadata:
  name: admin-user
  namespace: kube-system
  annotations:
    kubernetes.io/service-account.name: "admin-user"   
type: kubernetes.io/service-account-token  
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding 
metadata: 
  name: admin-user
  annotations:
    rbac.authorization.kubernetes.io/autoupdate: "true"
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: admin-user
  namespace: kube-system
EOF

# 执行
kubectl apply -f admin.yaml

systemctl restart kubelet

kubectl edit cm kube-proxy -n kube-system

kubectl patch daemonset kube-proxy -p "{\"spec\":{\"template\":{\"metadata\":{\"annotations\":{\"date\":\"`date +'%s'`\"}}}}}" -n kube-system

# 查看Taints：
root@master01:~# kubectl describe node -l node-role.kubernetes.io/control-plane=  |grep Taints
Taints:             node-role.kubernetes.io/control-plane:NoSchedule
Taints:             node-role.kubernetes.io/control-plane:NoSchedule
Taints:             node-role.kubernetes.io/control-plane:NoSchedule

# 删除Taint：
kubectl taint node -l node-role.kubernetes.io/master node-role.kubernetes.io/control-plane:NoSchedule-

kubectl get po --al-namespaces

# 监控数据
kubectl top po -n kube-system

# 网络
kubectl get svc  # k8s的svc一般为 servic ip 段的第一个 ip
kubectl get svc -n kube-system # coredns 一般为 cluster ip 的第 10 个 ip

telnet 10.96.0.1 443
telnet 10.96.0.10 53

# 验证跨服务器访问 pod 是否通
kubectl get po --all-namespaces -owide

# 然后其他节点去访问
ping 172.168.32.130

# 验证pod之间是否通
kubectl get po --all-namespaces -owide
kubectl exec -it calico-node-qxs56 -n kube-system -- sh # 进入容器内访问

# dashboard
kubectl get po -n kubernetes-dashboard
kubectl get svc -n kubernetes-dashboard

kubectl edit svc  kubernetes-dashboard -n kubernetes-dashboard
type: NodePort #修改为 NodePort
kubectl get po -n kubernetes-dashboard
https://10.0.0.101:32534 # 浏览器访问

systemctl stop containerd kubelet
rm -fr /etc/kubernetes
rm -f /var/lib/containerd /var/lib/kubelet