Kubernetes: k8s-安装篇-kueadm-1.26

yum install docker-ce-20.10.* docker-ce-cli-20.10.* -y

# cat <<EOF | sudo tee /etc/modules-load.d/containerd.conf
overlay
br_netfilter
EOF

# modprobe -- overlay
# modprobe -- br_netfilter

# cat <<EOF | sudo tee /etc/sysctl.d/99-kubernetes-cri.conf
net.bridge.bridge-nf-call-iptables  = 1
net.ipv4.ip_forward                 = 1
net.bridge.bridge-nf-call-ip6tables = 1
EOF

# sysctl --system

# mkdir -p /etc/containerd
# containerd config default | tee /etc/containerd/config.toml

# vim /etc/containerd/config.toml

# systemctl daemon-reload
# systemctl enable --now containerd

# cat > /etc/crictl.yaml <<EOF
runtime-endpoint: unix:///run/containerd/containerd.sock
image-endpoint: unix:///run/containerd/containerd.sock
timeout: 10
debug: false
EOF

yum install docker-ce-20.10.* docker-ce-cli-20.10.* -y

 mkdir /etc/docker
cat > /etc/docker/daemon.json <<EOF
{
  "exec-opts": ["native.cgroupdriver=systemd"]
}
EOF

systemctl daemon-reload && systemctl enable --now docker

yum list kubeadm.x86_64 --showduplicates | sort -r

yum install -y kubelet kubeadm kubectl
#指定版本 yum install kubeadm-1.23* kubelet-1.23* kubectl-1.23* -y

cat >/etc/sysconfig/kubelet<<EOF
KUBELET_EXTRA_ARGS="--pod-infra-container-image=registry.cn-6
hangzhou.aliyuncs.com/google_containers/pause:3.6"
EOF

cat >/etc/sysconfig/kubelet<<EOF
KUBELET_KUBEADM_ARGS="--container-runtime=remote --runtime-request-timeout=15m --container-runtime-endpoint=unix:///run/containerd/containerd.sock"
EOF

systemctl daemon-reload
systemctl enable --now kubelet

Available Commands:
    alpha #kubeadm 处于测试阶段的命令
    completion #bash 命令补全，需要安装 bash-completion
        #mkdir /data/scripts -p
        #kubeadm completion bash > /data/scripts/kubeadm_completion.sh
        #source /data/scripts/kubeadm_completion.sh
        #vim /etc/profile #添加内容
        source /data/scripts/kubeadm_completion.sh
    config #管理 kubeadm 集群的配置，该配置保留在集群的 ConfigMap 中
        #kubeadm config print init-defaults
    help Help about any command
    init    #启动一个 Kubernetes 主节点
    join    #将节点加入到已经存在的 k8s master
    reset   #还原使用 kubeadm init 或者 kubeadm join 对系统产生的环境变化。清空节点数据
    token   #管理 token
    upgrade #升级 k8s 版本
    version #查看版本信息

root@docker-k8s-node1:~# kubeadm init --help
--apiserver-advertise-address string    #K8S API Server 将要监听的监听的本机 IP
--apiserver-bind-port int32             #API Server 绑定的端口,默认为 6443

--apiserver-cert-extra-sans stringSlice #可选的证书额外信息，用于指定 API Server 的服务器证书。可以是 IP 地址也可以是 DNS 名称。
--cert-dir string                       #证书的存储路径，缺省路径为 /etc/kubernetes/pki
--certificate-key string                #定义一个用于加密 kubeadm-certs Secret 中的控制平台证书的密钥
--config string #kubeadm                #配置文件的路径
--control-plane-endpoint string         #为控制平台指定一个稳定的 IP 地址或 DNS 名称，即配置一个可以长期使用切是高可用的 VIP 或者域名，
                                        k8s多master 高可用基于此参数实现
--cri-socket string                     #要连接的 CRI(容器运行时接口，Container Runtime Interface, 简称 CRI)套接字的路径，如果为空，则 kubeadm 
                                        将尝试自动检测此值，"仅当安装了多个 CRI 或具有非标准 CRI 插槽时，才使用此选项"
--dry-run                               #不要应用任何更改，只是输出将要执行的操作，其实就是测试运行。
--experimental-kustomize string         #用于存储 kustomize 为静态 pod 清单所提供的补丁的路径。
--feature-gates string                  #一组用来描述各种功能特性的键值（key=value）对，选项是：IPv6DualStack=true|false (ALPHA - default=false)
--ignore-preflight-errors strings       #可以忽略检查过程 中出现的错误信息，比如忽略 swap，如果为 all 就忽略所有
--image-repository string               #设置一个镜像仓库，默认为 k8s.gcr.io
--kubernetes-version string             #指定安装 k8s 版本，默认为 stable-1
--node-name string                      #指定 node 节点名称
--pod-network-cidr                      #设置 pod ip 地址范围
--service-cidr                          #设置 service 网络地址范围，默认值："10.96.0.0/12"
--service-dns-domain string             #设置 k8s内部域名，默认为 cluster.local，会有相应的 DNS 服务(kube-dns/coredns)解析生成的域名记录。可以换成公司内部使用的域名，如me.local
--skip-certificate-key-print            #不打印用于加密的key 信息
--skip-phases strings                   #要跳过哪些阶段
--skip-token-print                      #跳过打印 token 信息
--token                                 #指定 token
--token-ttl                             #指定 token 过期时间，默认为 24 小时，0 为永不过期
--upload-certs                          #更新证书

#全局可选项：
--add-dir-header                        #如果为 true，在日志头部添加日志目录
--log-file string                       #如果不为空，将使用此日志文件
--log-file-max-size uint                #设置日志文件的最大大小，单位为兆，默认为 1800 兆，0 为没有限制
--rootfs                                #宿主机的根路径，也就是绝对路径
--skip-headers                          #如果为 true，在 log 日志里面不显示标题前缀
--skip-log-headers                      #如果为 true，在 log 日志里里不显示标题

# kubeadm version #查看当前 kubeadm 版本
# kubeadm version
kubeadm version: &version.Info{Major:"1", Minor:"17", GitVersion:"v1.23.3",

#查看安装指定版本 k8s 需要的镜像有哪些
# kubeadm config images list --kubernetes-version v1.23.3
k8s.gcr.io/kube-apiserver:
k8s.gcr.io/kube-controller-manager:
k8s.gcr.io/kube-scheduler:
k8s.gcr.io/kube-proxy:
k8s.gcr.io/pause: #为每个pod封装一个底层的网络
k8s.gcr.io/etcd:
k8s.gcr.io/coredns/coredns:

kubeadm config print init-defaults > kubeadm-config.yaml #将默认配置输出至文件
vim kubeadm-config.yaml

apiVersion: kubeadm.k8s.io/v1beta2
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: 7t2weq.bjbawausm0jaxury # token，第二天可能过期，后面介绍防止过期
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 192.168.0.107 # 本地监听地址
  bindPort: 6443
nodeRegistration:
  # criSocket: /var/run/dockershim.sock  # 如果是Docker作为Runtime配置此项
  criSocket: unix:///var/run/containerd/containerd.sock # 如果是Containerd作为Runtime配置此项
  name: k8s-master01
  taints:               # 容忍，不可调度
  - effect: NoSchedule
    key: node-role.kubernetes.io/control-plane
---
apiServer:
  certSANs:
  - 192.168.0.236 # 证书颁发给谁， 改成 vip 或者是公有云负载均衡地址
  timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controlPlaneEndpoint: 192.168.0.236:16443 #基于 VIP 的 Endpoint
controllerManager: {}
dns:
  type: CoreDNS
etcd:
  local:
    dataDir: /var/lib/etcd
imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containers # 镜像库
kind: ClusterConfiguration
kubernetesVersion: v1.26.0 # 更改此处的版本号和kubeadm version一致
networking:                # 网络信息
  dnsDomain: cluster.local
  podSubnet: 172.16.0.0/12
  serviceSubnet: 10.96.0.0/12
scheduler: {}

kubeadm config migrate --old-config kubeadm-config.yaml --new-config new.yaml

for i in k8s-master02 k8s-master03; do scp new.yaml $i:/root/; done
kubeadm config images pull --config /root/new.yaml 

kubeadm init --config /root/new.yaml  --upload-certs

kubeadm reset -f ; ipvsadm --clear  ; rm -rf ~/.kube

Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

Alternatively, if you are the root user, you can run:

  export KUBECONFIG=/etc/kubernetes/admin.conf

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

You can now join any number of the control-plane node running the following command on each as root:

  kubeadm join 192.168.0.236:16443 --token 7t2weq.bjbawausm0jaxury \
    --discovery-token-ca-cert-hash sha256:8c92ecb336be2b9372851a9af2c7ca1f7f60c12c68f6ffe1eb513791a1b8a908 \
    --control-plane --certificate-key ac2854de93aaabdf6dc440322d4846fc230b290c818c32d6ea2e500fc930b0aa

Please note that the certificate-key gives access to cluster sensitive data, keep it secret!
As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use
"kubeadm init phase upload-certs --upload-certs" to reload certs afterward.

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 192.168.0.236:16443 --token 7t2weq.bjbawausm0jaxury \
    --discovery-token-ca-cert-hash sha256:8c92ecb336be2b9372851a9af2c7ca1f7f60c12c68f6ffe1eb513791a1b8a908

cat <<EOF >> /root/.bashrc
export KUBECONFIG=/etc/kubernetes/admin.conf
EOF
source /root/.bashrc

 [root@k8s-master01 ~]# kubectl get nodes
NAME           STATUS     ROLES                  AGE   VERSION
k8s-master01   NotReady   control-plane,master   74s   v1.20.0
# kubectl get svc

[root@k8s-master01 ~]# kubectl get pods -n kube-system -o wide
NAME                                   READY     STATUS    RESTARTS   AGE       IP              NODE
coredns-777d78ff6f-kstsz               0/1       Pending   0          14m       <none>          <none>
coredns-777d78ff6f-rlfr5               0/1       Pending   0          14m       <none>          <none>
etcd-k8s-master01                      1/1       Running   0          14m       192.168.0.107   k8s-master01
kube-apiserver-k8s-master01            1/1       Running   0          13m       192.168.0.107   k8s-master01
kube-controller-manager-k8s-master01   1/1       Running   0          13m       192.168.0.107   k8s-master01
kube-proxy-8d4qc                       1/1       Running   0          14m       192.168.0.107   k8s-master01
kube-scheduler-k8s-master01            1/1       Running   0          13m       192.168.0.107   k8s-master01

yum install -y bash-completion
source /usr/share/bash-completion/bash_completion
source <(kubectl completion bash)
echo "source <(kubectl completion bash)" >> ~/.bashrc

# Token过期后生成新的token：
[root@k8s-master01 ~]# kubeadm token create --print-join-command
kubeadm join 192.168.1.236:16443 --token 8k8qzk.d43ed9gfgw1st3xi     --discovery-token-ca-cert-hash sha256:3aa4cf3c52c1956cb86d2911fe0f6b8898bfa43c06966b2f1095e5000a00d1a4 

# Master需要生成--certificate-key
[root@k8s-master01 ~]# kubeadm init phase upload-certs  --upload-certs
[upload-certs] Storing the certificates in Secret "kubeadm-certs" in the "kube-system" Namespace
[upload-certs] Using certificate key:
43c5695789c0dc4433f480a05683d55887e836b71b452b407138d8dd54cad937

# 检查初始集群时的配置文件，找到 token: 7t2weq.bjbawausm0jaxury
secret=$(kubectl get secret -n kube-system|grep bootstrap |grep 7t2weq | awk '{print $1}')
kubectl get secret $secret -o json | jq -r '.data["expiration"]' | base64 -d

systemctl stop containerd kubelet
rm -fr /etc/kubernetes
rm -fr /var/lib/containerd /var/lib/kubelet # umount /var/lib/kubelet/pods/xxx
# 重新配置节点 Containerd 的配置文件
systemctl daemon-reload
systemctl restart containerd
systemctl restart kubelet
tail /var/log/message
# kubeadmin join ....

#https://github.com/coreos/flannel/
kubectl apply -f kube-flannel.yml

kubectl apply -f https://docs.projectcalico.org/v3.1/getting-started/kubernetes/installation/hosted/rbac-kdd.yaml
kubectl apply -f https://docs.projectcalico.org/v3.1/getting-started/kubernetes/installation/hosted/kubernetes-datastore/calico-networking/1.7/calico.yaml

cd /root/k8s-ha-install && git checkout manual-installation-v1.26.x && cd calico/
# 修改Pod网段
POD_SUBNET=`cat /etc/kubernetes/manifests/kube-controller-manager.yaml | grep cluster-cidr= | awk -F= '{print $NF}'`

sed -i "s#POD_CIDR#${POD_SUBNET}#g" calico.yaml
kubectl apply -f calico.yaml

#镜像地址换为国内的
#生产环境建议把镜像放到公司内部
# grep -Po 'image:\s+\K\S+' calico-etcd.yaml 

[root@k8s-master01 dashboard]# kubectl  get po -n kube-system   
NAME                                       READY   STATUS    RESTARTS   AGE
calico-kube-controllers-5f6d4b864b-khq4h   1/1     Running   0          13m
calico-node-5tvxh                          1/1     Running   0          13m
calico-node-kffn7                          1/1     Running   0          13m
calico-node-lltfs                          1/1     Running   0          13m
calico-node-nhgn8                          1/1     Running   0          13m
coredns-54d67798b7-8w5hd                   1/1     Running   0          117m
coredns-54d67798b7-vb2ll                   1/1     Running   0          117m
etcd-k8s-master01                          1/1     Running   0          117m
etcd-k8s-master02                          1/1     Running   0          104m
kube-apiserver-k8s-master01                1/1     Running   0          117m
kube-apiserver-k8s-master02                1/1     Running   0          104m
kube-controller-manager-k8s-master01       1/1     Running   1          117m
kube-controller-manager-k8s-master02       1/1     Running   0          104m
kube-proxy-5bws8                           1/1     Running   0          117m
kube-proxy-pbqjc                           1/1     Running   0          104m
kube-proxy-tpwbt                           1/1     Running   0          86m
kube-proxy-vbpc5                           1/1     Running   0          86m
kube-scheduler-k8s-master01                1/1     Running   1          117m
kube-scheduler-k8s-master02                1/1     Running   0          104m
metrics-server-545b8b99c6-hkgnz            1/1     Running   0          2m38s

sysctl net.bridge.bridge-nf-call-iptables=1
kubectl apply -f "https://cloud.weave.works/k8s/net?k8s-version=$(kubectl version | base64 | tr -d '\n')"

vim /etc/kubernetes/manifests/kube-apiserver.yaml

systemctl restart kubelet

cd /root/k8s-ha-install/dashboard/

[root@k8s-master01 dashboard]# kubectl  create -f .
serviceaccount/admin-user created
clusterrolebinding.rbac.authorization.k8s.io/admin-user created
namespace/kubernetes-dashboard created
serviceaccount/kubernetes-dashboard created
service/kubernetes-dashboard created
secret/kubernetes-dashboard-certs created
secret/kubernetes-dashboard-csrf created
secret/kubernetes-dashboard-key-holder created
configmap/kubernetes-dashboard-settings created
role.rbac.authorization.k8s.io/kubernetes-dashboard created
clusterrole.rbac.authorization.k8s.io/kubernetes-dashboard created
rolebinding.rbac.authorization.k8s.io/kubernetes-dashboard created
clusterrolebinding.rbac.authorization.k8s.io/kubernetes-dashboard created
deployment.apps/kubernetes-dashboard created
service/dashboard-metrics-scraper created
deployment.apps/dashboard-metrics-scraper created

kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.5.0/aio/deploy/recommended.yaml

vim admin.yaml

apiVersion: v1
kind: ServiceAccount
metadata:
  name: admin-user
  namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding 
metadata: 
  name: admin-user
  annotations:
    rbac.authorization.kubernetes.io/autoupdate: "true"
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: admin-user
  namespace: kube-system

# kubectl apply -f admin.yaml -n kube-system  

# 在最后面加
--test-type --ignore-certificate-errors

kubectl edit svc kubernetes-dashboard -n kubernetes-dashboard

[root@k8s-master01 1.1.1]# kubectl -n kube-system describe secret $(kubectl -n kube-system get secret | grep admin-user | awk '{print $1}')
token:  xxxxxx

# 查看所有容器状态
[root@k8s-master01 dashboard]# kubectl get po -A
NAMESPACE              NAME                                         READY   STATUS    RESTARTS   AGE
kube-system            calico-kube-controllers-5f6d4b864b-khq4h     1/1     Running   0          16m
kube-system            calico-node-5tvxh                            1/1     Running   0          16m
kube-system            calico-node-kffn7                            1/1     Running   0          16m
kube-system            calico-node-lltfs                            1/1     Running   0          16m
kube-system            calico-node-nhgn8                            1/1     Running   0          16m
kube-system            coredns-54d67798b7-8w5hd                     1/1     Running   0          121m
kube-system            coredns-54d67798b7-vb2ll                     1/1     Running   0          121m
kube-system            etcd-k8s-master01                            1/1     Running   0          121m
kube-system            etcd-k8s-master02                            1/1     Running   0          107m
kube-system            kube-apiserver-k8s-master01                  1/1     Running   0          121m
kube-system            kube-apiserver-k8s-master02                  1/1     Running   0          107m
kube-system            kube-controller-manager-k8s-master01         1/1     Running   1          121m
kube-system            kube-controller-manager-k8s-master02         1/1     Running   0          107m
kube-system            kube-proxy-5bws8                             1/1     Running   0          121m
kube-system            kube-proxy-pbqjc                             1/1     Running   0          107m
kube-system            kube-proxy-tpwbt                             1/1     Running   0          90m
kube-system            kube-proxy-vbpc5                             1/1     Running   0          90m
kube-system            kube-scheduler-k8s-master01                  1/1     Running   1          121m
kube-system            kube-scheduler-k8s-master02                  1/1     Running   0          107m
kube-system            metrics-server-545b8b99c6-hkgnz              1/1     Running   0          6m25s
kubernetes-dashboard   dashboard-metrics-scraper-7645f69d8c-hftvd   1/1     Running   0          2m31s
kubernetes-dashboard   kubernetes-dashboard-78cb679857-zksm9        1/1     Running   0          2m32s

# 将type的ClusterIP更改为NodePort（如果已经为NodePort忽略此步骤）：
[root@k8s-master01 dashboard]# kubectl edit svc kubernetes-dashboard -n kubernetes-dashboard
type: NodePort

# 检查 kubeadm 管理的本地 PKI 中证书的到期时间
kubeadm certs check-expiration 

#检查证书过期时间
kubeadm alpha certs check-expiration

#更新证书
#备份集群配置
kubectl -n kube-system get cm kubeadm-config -o yaml > kubeadm-config.yaml
#更新证书
kubeadm alpha certs renew all

#再次检查证书
kubeadm alpha certs check-expiration

#重启 kubelet 服务
systemctl restart kubelet
#更新配置
cp /etc/kubernetes/admin.conf ~/.kube/config
# 验证集群
kubectl get node

kubeadm version
# 下载源码，切换集群一致的版本
git clone https://gitee.com/mirrors/kubernetes.git 
git checkout v1.26.0
# 重新编译
docker run -it --rm -v `pwd`:/go/src/ registry.cn-beijing.aliyuncs.com/dotbalo/golang:kubeadm bash
cd /go/src
go env -w GOPROXY=https://goproxy.cn,direct
go env -w GOSUMDB=off
grep "365" cmd/kubeadm/app/constants/constants.go
sed -i 's@365@365 * 100@g' cmd/kubeadm/app/constants/constants.go
grep '365'  cmd/kubeadm/app/constants/constants.go
mkdir -p _output/
chmod 777 -R _output/
make WHAT=cmd/kubeadm
cp _output/bin/kubeadm ./kubeadm

# 更新证书，各个 mater 节点操作
cp ./kubeadm /opt/
/opt/kubeadm verson
/opt/kubeadm certs renew all
kubeadm certs check-expiration
systemctl restart kubelet