Quick Start¶

curl -s https://books.8ops.top/attachment/kubernetes/bin/01-init.sh | bash

mkdir -p /data1/lib/containerd && \
    ([ -e /var/lib/containerd ] && mv /var/lib/containerd{,-$(date +%Y%m%d)} || /bin/true) && \
    ln -s /data1/lib/containerd /var/lib/containerd
ls -l /var/lib/containerd

mkdir -p /data1/lib/kubelet && \
    ([ -e /var/lib/kubelet ] && mv /var/lib/kubelet{,-$(date +%Y%m%d)} || /bin/true) && \
    ln -s /data1/lib/kubelet /var/lib/kubelet
ls -l /var/lib/kubelet      

mkdir -p /data1/lib/etcd && \
    ([ -e /var/lib/etcd ] && mv /var/lib/etcd{,-$(date +%Y%m%d)} || /bin/true) && \
    ln -s /data1/lib/etcd /var/lib/etcd
ls -l /var/lib/etcd

# CONTAINERD_VERSION=1.5.5-0ubuntu3~20.04.2
CONTAINERD_VERSION=1.6.12-0ubuntu1~20.04.1
apt install -y containerd=${CONTAINERD_VERSION}

apt-mark hold containerd
apt-mark showhold
dpkg -l | grep containerd

# 替换 ctr 运行时
mkdir -p /etc/containerd
containerd config default > /etc/containerd/config.toml-default
cp /etc/containerd/config.toml-default /etc/containerd/config.toml

#
# Example 
#   https://books.8ops.top/attachment/kubernetes/containerd-config.toml
#

sed -i 's#sandbox_image.*$#sandbox_image = "hub.8ops.top/google_containers/pause:3.6"#' /etc/containerd/config.toml  
sed -i 's#SystemdCgroup = false#SystemdCgroup = true#' /etc/containerd/config.toml 
grep -P 'sandbox_image|SystemdCgroup' /etc/containerd/config.toml  
systemctl restart containerd
systemctl status containerd

#
# Example
#   https://books.8ops.top/attachment/kubernetes/containerd-config.toml
#

# 方式一，操作系统受信私有CA
apt install -y ca-certificates
cp CERTIFICATE.crt /usr/local/share/ca-certificates/CERTIFICATE.crt
update-ca-certificates

# 方式二，编辑/etc/containerd/config.toml
# 支持私有Harbor，当使用私有CA签名时需要系统受信根CA或者采用非安全模式见编辑config.toml
……

[plugins] # relative
    ……
  [plugins."io.containerd.grpc.v1.cri"] # relative
        ……
        # sandbox_image
    sandbox_image = "hub.8ops.top/google_containers/pause:3.6"
        ……
      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
            # systemd
            SystemdCgroup = true
        ……
    # insecure
    [plugins."io.containerd.grpc.v1.cri".registry]
      [plugins."io.containerd.grpc.v1.cri".registry.mirrors] # relative
        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."hub.8ops.top"]
          endpoint = ["https://hub.8ops.top"]
      [plugins."io.containerd.grpc.v1.cri".registry.configs] # relative
        [plugins."io.containerd.grpc.v1.cri".registry.configs."hub.8ops.top".tls]
          insecure_skip_verify = true  

# QuickStart Usage
sed -i '/.registry.mirrors/a \        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."hub.8ops.top"]\n          endpoint = ["https://hub.8ops.top"]' /etc/containerd/config.toml
sed -i '/.registry.configs/a \         [plugins."io.containerd.grpc.v1.cri".registry.configs."hub.8ops.top".tls]\n          insecure_skip_verify = true ' /etc/containerd/config.toml
systemctl restart containerd
crictl pull hub.8ops.top/google_containers/pause:3.6

KUBERNETES_VERSION=1.23.0-00
apt install -y -q kubeadm=${KUBERNETES_VERSION} kubectl=${KUBERNETES_VERSION} kubelet=${KUBERNETES_VERSION}

apt-mark hold kubeadm kubectl kubelet
apt-mark showhold
dpkg -l | grep kube

# deprecated
cat > /etc/systemd/system/kubelet.service.d/0-containerd.conf <<EOF
[Service]
Environment="KUBELET_EXTRA_ARGS=--container-runtime=remote --runtime-request-timeout=15m --container-runtime-endpoint=unix:///run/containerd/containerd.sock"
EOF

# 用于运行crictl
cat > /etc/crictl.yaml <<EOF
runtime-endpoint: unix:///run/containerd/containerd.sock
image-endpoint: unix:///run/containerd/containerd.sock
timeout: 10
debug: false
EOF

systemctl restart containerd
crictl images
crictl ps -a

#
# 不配置指定时会默认依次按顺序使用：docker--> containerd --> cri-o，缺省默认使用docker
# 常见输出警告如下
# WARN[0000] image connect using default endpoints: [unix:///var/run/dockershim.sock unix:///run/containerd/containerd.sock unix:///run/crio/crio.sock]
#
# 若不存在容器运行时则报错如下
# FATA[0010] failed to connect: failed to connect: context deadline exceeded
#

# 默认配置
kubeadm config print init-defaults > kubeadm-init.yaml-default

#
# Example 
#   https://books.8ops.top/attachment/kubernetes/kubeadm-init.yaml-v1.23.0
#

# 默认镜像
kubeadm config images list -v 5

# 打印镜像
kubeadm config images list --config kubeadm-init.yaml -v 5

# 预取镜像
kubeadm config images pull --config kubeadm-init.yaml -v 5

apiVersion: kubeadm.k8s.io/v1beta3
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 10.101.11.240
  bindPort: 6443
nodeRegistration:
  criSocket: /run/containerd/containerd.sock
  name: K-KUBE-LAB-01
  taints: null
  kubeletExtraArgs: null
---
apiServer:
  timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns:
  imageRepository: hub.8ops.top/google_containers
  imageTag: 1.8.6
etcd:
  local:
    dataDir: /var/lib/etcd
imageRepository: hub.8ops.top/google_containers
kind: ClusterConfiguration
kubernetesVersion: 1.23.0
controlPlaneEndpoint: 10.101.11.110:6443
networking:
  dnsDomain: cluster.local
  podSubnet: 172.20.0.0/16
  serviceSubnet: 192.168.0.0/16
scheduler: {}
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
cgroupDriver: systemd
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration

kubeadm init --config kubeadm-init.yaml --upload-certs -v 5

mkdir -p ~/.kube && ln -s /etc/kubernetes/admin.conf ~/.kube/config 

kubectl get no

# kubeadm
kubectl -n kube-system edit cm kubeadm-config

# kubelet

kubectl -n kube-system edit cm kubelet-config-1.23
……
        # GC
    imageGCLowThresholdPercent: 40
    imageGCHighThresholdPercent: 50
    # Resource
    systemReserved:
      cpu: 500m
      memory: 500m
    kubeReserved:
      cpu: 500m
      memory: 500m
    evictionPressureTransitionPeriod: 300s # upgrade
    nodeStatusReportFrequency: 10s         # upgrade
    nodeStatusUpdateFrequency: 10s         # upgrade
    cgroupDriver: systemd
    maxPods: 200
    resolvConf: /etc/resolv.conf
kind: ConfigMap                            # relative
……

# kube-proxy 
# 当flannel采用host-gw时，需要开启ipvs
kubectl -n kube-system edit cm kube-proxy

……
    configSyncPeriod: 5s # upgrade
    mode: "ipvs"         # upgrade
    ipvs:                # relative
      tcpTimeout: 900s   # upgrade
      syncPeriod: 5s     # upgrade
      minSyncPeriod: 5s  # upgrade
……

# kube-proxy
kubectl -n kube-system edit daemonset kube-proxy
……
    resources:
      requests:
        cpu: 100m
        memory: 128Mi
……

# coredns
kubectl -n kube-system edit deployment coredns
……
    resources:
      requests:
        cpu: 50m
        memory: 64Mi
……

# kube-flannel-ds
kubectl -n kube-system edit daemonset kube-flannel-ds
……
    resources:
      requests:
        cpu: 50m
        memory: 64Mi
……

# kube-apiserver - [Burstable]
# kube-controller-manager - [Burstable]
# kube-scheduler - [Burstable]

# 上传 cert
kubeadm init phase upload-certs --upload-certs

#
## 或者获取已经上传的certs
# openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | \
#    openssl rsa -pubin -outform der 2>/dev/null | \
#    openssl dgst -sha256 -hex | sed 's/^.* //'
#

# 生成 token
kubeadm token generate

# 打印 join control-plane,master
kubeadm token create um69xx.qg70e11b92fmsevs --print-join-command -v 5 \
  --certificate-key 3c7b6191702471e8ee2f3dea4ce422044b266928c808b6957fb0c81ef2388bed

# 打印 join node
kubeadm token create kvj7wh.buud7djga5j76xil --print-join-command -v 5

# ---
# 一步就位

# join control-plane
kubeadm token create --print-join-command --certificate-key xx

# join node
kubeadm token create --print-join-command 

kubeadm token list

kubeadm join 10.101.11.110:6443 --token abcdef.0123456789abcdef \
 --discovery-token-ca-cert-hash sha256:ae1d593bbadecf245c30f4c1cfe9250faa0aaa9e4c27b7f34bcb10142d0dd0c8 \
 --control-plane --certificate-key 811e33703005a1df116201ae6469d86746274c3579e62b7c924cc4c13a804bca -v 5in the cluster.

kubeadm join 10.101.11.110:6443 --token abcdef.0123456789abcdef \
  --discovery-token-ca-cert-hash sha256:ae1d593bbadecf245c30f4c1cfe9250faa0aaa9e4c27b7f34bcb10142d0dd0c8

#
# Reference 
#   https://raw.githubusercontent.com/flannel-io/flannel/master/Documentation/kube-flannel.yml
#
# Example
#   https://books.8ops.top/attachment/kubernetes/kube-flannel.yaml-v0.15.1
#

kubectl apply -f kube-flannel.yaml

……
  net-conf.json: | # relative
    {
      "Network": "172.20.0.0/16",
      "Backend": {
        "Type": "host-gw"
      }
    }
……
    # 镜像替换为私有地址
      initContainers:
      - name: install-cni-plugin
        image: hub.8ops.top/google_containers/flannel-cni-plugin:v1.0.0
        ……
      - name: install-cni
        image: hub.8ops.top/google_containers/flannel:v0.15.1
        ……
      containers:
      - name: kube-flannel
        image: hub.8ops.top/google_containers/flannel:v0.15.1

kubectl cluster-info

kubectl get cs

etcdctl member list \
    --endpoints=https://10.101.11.240:2379 \
    --cacert=/etc/kubernetes/pki/etcd/ca.crt \
    --cert=/etc/kubernetes/pki/etcd/server.crt \
    --key=/etc/kubernetes/pki/etcd/server.key

etcdctl endpoint status \
    --endpoints=https://10.101.11.240:2379,https://10.101.11.114:2379,https://10.101.11.154:2379 \
    --cacert=/etc/kubernetes/pki/etcd/ca.crt \
    --cert=/etc/kubernetes/pki/etcd/server.crt \
    --key=/etc/kubernetes/pki/etcd/server.key

etcdctl endpoint status \
    --cluster \
    --cacert=/etc/kubernetes/pki/etcd/ca.crt \
    --cert=/etc/kubernetes/pki/etcd/server.crt \
    --key=/etc/kubernetes/pki/etcd/server.key

kubectl get all -A

kubectl get no

apt install -y ipvsadm

ipvsadm -ln

# 偶尔出现 coredns 不就位情况是因为加 Node 之前装网络插件 flannel 
kubectl get pod -A -o wide

# 亦或 coredns 在 control plane 节点上
# 重置后重新初始化集群，先安装 flannel，再 Join 其他节点
# 可以解决此现象

~ # kubectl edit clusterrole system:coredns
……
# append
- apiGroups:
  - discovery.k8s.io
  resources:
  - endpointslices
  verbs:
  - list
  - watch

kubectl -n kube-system delete pod/coredns-55866688ff-hwp4m pod/coredns-55866688ff-tn8sj

sed -i '/--port/d' /etc/kubernetes/manifests/kube-controller-manager.yaml
sed -i '/--port/d' /etc/kubernetes/manifests/kube-scheduler.yaml

#
# 方式一，启动busybox
## 建议使用1.28.0，新版本会有nslookup的BUG
# 
kubectl run busybox --image hub.8ops.top/third/busybox:1.28.0 --command -- sh -c "while true;do sleep 60;date;done"

# kubectl apply -f https://books.8ops.top/attachment/kubernetes/app/51-busybox-daemonset.yaml
kubectl run nginx --image hub.8ops.top/third/nginx:1.21.3

#
# 方式二，启动jessie-dnsutils:1.3
## 自带dns检测工具

# 
# Case by case 看报什么错
# e.g. kubectl apply -f https://books.8ops.top/attachment/kubernetes/app/51-nginx-deployment.yaml
#

# 
# Example
#   https://books.8ops.top/attachment/kubernetes/app/50-busybox-daemonset.yaml
#   https://books.8ops.top/attachment/kubernetes/app/51-nginx-deployment.yaml
#   https://books.8ops.top/attachment/kubernetes/app/52-echoserver.yaml
#