建筑公司企业标语无线网络优化
一、前言
简要记录一下离线环境下 K3S 的搭建，版本为 v1.23.17+k3s1，使用外部数据库 MySQL
作元数据存储，禁用默认组件（coredns、servicelb、traefik、local-storage、metrics-server）并使用 Helm
单独安装（coredns、metrics-server、traefik、longhorn）。
需要一台联网主机（虚拟机），和多台未联网主机（服务器）。
二、联网虚拟机
2.1 快速引导一个单节点集群
curl -fsSL https://rancher-mirror.oss-cn-beijing.aliyuncs.com/k3s/k3s-install.sh | INSTALL_K3S_MIRROR=cn INSTALL_K3S_VERSION=v1.23.17+k3s1 bash -s - server \--data-dir /data/k3s/var/lib/rancher/k3s \--cluster-cidr 10.8.0.0/16 \--service-cidr 10.16.0.0/16 \--cluster-dns 10.16.0.10 \--service-node-port-range 1-65535 \--kube-proxy-arg proxy-mode=ipvs \--disable coredns \--disable servicelb \--disable traefik \--disable local-storage \--disable metrics-server
2.2 快速安装 Longhorn 的依赖
Installation Requirements
# yum
yum install iscsi-initiator-utils nfs-utils# ubuntu
apt install open-iscsi nfs-common# 启动
systemctl enable iscsid --now
2.3 快速安装应用（通过 Helm Controller）
### coredns
---
apiVersion: helm.cattle.io/v1
kind: HelmChart
metadata:name: corednsnamespace: kube-systemlabels:app: coredns
spec:repo: https://coredns.github.io/helmchart: corednstargetNamespace: kube-systembootstrap: truevaluesContent: |-fullnameOverride: corednsserviceType: ClusterIPservice:clusterIP: 10.16.0.10name: corednsservers:- zones:- zone: .port: 53plugins:- name: errors- name: healthconfigBlock: |-lameduck 5s- name: ready- name: kubernetesparameters: cluster.local in-addr.arpa ip6.arpaconfigBlock: |-pods insecurefallthrough in-addr.arpa ip6.arpattl 30- name: prometheusparameters: 0.0.0.0:9153- name: forwardparameters: . /etc/resolv.conf- name: cacheparameters: 30- name: loop- name: reload- name: loadbalance### metrics-server
---
apiVersion: helm.cattle.io/v1
kind: HelmChart
metadata:name: metrics-servernamespace: kube-systemlabels:app: metrics-server
spec:repo: https://charts.bitnami.com/bitnamichart: metrics-servertargetNamespace: kube-systembootstrap: truevaluesContent: |apiService:create: trueextraArgs:- --kubelet-insecure-tls- --kubelet-use-node-status-port- --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname- --metric-resolution=15s### traefik
---
apiVersion: v1
kind: Namespace
metadata:name: traefik-system---
apiVersion: helm.cattle.io/v1
kind: HelmChart
metadata:name: traefiknamespace: traefik-systemlabels:app: traefik
spec:repo: https://traefik.github.io/chartschart: traefiktargetNamespace: traefik-systembootstrap: truevaluesContent: |-deployment:kind: DeploymentingressClass:enabled: trueisDefaultClass: trueproviders:kubernetesCRD:enabled: trueallowCrossNamespace: trueallowExternalNameServices: trueallowEmptyServices: truekubernetesIngress:enabled: trueallowExternalNameServices: trueallowEmptyServices: truepublishedService:enabled: trueports:traefik:port: 9000protocol: TCPexpose: falseexposedPort: 9000metrics:port: 9100protocol: TCPexpose: falseexposedPort: 9100web:port: 80protocol: TCPexpose: trueexposedPort: 80nodePort: 30080websecure:port: 443protocol: TCPexpose: trueexposedPort: 443ndoePort: 30443tls:enabled: trueservice:type: NodePortsecurityContext:capabilities:drop: []add: [ALL]readOnlyRootFilesystem: falsepodSecurityContext:runAsGroup: 0runAsNonRoot: falserunAsUser: 0### longhorn
---
apiVersion: v1
kind: Namespace
metadata:name: longhorn-system---
apiVersion: helm.cattle.io/v1
kind: HelmChart
metadata:name: longhornnamespace: longhorn-systemlabels:app: longhorn
spec:repo: https://charts.longhorn.iochart: longhorntargetNamespace: longhorn-systembootstrap: truevaluesContent: |-persistence:defaultClassReplicaCount: 1csi:attacherReplicaCount: 1provisionerReplicaCount: 1resizerReplicaCount: 1snapshotterReplicaCount: 1defaultSettings:defaultDataPath: /data/longhorndefaultReplicaCount: 1deletingConfirmationFlag: truelonghornUI:replicas: 1longhornConversionWebhook:replicas: 1longhornAdmissionWebhook:replicas: 1longhornRecoveryBackend:replicas: 1ingress:enabled: truehost: longhorn.example.org
kubectl apply -f charts.yaml
三、资源准备
3.1 下载 Longhorn 依赖
-
查看服务器 glibc 版本
ldd --versionos os version glibc version centos 7.9 2.17 centos 8.4 2.28 ubuntu 18.04 2.27 ubuntu 20.04 2.31 ubuntu 22.04 2.35 -
创建对应 glibc 版本的容器
# centos 7 kubectl run centos --image=centos:7.9.2009 --command -- /bin/sleep infinity kubectl exec -it pod/centos -- /bin/bash# ubuntu 22 kubectl run ubuntu --image=ubuntu:22.04 --command -- /bin/sleep infinity kubectl exec -it pod/ubuntu -- /bin/bash -
下载依赖
# yum yum install iscsi-initiator-utils nfs-utils --downloadonly --downloaddir=rpm -y tar -czvf ./rpm.tar.gz ./rpm# apt apt update && apt install open-iscsi nfs-common --download-only -y && mkdir -p deb && cp /var/cache/apt/archives/*.deb deb tar -czvf ./deb.tar.gz ./deb -
复制出依赖
# yum kubectl cp centos:/rpm.tar.gz ./rpm.tar.gz# apt kubectl cp ubuntu:/deb.tar.gz ./deb.tar.gz
3.2 下载 K3S 资源
参考文档：离线安装
wget https://github.com/k3s-io/k3s/releases/download/v1.23.17+k3s1/k3s-airgap-images-amd64.tar.gz
wget https://github.com/k3s-io/k3s/releases/download/v1.23.17+k3s1/k3s
wget https://get.helm.sh/helm-v3.12.2-linux-amd64.tar.gz
wget https://get.k3s.io -O install.sh
3.3 下载 HelmChart 和导出镜像
# 下载 helm chart 包
helm repo add coredns https://coredns.github.io/helm && helm pull coredns/coredns --version 1.26.0
helm repo add bitnami https://charts.bitnami.com/bitnami && helm pull bitnami/metrics-server --version 6.5.2
helm repo add traefik https://traefik.github.io/charts && helm pull traefik/traefik --version 24.0.0
helm repo add longhorn https://charts.longhorn.io && helm pull longhorn/longhorn --version 1.5.1# 导出镜像
k3s ctr image ls -q | grep -v 'sha256' | sort -u | xargs k3s ctr image export image.tar
四、未联网服务器
4.1 准备 K3S 资源
# 准备 k3s 镜像
mkdir -p /data/k3s/var/lib/rancher/k3s/agent/images
cp ./k3s-airgap-images-amd64.tar.gz /data/k3s/var/lib/rancher/k3s/agent/images# 准备 k3s 二进制可执行文件
install ./k3s /usr/local/bin# 准备 helm 二进制可执行文件
tar -zxvf ./helm-v3.12.2-linux-amd64.tar.gz
install ./linux-amd64/helm /usr/local/bin# 准备 k3s 安装脚本
chmod +x ./install.sh
4.2 引导第一个 Server 节点启动
# 引导 Server
INSTALL_K3S_MIRROR=cn INSTALL_K3S_VERSION=v1.23.17+k3s1 ./install.sh server \--data-dir /data/k3s/var/lib/rancher/k3s \--cluster-cidr 10.8.0.0/16 \--service-cidr 10.16.0.0/16 \--cluster-dns 10.16.0.10 \--service-node-port-range 1-65535 \--kube-proxy-arg proxy-mode=ipvs \--disable coredns \--disable servicelb \--disable traefik \--disable local-storage \--disable metrics-server \--datastore-endpoint="mysql://<USERNAME>:<PASSWORD>@tcp(<HOST>:3306)/<DATABASE>"# 查看 Token
cat /data/k3s/var/lib/rancher/k3s/server/token
4.3 引导其它 Server 节点加入
配置标识在所有 Server 节点必须是相同的。
INSTALL_K3S_MIRROR=cn INSTALL_K3S_VERSION=v1.23.17+k3s1 ./install.sh server \--data-dir /data/k3s/var/lib/rancher/k3s \--cluster-cidr 10.8.0.0/16 \--service-cidr 10.16.0.0/16 \--cluster-dns 10.16.0.10 \--service-node-port-range 1-65535 \--kube-proxy-arg proxy-mode=ipvs \--disable coredns \--disable servicelb \--disable traefik \--disable local-storage \--disable metrics-server \--datastore-endpoint="mysql://<USERNAME>:<PASSWORD>@tcp(<HOST>:3306)/<DATABASE>" \--token <TOKEN>
4.4 引导 Agent 节点加入
INSTALL_K3S_MIRROR=cn INSTALL_K3S_VERSION=v1.23.17+k3s1 ./install.sh server \--data-dir /data/k3s/var/lib/rancher/k3s \--datastore-endpoint="mysql://<USERNAME>:<PASSWORD>@tcp(<HOST>:3306)/<DATABASE>" \--token <TOKEN>
4.5 安装 Longhorn 依赖
# yum
tar -zxvf rpm.tar.gz
rpm -ivh ./rpm/*.rpm# apt
tar -zxvf deb.tar.gz
apt install ./deb/*.deb
4.4 导入镜像和安装应用
# 导出镜像
k3s ctr image import ./image.tar# coredns
helm install coredns coredns-1.26.0.tgz --namespace kube-system --values <VALUES_YAML_FILE>