Contents
k8s安装及基本配置
k8s基础环境配置
centos7
三个机器添加解析
cat >> /etc/hosts <<EOF
192.168.126.21 k8s-master01
192.168.126.21 k8s-worker01
192.168.126.21 k8s-worker02
EOF
docker和k8s基本安装
安装依赖
yum install -y conntrack ntpdate ntp ipvsadm ipset jq iptables curl sysstat libseccomp wget vim net-tools git
### 安装docker
yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
yum install -y docker-ce
#kubernetes 官方推荐 docker 等使用 systemd 作为 cgroupdriver，否则 kubelet 启动不了
cat > /etc/docker/daemon.json <<EOF
{
"exec-opts": ["native.cgroupdriver=systemd"],
"registry-mirrors": ["https://ud6340vz.mirror.aliyuncs.com"]
}
EOF
systemctl daemon-reload
systemctl restart docker
安装k8s
cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF
yum install -y kubeadm-1.26.8-0 kubectl-1.26.8-0 kubelet-1.26.8-0 --disableexcludes=kubernetes && systemctl enable kubelet
锁定kubelet的版本
#安装
yum install -y yum-plugin-versionlock
#锁定软件包
yum versionlock add kubeadm kubectl kubelet
#查看锁定列表
yum versionlock list
yum versionlock clear
k8s初始化
获取初始化的基本配置
kubeadm config print init-defaults > init-config.yaml
apiVersion: kubeadm.k8s.io/v1beta3
bootstrapTokens:
- groups:
- system:bootstrappers:kubeadm:default-node-token
token: abcdef.0123456789abcdef
ttl: 24h0m0s
usages:
- signing
- authentication
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 1.2.3.4
bindPort: 6443
nodeRegistration:
criSocket: unix:///var/run/containerd/containerd.sock
imagePullPolicy: IfNotPresent
name: node
taints: null
---
apiServer:
timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns: {}
etcd:
local:
dataDir: /var/lib/etcd
imageRepository: registry.k8s.io
kind: ClusterConfiguration
kubernetesVersion: 1.28.0
networking:
dnsDomain: cluster.local
serviceSubnet: 10.96.0.0/12
scheduler: {}
k8s初始化前准备
1.关闭防火墙
systemctl stop firewalld
systemctl disable firewalld
2.关闭selinux和内存交换（看内存是否充足）
swapoff -a && sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab
setenforce 0 && sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config
3.禁止交换内存报错
cat > /etc/sysconfig/kubelet <<EOF
KUBELET_EXTRA_ARGS="--fail-swap-on=false"
EOF
4.修改配置禁用CRI，防CRI报错
rm -f /etc/containerd/config.toml
systemctl restart containerd
5.为了让k8s能够检查和转发网络流量，需要修改iptables配置，启用br_netfilter
cat > /etc/modules-load.d/k8s.conf <<EOF
br_netfilter
EOF
cat > /etc/sysctl.d/k8s.conf <<EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward=1 # better than modify /etc/sysctl.conf
EOF
立即生效配置
sysctl --system
6.提前下载镜像，防止初始化出错
kubeadm config images pull --kubernetes-version v1.28.6 --image-repository registry.aliyuncs.com/google_containers
k8s在1.24版本后就不使用docker了，使用container来管理容器-n是指定namespace，docker也是使用container，它的默认namespace是moby
ctr namespaces ls 查看namespace
#需要手动下载这个镜像，默认从官方下载会卡死导致初始化出错
ctr -n k8s.io images pull -k registry.aliyuncs.com/google_containers/pause:3.6
ctr -n k8s.io images tag registry.aliyuncs.com/google_containers/pause:3.6 registry.k8s.io/pause:3.6
#重命名镜像registry.aliyuncs.com/google_containers/pause:3.6的tag为registry.k8s.io/pause:3.6
kubeadm reset -f
除了初始化，以上所有节点都要做
需要修改镜像地址，换成国内的
kubeadm init --apiserver-advertise-address=192.168.126.21 --kubernetes-version v1.28.6 --image-repository registry.aliyuncs.com/google_containers --service-cidr=10.96.0.0/12 --pod-network-cidr=10.244.0.0/16
To start using your cluster, you need to run the following as a regular user:
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
Alternatively, if you are the root user, you can run:
export KUBECONFIG=/etc/kubernetes/admin.conf
You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
https://kubernetes.io/docs/concepts/cluster-administration/addons/
Then you can join any number of worker nodes by running the following on each as root:
kubeadm join 192.168.126.21:6443 --token xub4u4.h3m8xxhdl2y0145g \
--discovery-token-ca-cert-hash sha256:5cb19e5ede74cf6681b3f174b4174a5b18f475568c9ee7f058ac00ddbd79f927
初始化出错恢复
kubeadm reset -f
从配置文件初始化的方法
kubeadm init --config=kubeadm-config.yaml
初始化会获取token用于工作节点加入，24h过期重新获取
kubeadm token create --print-join-command
根据提示信息进行配置，root用户简化操作
mkdir -p $HOME/.kube
cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
工作节点的加入
kubeadm join 192.168.126.21:6443 --token oawdil.zzcnf5p8exsovhtb --discovery-token-ca-cert-hash sha256:03940b9ebef2ecce7ea310a5096467da328224a3a544b08c340f81ef9dec9c84
节点无状态，需要配置网络服务
主节点运行
[root@k8s-master01 ~]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
k8s-master01 NotReady control-plane 20m v1.26.8
k8s-worker01 NotReady <none> 8m6s v1.26.8
```yml
配置主节点
wget https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
下载不下来就网页打开复制吧，或者运行
cat > kube-flannel.yml <<EOF
kind: Namespace
apiVersion: v1
metadata:
name: kube-flannel
labels:
k8s-app: flannel
pod-security.kubernetes.io/enforce: privileged
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
labels:
k8s-app: flannel
name: flannel
rules:
- apiGroups:
- ""
resources:
- pods
verbs:
- get
- apiGroups:
- ""
resources:
- nodes
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- nodes/status
verbs:
- patch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
labels:
k8s-app: flannel
name: flannel
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: flannel
subjects:
- kind: ServiceAccount
name: flannel
namespace: kube-flannel
---
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
k8s-app: flannel
name: flannel
namespace: kube-flannel
---
kind: ConfigMap
apiVersion: v1
metadata:
name: kube-flannel-cfg
namespace: kube-flannel
labels:
tier: node
k8s-app: flannel
app: flannel
data:
cni-conf.json: |
{
"name": "cbr0",
"cniVersion": "0.3.1",
"plugins": [
{
"type": "flannel",
"delegate": {
"hairpinMode": true,
"isDefaultGateway": true
}
},
{
"type": "portmap",
"capabilities": {
"portMappings": true
}
}
]
}
net-conf.json: |
{
"Network": "10.244.0.0/16",
"EnableNFTables": false,
"Backend": {
"Type": "vxlan"
}
}
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: kube-flannel-ds
namespace: kube-flannel
labels:
tier: node
app: flannel
k8s-app: flannel
spec:
selector:
matchLabels:
app: flannel
template:
metadata:
labels:
tier: node
app: flannel
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/os
operator: In
values:
- linux
hostNetwork: true
priorityClassName: system-node-critical
tolerations:
- operator: Exists
effect: NoSchedule
serviceAccountName: flannel
initContainers:
- name: install-cni-plugin
image: docker.io/flannel/flannel-cni-plugin:v1.4.1-flannel1
command:
- cp
args:
- -f
- /flannel
- /opt/cni/bin/flannel
volumeMounts:
- name: cni-plugin
mountPath: /opt/cni/bin
- name: install-cni
image: docker.io/flannel/flannel:v0.25.1
command:
- cp
args:
- -f
- /etc/kube-flannel/cni-conf.json
- /etc/cni/net.d/10-flannel.conflist
volumeMounts:
- name: cni
mountPath: /etc/cni/net.d
- name: flannel-cfg
mountPath: /etc/kube-flannel/
containers:
- name: kube-flannel
image: docker.io/flannel/flannel:v0.25.1
command:
- /opt/bin/flanneld
args:
- --ip-masq
- --kube-subnet-mgr
resources:
requests:
cpu: "100m"
memory: "50Mi"
securityContext:
privileged: false
capabilities:
add: ["NET_ADMIN", "NET_RAW"]
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: EVENT_QUEUE_DEPTH
value: "5000"
volumeMounts:
- name: run
mountPath: /run/flannel
- name: flannel-cfg
mountPath: /etc/kube-flannel/
- name: xtables-lock
mountPath: /run/xtables.lock
volumes:
- name: run
hostPath:
path: /run/flannel
- name: cni-plugin
hostPath:
path: /opt/cni/bin
- name: cni
hostPath:
path: /etc/cni/net.d
- name: flannel-cfg
configMap:
name: kube-flannel-cfg
- name: xtables-lock
hostPath:
path: /run/xtables.lock
type: FileOrCreate
EOF
kubectl apply -f kube-flannel.yml
等一会就好了ready
[root@k8s-master01 ~]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
k8s-master01 Ready control-plane 33m v1.26.8
k8s-worker01 Ready <none> 20m v1.26.8
查看节点cs状态
kubectl get cs
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME STATUS MESSAGE ERROR
controller-manager Healthy ok
etcd-0 Healthy ok
scheduler Healthy ok
命令自动补全
前提安装了bash-completion
yum install -y bash-completion
source <(kubectl completion bash)
将以上命令添加到你的~/.bashrc或者~/.bash_profile文件中：
镜像下载不下来
方法1：从其他库下载
必须设置
image：
imagePullPolicy: IfNotPresent
查看镜像
ctr -n k8s.io image ls
以下docker地址都是可以的
"https://hub.docker.com/",
"https://registry.docker-cn.com",
"https://ue05qxiu.mirror.aliyuncs.com",
"https://docker.mirrors.ustc.edu.cn/",
"https://hub-mirror.c.163.com/",
"https://reg-mirror.qiniu.com"
IMAGE是需要的镜像版本，SOURCE是加速镜像地址，每个节点都要下载镜像，除非固定节点
IMAGE=nginx:1.21
SOURCE=ue05qxiu.mirror.aliyuncs.com
ctr -n k8s.io images pull $SOURCE/library/$IMAGE
ctr -n k8s.io images tag $SOURCE/library/$IMAGE docker.io/library/$IMAGE
ctr -n k8s.io i rm $SOURCE/library/$IMAGE
方法2：直接将docker镜像导入k8s
IMAGE=
docker save -o $IMAGE $IMAGE
ctr -n=k8s.io images import $IMAGE
解决初始化报错
[preflight] Running pre-flight checks
[WARNING Swap]: swap is enabled; production deployments should disable swap unless testing the NodeSwap feature gate of the kubelet
[WARNING Service-Kubelet]: kubelet service is not enabled, please run 'systemctl enable kubelet.service'
error execution phase preflight: [preflight] Some fatal errors occurred:
[ERROR CRI]: container runtime is not running: output: time="2024-05-12T19:55:54+08:00" level=fatal msg="validate service connection: CRI v1 runtime API is not implemented for endpoint \"unix:///var/run/containerd/containerd.sock\": rpc error: code = Unimplemented desc = unknown service runtime.v1.RuntimeService"
, error: exit status 1
[preflight] If you know what you are doing, you can make a check non-fatal with `--ignore-preflight-errors=...`
To see the stack trace of this error execute with --v=5 or higher
cat > /etc/docker/daemon.json <<EOF
{
"exec-opts": ["native.cgroupdriver=systemd"],
"log-driver": "json-file",
"log-opts": {
"max-size": "100m"
},
"storage-driver": "overlay2"
}
EOF
内存足够的话就关闭swap
swapoff -a && sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab
setenforce 0 && sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config
否则
修改/etc/sysconfig/kubelet
添加如下配置项
cat > /etc/sysconfig/kubelet <<EOF
KUBELET_EXTRA_ARGS="--fail-swap-on=false"
EOF
解决CRI报错，注释该项或者删除该文件都行
vim /etc/containerd/config.toml
#disabled_plugins = ["cri"]
rm -f /etc/containerd/config.toml
systemctl restart containerd
参考
手把手教你在centos7安装k8s集群 https://blog.csdn.net/Honest_wolf_king/article/details/127842022
Kubernetes(k8s)环境部署（写的很细） https://blog.csdn.net/ZGL_cyy/article/details/124682630