请稍等 ...

calico-node健康检查失败一直重启

[root@node-1 ~]# kubectl get pod -n kube-system
NAME READY STATUS RESTARTS AGE
calico-kube-controllers-6fcb5c5bcf-grhzj 0/1 ContainerCreating 0 8h
calico-node-lcqpd 0/1 CrashLoopBackOff 11 8h
calico-node-mnlt6 0/1 CrashLoopBackOff 11 8h

查看calico-node-mnlt6 pod事件，如下：
[root@node-1 ~]# kubectl describe pod calico-node-mnlt6 -n kube-system
Name: calico-node-mnlt6
Namespace: kube-system
Priority: 2000001000
Priority Class Name: system-node-critical
Node: node-3/192.168.216.103
Start Time: Tue, 03 May 2022 01:04:00 +0800
Labels: controller-revision-hash=855949f8d8
k8s-app=calico-node
pod-template-generation=1
Annotations:
Status: Running
IP: 192.168.216.103
IPs:
IP: 192.168.216.103
Controlled By: DaemonSet/calico-node
Init Containers:
upgrade-ipam:
Container ID: containerd://39d3134d6912a322ba6be6041429ded61ed6a2ff5a1a53b09e5fa17666607c2d
Image: docker.io/calico/cni:v3.22.2
Image ID: docker.io/calico/cni@sha256:757d06fe361c0083ee12dcd7d01a5ad98d6afd84bc92566e9cbd15ab2434e855
Port:
Host Port:
Command:
/opt/cni/bin/calico-ipam
-upgrade
State: Terminated
Reason: Completed
Exit Code: 0
Started: Tue, 03 May 2022 01:04:25 +0800
Finished: Tue, 03 May 2022 01:04:25 +0800
Ready: True
Restart Count: 0
Environment Variables from:
kubernetes-services-endpoint ConfigMap Optional: true
Environment:
KUBERNETES_NODE_NAME: (v1:spec.nodeName)
CALICO_NETWORKING_BACKEND: <set to the key ‘calico_backend’ of config map ‘calico-config’> Optional: false
Mounts:
/host/opt/cni/bin from cni-bin-dir (rw)
/var/lib/cni/networks from host-local-net-dir (rw)
/var/run/secrets/kubernetes.io/serviceaccount from calico-node-token-l6bqt (ro)
install-cni:
Container ID: containerd://6f250013291e4facbbfb587f86c2dee76bddf51b4fc51a493def058822acdc80
Image: docker.io/calico/cni:v3.22.2
Image ID: docker.io/calico/cni@sha256:757d06fe361c0083ee12dcd7d01a5ad98d6afd84bc92566e9cbd15ab2434e855
Port:
Host Port:
Command:
/opt/cni/bin/install
State: Terminated
Reason: Completed
Exit Code: 0
Started: Tue, 03 May 2022 01:04:25 +0800
Finished: Tue, 03 May 2022 01:04:29 +0800
Ready: True
Restart Count: 0
Environment Variables from:
kubernetes-services-endpoint ConfigMap Optional: true
Environment:
CNI_CONF_NAME: 10-calico.conflist
CNI_NETWORK_CONFIG: <set to the key ‘cni_network_config’ of config map ‘calico-config’> Optional: false
KUBERNETES_NODE_NAME: (v1:spec.nodeName)
CNI_MTU: <set to the key ‘veth_mtu’ of config map ‘calico-config’> Optional: false
SLEEP: false
Mounts:
/host/etc/cni/net.d from cni-net-dir (rw)
/host/opt/cni/bin from cni-bin-dir (rw)
/var/run/secrets/kubernetes.io/serviceaccount from calico-node-token-l6bqt (ro)
flexvol-driver:
Container ID: containerd://bceb712d5c18b2fb08e4f68eb4d8b7b10fcdb5b2d8c4f50faa736aa478af9a04
Image: docker.io/calico/pod2daemon-flexvol:v3.22.2
Image ID: docker.io/calico/pod2daemon-flexvol@sha256:413c5ebad6a5796a5571796b5ba2d2114bc0a318144db3070a631dd256a96d8a
Port:
Host Port:
State: Terminated
Reason: Completed
Exit Code: 0
Started: Tue, 03 May 2022 01:04:44 +0800
Finished: Tue, 03 May 2022 01:04:44 +0800
Ready: True
Restart Count: 0
Environment:
Mounts:
/host/driver from flexvol-driver-host (rw)
/var/run/secrets/kubernetes.io/serviceaccount from calico-node-token-l6bqt (ro)
Containers:
calico-node:
Container ID: containerd://e3ae462d1ec431ba69131cd991ac6fc6a74651d7c3ece60d888f3a048ba9aa84
Image: docker.io/calico/node:v3.22.2
Image ID: docker.io/calico/node@sha256:41aac6d0a440746f03d03a2912d92809681d07b6ce0ca4649241a1b95ff5e513
Port:
Host Port:
State: Waiting
Reason: CrashLoopBackOff
Last State: Terminated
Reason: Error
Exit Code: 137
Started: Tue, 03 May 2022 01:30:39 +0800
Finished: Tue, 03 May 2022 01:31:49 +0800
Ready: False
Restart Count: 11
Requests:
cpu: 250m
Liveness: exec [/bin/calico-node -felix-live -bird-live] delay=10s timeout=10s period=10s #success=1 #failure=6
Readiness: exec [/bin/calico-node -felix-ready -bird-ready] delay=0s timeout=10s period=10s #success=1 #failure=3
Environment Variables from:
kubernetes-services-endpoint ConfigMap Optional: true
Environment:
DATASTORE_TYPE: kubernetes
WAIT_FOR_DATASTORE: true
NODENAME: (v1:spec.nodeName)
CALICO_NETWORKING_BACKEND: <set to the key ‘calico_backend’ of config map ‘calico-config’> Optional: false
CLUSTER_TYPE: k8s,bgp
IP: (v1:status.hostIP)
CALICO_IPV4POOL_IPIP: Always
CALICO_IPV4POOL_VXLAN: Never
FELIX_IPINIPMTU: <set to the key ‘veth_mtu’ of config map ‘calico-config’> Optional: false
FELIX_VXLANMTU: <set to the key ‘veth_mtu’ of config map ‘calico-config’> Optional: false
FELIX_WIREGUARDMTU: <set to the key ‘veth_mtu’ of config map ‘calico-config’> Optional: false
CALICO_IPV4POOL_CIDR: 10.200.0.0/16
CALICO_DISABLE_FILE_LOGGING: true
FELIX_DEFAULTENDPOINTTOHOSTACTION: ACCEPT
FELIX_IPV6SUPPORT: false
FELIX_HEALTHENABLED: true
Mounts:
/host/etc/cni/net.d from cni-net-dir (rw)
/lib/modules from lib-modules (ro)
/run/xtables.lock from xtables-lock (rw)
/sys/fs/ from sysfs (rw)
/var/lib/calico from var-lib-calico (rw)
/var/log/calico/cni from cni-log-dir (ro)
/var/run/calico from var-run-calico (rw)
/var/run/nodeagent from policysync (rw)
/var/run/secrets/kubernetes.io/serviceaccount from calico-node-token-l6bqt (ro)
Conditions:
Type Status
Initialized True
Ready False
ContainersReady False
PodScheduled True
Volumes:
lib-modules:
Type: HostPath (bare host directory volume)
Path: /lib/modules
HostPathType:
var-run-calico:
Type: HostPath (bare host directory volume)
Path: /var/run/calico
HostPathType:
var-lib-calico:
Type: HostPath (bare host directory volume)
Path: /var/lib/calico
HostPathType:
xtables-lock:
Type: HostPath (bare host directory volume)
Path: /run/xtables.lock
HostPathType: FileOrCreate
sysfs:
Type: HostPath (bare host directory volume)
Path: /sys/fs/
HostPathType: DirectoryOrCreate
cni-bin-dir:
Type: HostPath (bare host directory volume)
Path: /opt/cni/bin
HostPathType:
cni-net-dir:
Type: HostPath (bare host directory volume)
Path: /etc/cni/net.d
HostPathType:
cni-log-dir:
Type: HostPath (bare host directory volume)
Path: /var/log/calico/cni
HostPathType:
host-local-net-dir:
Type: HostPath (bare host directory volume)
Path: /var/lib/cni/networks
HostPathType:
policysync:
Type: HostPath (bare host directory volume)
Path: /var/run/nodeagent
HostPathType: DirectoryOrCreate
flexvol-driver-host:
Type: HostPath (bare host directory volume)
Path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds
HostPathType: DirectoryOrCreate
calico-node-token-l6bqt:
Type: Secret (a volume populated by a Secret)
SecretName: calico-node-token-l6bqt
Optional: false
QoS Class: Burstable
Node-Selectors: kubernetes.io/os=linux
Tolerations: :NoSchedule op=Exists
:NoExecute op=Exists
CriticalAddonsOnly op=Exists
node.kubernetes.io/disk-pressure:NoSchedule op=Exists
node.kubernetes.io/memory-pressure:NoSchedule op=Exists
node.kubernetes.io/network-unavailable:NoSchedule op=Exists
node.kubernetes.io/not-ready:NoExecute op=Exists
node.kubernetes.io/pid-pressure:NoSchedule op=Exists
node.kubernetes.io/unreachable:NoExecute op=Exists
node.kubernetes.io/unschedulable:NoSchedule op=Exists
Events:
Type Reason Age From Message

Normal Scheduled 28m Normal Pulling 8h Normal Pulled 8h Normal Created 8h Normal Pulled 8h Normal Created 8h Normal Started 8h Normal Started 8h Normal Pulling 8h Normal Created 8h Normal Pulled 8h Normal Started 8h Normal Pulling 8h Normal Pulled 8h Normal Created 8h Normal Started 8h Warning Unhealthy Warning Unhealthy Warning Unhealthy Warning Unhealthy Normal Killing 8h Normal Created 8h Normal Pulled 8h Warning Unhealthy 8h Normal Started 8h Warning Unhealthy Warning BackOff Warning Unhealthy default-scheduler Successfully assigned kube-system/calico-node-mnlt6 to node-3
kubelet Pulling image "docker.io/calico/cni:v3.22.2"
kubelet Successfully pulled image “docker.io/calico/cni:v3.22.2” in 22.401012712s
kubelet Created container upgrade-ipam
kubelet Container image “docker.io/calico/cni:v3.22.2” already present on machine
kubelet Created container install-cni
kubelet Started container install-cni
kubelet Started container upgrade-ipam
kubelet Pulling image "docker.io/calico/pod2daemon-flexvol:v3.22.2"
kubelet Created container flexvol-driver
kubelet Successfully pulled image “docker.io/calico/pod2daemon-flexvol:v3.22.2” in 13.908326521s
kubelet Started container flexvol-driver
kubelet Pulling image "docker.io/calico/node:v3.22.2"
kubelet Successfully pulled image “docker.io/calico/node:v3.22.2” in 20.393578905s
kubelet Created container calico-node
kubelet Started container calico-node
8h (x3 over 8h) kubelet Liveness probe failed: calico/node is not ready: Felix is not live: Get “http://localhost:9099/liveness”: dial tcp [::1]:9099: connect: connection refused
8h (x5 over 8h) kubelet Readiness probe failed: calico/node is not ready: BIRD is not ready: Failed to stat() nodename file: stat /var/lib/calico/nodename: no such file or directory
8h (x2 over 8h) kubelet Liveness probe failed: calico/node is not ready: bird/confd is not live: exit status 1
8h (x5 over 8h) kubelet Liveness probe failed: calico/node is not ready: bird/confd is not live: exit status 1
kubelet Container calico-node failed liveness probe, will be restarted
kubelet Created container calico-node
kubelet Container image “docker.io/calico/node:v3.22.2” already present on machine
kubelet Readiness probe errored: rpc error: code = Unknown desc = failed to exec in container: container is in CONTAINER_EXITED state
kubelet Started container calico-node
8h (x26 over 8h) kubelet Readiness probe failed: calico/node is not ready: BIRD is not ready: Failed to stat() nodename file: stat /var/lib/calico/nodename: no such file or directory
8h (x43 over 8h) kubelet Back-off restarting failed container
8h (x42 over 8h) kubelet Liveness probe failed: calico/node is not ready: Felix is not live: Get “http://localhost:9099/liveness”: dial tcp [::1]:9099: connect: connection refused