Kubernetes Troubleshooting

Pod not running?
  └── kubectl get pod → check STATUS + RESTARTS
  └── kubectl describe pod → Events section (bottom)
  └── kubectl logs → application error?
  └── kubectl exec → can you reach the pod shell?

Service not reachable?
  └── kubectl get endpoints → any IPs listed?
  └── kubectl describe svc → selector matches pod labels?
  └── kubectl exec into pod → curl the ClusterIP directly

DNS not resolving?
  └── kubectl exec debug-pod -- nslookup svc-name
  └── kubectl get pods -n kube-system → coredns running?

# current and previous logs
kubectl logs my-pod
kubectl logs my-pod --previous
kubectl logs my-pod -c init-container

# full event history
kubectl describe pod my-pod

# get exit code of last container run
kubectl get pod my-pod -o jsonpath='{.status.containerStatuses[0].lastState.terminated.exitCode}'

# live events across the namespace
kubectl get events --sort-by='.lastTimestamp'
kubectl get events --field-selector reason=BackOff
kubectl get events -A --sort-by='.lastTimestamp' | tail -30

kubectl describe pod my-pod | grep -A3 "Last State"
# Reason: OOMKilled  → increase memory limit

kubectl logs my-pod --previous          # logs from the crashed instance
kubectl describe pod my-pod             # look at Exit Code, Signal

kubectl describe pod my-pod | grep -A5 "Events"
# message: "Failed to pull image ... 404 Not Found"

# force delete (loses graceful shutdown)
kubectl delete pod my-pod --grace-period=0 --force

# if still stuck — remove finalizers
kubectl patch pod my-pod -p '{"metadata":{"finalizers":null}}'

# are there any backing pods?
kubectl get endpoints api

# check the selector
kubectl describe svc api | grep Selector
kubectl get pods -l app=api             # do any pods match?

# test connectivity from inside cluster
kubectl run curl --image=curlimages/curl --rm -it --restart=Never \
  -- curl -v http://api.default.svc.cluster.local/health

# test by Pod IP directly (bypasses Service)
kubectl get pods -o wide
kubectl run curl --image=curlimages/curl --rm -it --restart=Never \
  -- curl -v http://<pod-ip>:8080/health

# check CoreDNS is healthy
kubectl get pods -n kube-system -l k8s-app=kube-dns

# run DNS lookup from inside cluster
kubectl run dnsutils --image=registry.k8s.io/e2e-test-images/jessie-dnsutils:1.3 \
  --rm -it --restart=Never -- bash

# inside the pod:
nslookup api                                   # short name (same namespace)
nslookup api.production                        # cross-namespace
nslookup api.production.svc.cluster.local      # FQDN
cat /etc/resolv.conf                           # search domains
nslookup google.com                            # external DNS working?

kubectl get nodes                       # check STATUS (Ready / NotReady)
kubectl describe node my-node           # conditions, events, resource pressure
kubectl get events --field-selector involvedObject.name=my-node

# node pressure conditions
# MemoryPressure, DiskPressure, PIDPressure, NetworkUnavailable

# ssh to node and check kubelet
journalctl -u kubelet -n 100 --no-pager

kubectl top nodes                        # CPU/memory per node
kubectl top pods --sort-by=memory       # hungriest pods
kubectl top pods --containers            # per-container breakdown

# find pods with no resource requests (dangerous)
kubectl get pods -A -o json | jq \
  '.items[] | select(.spec.containers[].resources.requests == null) | .metadata.name'

# check which network policies apply to a pod
kubectl get networkpolicies -A
kubectl describe networkpolicy my-policy

# test with a raw curl pod (if blocked, connection times out or is refused)
kubectl run test --image=curlimages/curl --rm -it --restart=Never \
  -- curl -v --max-time 5 http://api.production.svc.cluster.local

# all non-running pods
kubectl get pods -A --field-selector=status.phase!=Running

# pods on a specific node
kubectl get pods -A --field-selector spec.nodeName=my-node

# pods with high restart count
kubectl get pods -A --sort-by='.status.containerStatuses[0].restartCount'

# decode a secret value
kubectl get secret my-secret -o jsonpath='{.data.password}' | base64 -d

# dump all resources in a namespace to YAML
kubectl get all -n production -o yaml > backup.yaml

# check RBAC — can a user do something?
kubectl auth can-i create pods --as=jane
kubectl auth can-i '*' '*' --as=system:serviceaccount:production:my-sa

# which admission webhooks are installed?
kubectl get validatingwebhookconfigurations
kubectl get mutatingwebhookconfigurations

kubectl debug -it my-pod --image=nicolaka/netshoot --target=app
# or
kubectl debug -it my-pod --image=busybox --copy-to=debug-pod

# follow logs for all pods matching a label
kubectl logs -l app=api -f --all-containers --max-log-requests=10

# use stern for multi-pod tailing (krew plugin)
kubectl stern api                         # matches pod names containing "api"
kubectl stern -l app=api --since=15m