k8s-debug-service

#run a debug pod to troubleshoting from it
kubectl get all
kubectl get ev
kubectl get pv

kubectl run -it --rm --restart=Never busybox --image=busybox sh

#OR run a pod you have already
kubectl exec <POD-NAME> -c <CONTAINER-NAME> -- <COMMAND>

#SIMULATION
kubectl run hostnames --image=k8s.gcr.io/serve_hostname \
                        --labels=app=hostnames \
                        --port=9376 \
                        --replicas=3


#check pods are running
kubectl get pods -l app=hostnames

#check if exist a service associated to the pods
kubectl get svc hostnames

#lets create it
kubectl expose deployment hostnames --port=80 --target-port=9376

#checking dns resolution from inside a pod
nslookup hostnames
nslookup hostnames.default
nslookup hostnames.FQDN
#if some of the previous one fails will be required to check /etc/resolv.conf and application call configurations
nslookup kubernetes.default #kubernetes master resolution should always works, if it doesnt works, it could be a DNS system problem or kube-proxy problem

#check ports configuration and test with curl/wget to the endpoints:port within a pod container or the busybox container

#if it continues failing it is time to review kube-proxy
#first check if running
ps auxw | grep kube-proxy

#if it isnt running time to check logs
/var/log/kube-proxy.log
or using journalctl

#OR

#check if there are logs by DNS daemons subcomponents
#check by the beginning by E, W, F letters
kubectl logs --namespace=kube-system $(kubectl get pods --namespace=kube-system -l k8s-app=kube-dns -o name) -c kubedns
kubectl logs --namespace=kube-system $(kubectl get pods --namespace=kube-system -l k8s-app=kube-dns -o name) -c dnsmasq
kubectl logs --namespace=kube-system $(kubectl get pods --namespace=kube-system -l k8s-app=kube-dns -o name) -c sidecar

#check if the DNS k8s Service is UP
kubectl get svc --namespace=kube-system

#check if the DNS endpoints are exposed
kubectl get ep kube-dns --namespace=kube-system

#KUBE-PROXY IPTABLES DEBUG

#For old Userspace mode(which is deprecated) it should appear 2 rules for each port in your service KUBE-PORTALS-CONTAINER and KUBE-PORTALS-HOST
#For iptables mode it should appear a set of: 1 rule KUBE-SERVICES, 1/2 rule per endpoint KUBE-SVC-HASH (depends of SessionAfinitty) KUBE-SEP-(hash) chain per endpoint, and a few rules in each KUBE-SEP-(hash) chain


#run the iptables command to check
u@node$ iptables-save | grep hostnames #hostnames is the fiction/simulate Service in this debug guide