Linux Cheatsheet 19 — Debugging

journalctl -xe                       # latest errors
dmesg | tail                         # kernel
tail -f /var/log/syslog
systemctl status <unit>

strace cmd
strace -p PID
strace -f -p PID                     # follow forks
strace -e network cmd
strace -e openat cmd
strace -c cmd                        # summary
strace -e %file cmd                  # file syscalls
strace -tt -T cmd                    # timestamps + duration

ltrace cmd                           # library calls
ltrace -p PID

lsof -p PID                          # all open files
lsof -i :80                          # who's on port 80
lsof | grep deleted                  # deleted files still held
lsof +D /var/log                     # by directory

cat /proc/PID/cmdline
cat /proc/PID/status
cat /proc/PID/environ                # env vars (NULs)
cat /proc/PID/limits
cat /proc/PID/io                     # I/O stats
ls -la /proc/PID/fd/                 # open file descriptors
cat /proc/PID/maps                   # memory map

cat /proc/PID/stack                  # kernel stack
pstack PID                           # user stack

gdb -p PID                           # attach
gdb cmd
(gdb) run arg1 arg2
(gdb) break main
(gdb) continue
(gdb) bt                             # backtrace
(gdb) info threads
(gdb) thread 2
(gdb) print var
(gdb) detach
(gdb) quit

ulimit -c unlimited

# Where dumps go (systemd-coredump)
coredumpctl list
coredumpctl info PID
coredumpctl gdb PID

# Or set:
sysctl -w kernel.core_pattern="/tmp/core-%e-%p"

dmesg -T | grep -i oom
journalctl -k --grep "killed process"

# Per-process OOM score
cat /proc/PID/oom_score

df -h
df -i                                # inodes
du -sh /* | sort -rh | head
ncdu /                               # interactive
lsof | grep deleted                  # holding deleted files

ss -tlnp                             # listening
ss -tnp                              # established
ss -tn state syn-sent
nc -zv host 443                      # port reachable?
mtr host                             # path
curl -v https://host/
tcpdump -i any host 1.2.3.4

dig +trace example.com
nslookup example.com 1.1.1.1
resolvectl query example.com
cat /etc/resolv.conf
systemd-resolve --statistics

top -1                               # per-cpu
mpstat -P ALL 1
pidstat 1
perf top -p PID

top -o %MEM
ps -eo pid,rss,comm --sort=-rss | head
slabtop                              # kernel slab
cat /proc/meminfo

iostat -xz 1
iotop -ao
biotop                               # eBPF

strace -c -p PID                     # 30s, then ctrl-c
perf record -F 99 -p PID -g -- sleep 30
perf report

ps aux | awk '$8=="D"'
cat /proc/PID/wchan                  # waiting where?
cat /proc/PID/stack

systemctl status myapp
journalctl -u myapp -xe
sudo -u myapp /opt/myapp/bin/start   # run manually

file core.PID
coredumpctl gdb PID

# Install: apt install bpfcc-tools
opensnoop                            # opens
execsnoop                            # execs
biotop                               # disk I/O by process
tcpconnect                           # outbound conns
tcplife                              # connection lifetimes
profile -F 99 -p PID 30              # CPU profile

# bpftrace
bpftrace -e 'syscall:sys_enter_openat { @[comm] = count(); }'

apt-file search /usr/bin/foo
dpkg -S /usr/bin/foo
rpm -qf /usr/bin/foo

nsenter -t PID -m -u -n -p           # enter all namespaces