Linux Cheatsheet 17 — Performance Tuning

top / htop / btop          # processes
vmstat 1                   # CPU/mem/IO/swap
iostat -xz 1               # disk I/O
mpstat -P ALL 1            # per-CPU
free -h                    # memory
sar 1                      # historical
pidstat 1                  # per-process
dstat                      # combined
glances                    # all-in-one

top -1                     # per-CPU
mpstat -P ALL 1
pidstat -u 1
cat /proc/cpuinfo
nproc
lscpu

# Per-process CPU
ps -eo pid,pcpu,comm --sort=-pcpu | head

free -h
free -m -s 1               # update every second
vmstat 1
cat /proc/meminfo
slabtop                    # kernel slab
ps -eo pid,rss,vsz,comm --sort=-rss | head

# OOM
dmesg | grep -i oom
journalctl -k --grep oom

iostat -xz 1
iotop
ioping /var/lib/data       # latency
biotop                     # eBPF
nfsiostat                  # NFS
fio --name=test --filename=test --size=1G --bs=4k --rw=randread

sar -n DEV 1
ip -s link show eth0
ifstat
nload
iftop                      # interactive
nethogs                    # per-process
ss -s                      # socket summary
tcpdump

# Top CPU
ps -eo pid,pcpu,comm --sort=-pcpu | head

# Top mem
ps -eo pid,rss,comm --sort=-rss | head

# Top I/O
iotop -ao                  # accumulated

# Top network (per process)
nethogs

# /etc/sysctl.d/perf.conf
net.core.somaxconn = 65535
net.core.netdev_max_backlog = 65535
net.ipv4.tcp_max_syn_backlog = 65535
net.ipv4.ip_local_port_range = 1024 65535
net.ipv4.tcp_fin_timeout = 15
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_keepalive_time = 60
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216
net.ipv4.tcp_rmem = 4096 87380 16777216
net.ipv4.tcp_wmem = 4096 65536 16777216
net.core.default_qdisc = fq
net.ipv4.tcp_congestion_control = bbr

sysctl -p

fs.file-max = 2097152
fs.inotify.max_user_watches = 524288
vm.swappiness = 10
vm.vfs_cache_pressure = 50
vm.dirty_ratio = 10
vm.dirty_background_ratio = 5

# /etc/security/limits.conf or limits.d/
* soft nofile 65536
* hard nofile 65536
* soft nproc 65535

cpufreq-info
cpupower frequency-info
cpupower frequency-set -g performance

cat /sys/block/sda/queue/scheduler
echo none > /sys/block/nvme0n1/queue/scheduler        # NVMe usually
echo mq-deadline > /sys/block/sda/queue/scheduler     # SATA

cat /sys/kernel/mm/transparent_hugepage/enabled
echo never > /sys/kernel/mm/transparent_hugepage/enabled

perf top
perf stat -p PID
perf record -p PID -- sleep 30
perf report

apt install bpfcc-tools bpftrace

# bcc
opensnoop
execsnoop
biotop
tcpconnect
tcptop
profile -p PID 30          # 30s CPU profile

# bpftrace
bpftrace -e 'tracepoint:syscalls:sys_enter_openat { @[comm] = count(); }'

perf record -F 99 -p PID -g -- sleep 30
perf script > out.perf
# https://github.com/brendangregg/FlameGraph
./stackcollapse-perf.pl out.perf | ./flamegraph.pl > flame.svg

strace -p PID -c            # syscall summary
strace -fc cmd              # forks counted
strace -e network curl ...

trace-cmd record -p function_graph -P PID
trace-cmd report

numastat
numactl --hardware
numactl --cpunodebind=0 --membind=0 cmd

applications: perf, eBPF, java/python profilers
languages:    pyperf, py-spy, async-profiler
syscalls:     strace, ltrace, bpftrace
kernel:       perf, ftrace, eBPF
devices:      iostat, biotop, blktrace
network:      tcpdump, ss, ip -s