Nginx Cheatsheet 07 — Logging and Monitoring

log_format main '$remote_addr - $remote_user [$time_local] "$request" '
                '$status $body_bytes_sent "$http_referer" '
                '"$http_user_agent" "$http_x_forwarded_for" '
                'rt=$request_time uct="$upstream_connect_time" '
                'uht="$upstream_header_time" urt="$upstream_response_time"';

access_log /var/log/nginx/access.log main;

log_format json escape=json '{'
    '"time":"$time_iso8601",'
    '"remote_addr":"$remote_addr",'
    '"method":"$request_method",'
    '"path":"$request_uri",'
    '"status":$status,'
    '"bytes":$body_bytes_sent,'
    '"referer":"$http_referer",'
    '"ua":"$http_user_agent",'
    '"xff":"$http_x_forwarded_for",'
    '"rt":$request_time,'
    '"urt":"$upstream_response_time",'
    '"request_id":"$request_id"'
'}';

access_log /var/log/nginx/access.log json;

map $request_uri $loggable {
    default 1;
    ~*/health 0;
    ~*/metrics 0;
}

access_log /var/log/nginx/access.log main if=$loggable;

map $status $errlog {
    ~^[45]  1;
    default 0;
}

access_log /var/log/nginx/errors.log main if=$errlog;

error_log /var/log/nginx/error.log warn;
# levels: debug info notice warn error crit alert emerg

server {
    server_name api.example.com;
    access_log /var/log/nginx/api.access.log json;
    error_log /var/log/nginx/api.error.log;
}

location = /nginx_status {
    stub_status;
    access_log off;
    allow 127.0.0.1;
    deny all;
}

Active connections: 291
server accepts handled requests
 16630948 16630948 31070465
Reading: 6 Writing: 179 Waiting: 106

docker run -d \
  -p 9113:9113 \
  nginx/nginx-prometheus-exporter:latest \
  -nginx.scrape-uri=http://nginx:80/nginx_status

set_real_ip_from 10.0.0.0/8;
set_real_ip_from 192.168.0.0/16;
real_ip_header X-Forwarded-For;
real_ip_recursive on;

awk '$NF > 1.0 {print}' /var/log/nginx/access.log
# Field positions vary by log_format

jq 'select(.rt > 1) | "\(.time) \(.path) \(.rt)"' access.log

# /etc/logrotate.d/nginx
/var/log/nginx/*.log {
    daily
    missingok
    rotate 14
    compress
    delaycompress
    notifempty
    create 0640 www-data adm
    sharedscripts
    postrotate
        [ -f /var/run/nginx.pid ] && kill -USR1 `cat /var/run/nginx.pid`
    endscript
}

access_log syslog:server=127.0.0.1:514,tag=nginx main;

load_module modules/ngx_otel_module.so;

http {
    otel_exporter { endpoint otel-collector:4317; }
    otel_service_name nginx;
    otel_trace on;
    
    server {
        location / {
            otel_trace_context inject;
            proxy_pass http://app;
        }
    }
}

split_clients $request_id $sampled {
    1% 1;
    *  0;
}

access_log /var/log/nginx/sample.log main if=$sampled;