>>
The Complete NGINX on Ubuntu Series: Part 10 – Logging, Monitoring, and Analytics
Ubuntu Tutorial Monitoring Web Server NGINX

The Complete NGINX on Ubuntu Series: Part 10 – Logging, Monitoring, and Analytics

Welcome to Part 10 of our comprehensive NGINX on Ubuntu series! We’ll set up advanced logging, monitoring systems, and analytics to gain deep insights into your web server’s performance and user behavior.

NGINX Logging Architecture

Effective logging provides visibility into server performance, user patterns, security events, and troubleshooting information essential for maintaining a robust web infrastructure.

graph TD
    A[NGINX Logging System] --> B[Access Logs]
    A --> C[Error Logs]
    A --> D[Custom Logs]
    A --> E[Performance Logs]
    
    B --> F[User RequestsResponse CodesUser Agents]
    C --> G[Server ErrorsConfiguration IssuesSecurity Events]
    D --> H[API LogsApplication LogsCache Logs]
    E --> I[Response TimesUpstream TimingCache Performance]
    
    J[Log Processing] --> K[Real-time Analysis]
    J --> L[Log Rotation]
    J --> M[Analytics Dashboard]
    
    style A fill:#e1f5fe
    style J fill:#e8f5e8
    style F fill:#fff3e0
    style G fill:#ffebee

Advanced Log Formats

# Configure advanced log formats in nginx.conf
sudo nano /etc/nginx/nginx.conf
# Add to http block in nginx.conf
http {
    # Enhanced log format with timing information
    log_format detailed '$remote_addr - $remote_user [$time_local] '
                       '"$request" $status $body_bytes_sent '
                       '"$http_referer" "$http_user_agent" '
                       'rt=$request_time uct="$upstream_connect_time" '
                       'uht="$upstream_header_time" urt="$upstream_response_time" '
                       'cs=$upstream_cache_status';
    
    # JSON log format for structured logging
    log_format json_combined escape=json
        '{'
            '"timestamp":"$time_iso8601",'
            '"remote_addr":"$remote_addr",'
            '"request":"$request",'
            '"status":$status,'
            '"body_bytes_sent":$body_bytes_sent,'
            '"request_time":$request_time,'
            '"http_referrer":"$http_referer",'
            '"http_user_agent":"$http_user_agent",'
            '"upstream_response_time":"$upstream_response_time",'
            '"upstream_cache_status":"$upstream_cache_status"'
        '}';
    
    # Security focused log format
    log_format security '$remote_addr - $remote_user [$time_local] '
                       '"$request" $status $body_bytes_sent '
                       '"$http_referer" "$http_user_agent" '
                       '"$http_x_forwarded_for" "$request_id"';
    
    # API specific log format
    log_format api_log '$remote_addr - [$time_local] '
                      '"$request" $status $body_bytes_sent '
                      'rt=$request_time api_key="$http_x_api_key" '
                      'endpoint="$uri" method="$request_method"';
}

Specialized Logging Configuration

# Create comprehensive logging virtual host
sudo nano /etc/nginx/sites-available/logging.example.com
server {
    listen 80;
    server_name logging.example.com;
    root /var/www/logging/public_html;
    
    # Main access log with detailed format
    access_log /var/log/nginx/logging-access.log detailed;
    error_log /var/log/nginx/logging-error.log info;
    
    # JSON structured logging for log aggregation
    access_log /var/log/nginx/logging-json.log json_combined;
    
    # Main application
    location / {
        try_files $uri $uri/ =404;
        
        # Log successful requests separately
        access_log /var/log/nginx/app-success.log detailed if=$status_2xx;
    }
    
    # API endpoints with specialized logging
    location /api/ {
        proxy_pass http://backend_api/;
        
        # API specific logging
        access_log /var/log/nginx/api.log api_log;
        error_log /var/log/nginx/api-error.log warn;
        
        # Log slow API responses
        access_log /var/log/nginx/api-slow.log detailed if=$slow_request;
        
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Request-ID $request_id;
    }
    
    # Admin area with security logging
    location /admin/ {
        auth_basic "Admin Area";
        auth_basic_user_file /etc/nginx/auth/.htpasswd;
        
        # Security focused logging
        access_log /var/log/nginx/admin-access.log security;
        error_log /var/log/nginx/admin-error.log warn;
        
        try_files $uri $uri/ =404;
    }
    
    # Static files with minimal logging
    location ~* \.(css|js|png|jpg|jpeg|gif|ico|svg)$ {
        expires 1y;
        add_header Cache-Control "public, immutable";
        
        # Minimal logging for static files
        access_log /var/log/nginx/static.log combined if=$log_static;
        error_log off;
    }
}

# Variables for conditional logging
map $status $status_2xx {
    ~^2 1;
    default 0;
}

map $request_time $slow_request {
    ~^[0-9]\. 0;
    default 1;
}

map $uri $log_static {
    ~*\.(css|js|png|jpg|jpeg|gif|ico|svg)$ 0;
    default 1;
}

Log Rotation Management

# Configure logrotate for NGINX
sudo nano /etc/logrotate.d/nginx
/var/log/nginx/*.log {
    daily
    missingok
    rotate 52
    compress
    delaycompress
    notifempty
    create 640 www-data adm
    sharedscripts
    
    postrotate
        invoke-rc.d nginx rotate >/dev/null 2>&1
    endscript
}

# High-volume logs with hourly rotation
/var/log/nginx/api*.log {
    hourly
    missingok
    rotate 168
    compress
    delaycompress
    notifempty
    create 640 www-data adm
    sharedscripts
    
    postrotate
        invoke-rc.d nginx rotate >/dev/null 2>&1
    endscript
}

# Security logs with extended retention
/var/log/nginx/*security*.log {
    weekly
    missingok
    rotate 104
    compress
    delaycompress
    notifempty
    create 640 www-data adm
    sharedscripts
    
    postrotate
        invoke-rc.d nginx rotate >/dev/null 2>&1
    endscript
}

Real-time Monitoring

# Create real-time log monitoring script
sudo nano /usr/local/bin/nginx-monitor.sh
#!/bin/bash

# NGINX Real-time Monitor
ACCESS_LOG="/var/log/nginx/access.log"
ERROR_LOG="/var/log/nginx/error.log"

show_stats() {
    echo "=== NGINX Real-time Statistics ==="
    echo "Generated: $(date)"
    echo
    
    # Current connections
    echo "--- Active Connections ---"
    local connections=$(ss -tuln | grep ':80\|:443' | wc -l)
    echo "Listening sockets: $connections"
    echo "Established: $(ss -tun | grep ESTAB | grep ':80\|:443' | wc -l)"
    echo
    
    # Requests per minute
    echo "--- Requests per Minute ---"
    local current_minute=$(date '+%d/%b/%Y:%H:%M')
    local rpm=$(grep "$current_minute" "$ACCESS_LOG" 2>/dev/null | wc -l)
    echo "Current minute: $rpm requests"
    echo
    
    # Status codes (last 100 requests)
    echo "--- Status Code Distribution ---"
    tail -100 "$ACCESS_LOG" 2>/dev/null | awk '{print $9}' | sort | uniq -c | sort -nr
    echo
    
    # Top IPs (last 100 requests)
    echo "--- Top Client IPs ---"
    tail -100 "$ACCESS_LOG" 2>/dev/null | awk '{print $1}' | sort | uniq -c | sort -nr | head -5
    echo
    
    # Recent errors
    echo "--- Recent Errors ---"
    tail -5 "$ERROR_LOG" 2>/dev/null || echo "No recent errors"
    echo
    
    # Response time stats (if available)
    if grep -q 'rt=' "$ACCESS_LOG" 2>/dev/null; then
        echo "--- Response Time Statistics ---"
        tail -100 "$ACCESS_LOG" | grep 'rt=' | awk -F'rt=' '{print $2}' | awk '{print $1}' | awk '
        {
            sum += $1; count++
            if ($1 > max) max = $1
            if (min == 0 || $1 < min) min = $1
        }
        END {
            if (count > 0) {
                printf "Avg: %.3fs, Min: %.3fs, Max: %.3fs\n", sum/count, min, max
            }
        }'
        echo
    fi
}

case "${1:-}" in
    --watch)
        while true; do
            clear
            show_stats
            echo "Press Ctrl+C to exit..."
            sleep 5
        done
        ;;
    *)
        show_stats
        ;;
esac

# Make executable: sudo chmod +x /usr/local/bin/nginx-monitor.sh

Log Analytics Engine

graph TD
    A[Log Analytics] --> B[Traffic Analysis]
    A --> C[Performance Analysis]
    A --> D[Security Analysis]
    A --> E[User Behavior]
    
    B --> F[Request VolumePopular PagesTraffic Patterns]
    C --> G[Response TimesError RatesCache Performance]
    D --> H[Attack AttemptsSuspicious IPsBot Traffic]
    E --> I[User JourneysSession AnalysisGeographic Data]
    
    style A fill:#e1f5fe
    style B fill:#e8f5e8
    style C fill:#fff3e0
    style D fill:#ffebee
    style E fill:#e3f2fd

# Create log analytics script
sudo nano /usr/local/bin/nginx-analytics.sh
#!/bin/bash

# NGINX Log Analytics
ACCESS_LOG="/var/log/nginx/access.log"
DATE_FILTER="${1:-$(date '+%d/%b/%Y')}"
ANALYTICS_DIR="/var/log/nginx/analytics"

mkdir -p "$ANALYTICS_DIR"

generate_traffic_report() {
    local report="$ANALYTICS_DIR/traffic-$(date '+%Y%m%d').txt"
    
    echo "=== Traffic Analytics Report ===" > "$report"
    echo "Date: $DATE_FILTER" >> "$report"
    echo "Generated: $(date)" >> "$report"
    echo >> "$report"
    
    # Total requests
    local total=$(grep "$DATE_FILTER" "$ACCESS_LOG" | wc -l)
    echo "Total Requests: $total" >> "$report"
    echo >> "$report"
    
    # Hourly distribution
    echo "--- Requests by Hour ---" >> "$report"
    grep "$DATE_FILTER" "$ACCESS_LOG" | awk '{print $4}' | cut -d: -f2 | sort | uniq -c | sort -k2 -n >> "$report"
    echo >> "$report"
    
    # Top pages
    echo "--- Top 20 Pages ---" >> "$report"
    grep "$DATE_FILTER" "$ACCESS_LOG" | awk '{print $7}' | sort | uniq -c | sort -nr | head -20 >> "$report"
    echo >> "$report"
    
    # Top IPs
    echo "--- Top 10 IPs ---" >> "$report"
    grep "$DATE_FILTER" "$ACCESS_LOG" | awk '{print $1}' | sort | uniq -c | sort -nr | head -10 >> "$report"
    echo >> "$report"
    
    # Status codes
    echo "--- Status Codes ---" >> "$report"
    grep "$DATE_FILTER" "$ACCESS_LOG" | awk '{print $9}' | sort | uniq -c | sort -nr >> "$report"
    echo >> "$report"
    
    # Bandwidth
    local bytes=$(grep "$DATE_FILTER" "$ACCESS_LOG" | awk '{sum += $10} END {print sum}')
    local mb=$((bytes / 1024 / 1024))
    echo "Total Bandwidth: ${mb} MB" >> "$report"
    
    echo "Traffic report: $report"
}

generate_performance_report() {
    local report="$ANALYTICS_DIR/performance-$(date '+%Y%m%d').txt"
    
    echo "=== Performance Analytics Report ===" > "$report"
    echo "Date: $DATE_FILTER" >> "$report"
    echo "Generated: $(date)" >> "$report"
    echo >> "$report"
    
    if grep -q 'rt=' "$ACCESS_LOG"; then
        echo "--- Response Time Statistics ---" >> "$report"
        grep "$DATE_FILTER" "$ACCESS_LOG" | grep 'rt=' | awk -F'rt=' '{print $2}' | awk '{print $1}' | awk '
        {
            sum += $1; count++
            if ($1 > max) max = $1
            if (min == 0 || $1 < min) min = $1
            if ($1 < 0.1) fast++
            else if ($1 < 0.5) medium++
            else if ($1 < 1.0) slow++
            else very_slow++
        }
        END {
            if (count > 0) {
                printf "Average: %.3f seconds\n", sum/count
                printf "Min: %.3f seconds\n", min
                printf "Max: %.3f seconds\n", max
                printf "Fast (<0.1s): %d (%.1f%%)\n", fast, (fast/count)*100
                printf "Medium (0.1-0.5s): %d (%.1f%%)\n", medium, (medium/count)*100
                printf "Slow (0.5-1.0s): %d (%.1f%%)\n", slow, (slow/count)*100
                printf "Very Slow (>1.0s): %d (%.1f%%)\n", very_slow, (very_slow/count)*100
            }
        }' >> "$report"
        echo >> "$report"
        
        echo "--- Slowest Pages ---" >> "$report"
        grep "$DATE_FILTER" "$ACCESS_LOG" | grep 'rt=' | awk -F'rt=' '{
            rt = $2; gsub(/ .*/, "", rt)
            url = $1; gsub(/.*"[A-Z]+ /, "", url); gsub(/ HTTP.*/, "", url)
            print rt " " url
        }' | sort -nr | head -10 >> "$report"
        echo >> "$report"
    fi
    
    # Cache performance
    if grep -q 'cs=' "$ACCESS_LOG"; then
        echo "--- Cache Performance ---" >> "$report"
        grep "$DATE_FILTER" "$ACCESS_LOG" | grep 'cs=' | awk -F'cs=' '{print $2}' | awk '{print $1}' | sort | uniq -c | sort -nr >> "$report"
        echo >> "$report"
    fi
    
    echo "Performance report: $report"
}

generate_security_report() {
    local report="$ANALYTICS_DIR/security-$(date '+%Y%m%d').txt"
    
    echo "=== Security Analytics Report ===" > "$report"
    echo "Date: $DATE_FILTER" >> "$report"
    echo "Generated: $(date)" >> "$report"
    echo >> "$report"
    
    # 4xx errors
    echo "--- 4xx Errors ---" >> "$report"
    grep "$DATE_FILTER" "$ACCESS_LOG" | grep ' 4[0-9][0-9] ' | awk '{print $9}' | sort | uniq -c | sort -nr >> "$report"
    echo >> "$report"
    
    # Top error IPs
    echo "--- Top Error IPs ---" >> "$report"
    grep "$DATE_FILTER" "$ACCESS_LOG" | grep ' 4[0-9][0-9] ' | awk '{print $1}' | sort | uniq -c | sort -nr | head -10 >> "$report"
    echo >> "$report"
    
    # Suspicious user agents
    echo "--- Suspicious User Agents ---" >> "$report"
    grep "$DATE_FILTER" "$ACCESS_LOG" | awk -F'"' '{print $6}' | grep -iE '(scanner|crawler|bot)' | grep -v -iE '(google|bing)' | sort | uniq -c | sort -nr | head -10 >> "$report"
    echo >> "$report"
    
    echo "Security report: $report"
}

# Execute based on argument
case "${2:-all}" in
    traffic)
        generate_traffic_report
        ;;
    performance)
        generate_performance_report
        ;;
    security)
        generate_security_report
        ;;
    all)
        generate_traffic_report
        generate_performance_report
        generate_security_report
        echo "All reports generated in: $ANALYTICS_DIR"
        ;;
    *)
        echo "Usage: $0 [date] [traffic|performance|security|all]"
        echo "Date format: DD/Mon/YYYY (default: today)"
        ;;
esac

# Make executable: sudo chmod +x /usr/local/bin/nginx-analytics.sh

Automated Monitoring Setup

# Create automated monitoring with cron
sudo crontab -e

# Add monitoring jobs
# Collect metrics every 5 minutes
*/5 * * * * /usr/local/bin/nginx-monitor.sh > /var/log/nginx/monitoring.log 2>&1

# Generate daily analytics reports
0 1 * * * /usr/local/bin/nginx-analytics.sh $(date -d "yesterday" '+%d/%b/%Y') all

# Weekly log cleanup
0 2 * * 0 find /var/log/nginx/analytics -name "*.txt" -mtime +30 -delete

Testing and Validation

# Create log directories
sudo mkdir -p /var/log/nginx/analytics

# Enable logging configuration
sudo ln -s /etc/nginx/sites-available/logging.example.com /etc/nginx/sites-enabled/

# Test configuration
sudo nginx -t

# Reload NGINX
sudo systemctl reload nginx

# Test real-time monitoring
/usr/local/bin/nginx-monitor.sh

# Test analytics
/usr/local/bin/nginx-analytics.sh

# Watch logs in real-time
sudo tail -f /var/log/nginx/access.log

# Test JSON logging
curl -H "Host: logging.example.com" http://localhost/
cat /var/log/nginx/logging-json.log | tail -1 | jq '.'

# Generate some test traffic
for i in {1..50}; do
    curl -H "Host: logging.example.com" http://localhost/ > /dev/null 2>&1
    sleep 0.1
done

# Run analytics on test data
/usr/local/bin/nginx-analytics.sh

What’s Next?

Excellent! You’ve implemented comprehensive logging, monitoring, and analytics for your NGINX server. You now have deep visibility into traffic patterns, performance metrics, and security events with automated reporting.

Coming up in Part 11: NGINX Rate Limiting and Traffic Control

References

This is Part 10 of our 22-part NGINX series. Your server now has enterprise-grade monitoring and analytics! Next, we’ll implement advanced traffic control. Questions? Share them in the comments!

Share

Written by:

373 Posts

View All Posts
Follow Me :

Related Posts

You May Have Missed

c#
Default Code stuffs Tweaks C#

Runtime Compile C# Code

c#
Tweaks C# Code stuffs

Lets Create a Null Checking Extension on C#

c#
Tweaks C#

How to encrypt and decrypt using cryptography (AES)?

Default Tweaks

How to auto-generate unique-identifier column in Sql?

How to whitelist website on AdBlocker?

How to whitelist website on AdBlocker?

  1. 1 Click on the AdBlock Plus icon on the top right corner of your browser
  2. 2 Click on "Enabled on this site" from the AdBlock Plus option
  3. 3 Refresh the page and start browsing the site