Tuning Nginx | PHP | MySQL max concurrent connections

Step 1 – sysctl.conf

$ sudo vi /etc/sysctl.d/99-sysctl.conf

fs.nr_open = 20000500

# Turn on execshield
# 0 completely disables ExecShield and Address Space Layout Randomization
# 1 enables them ONLY if the application bits for these protections are set to “enable”
# 2 enables them by default, except if the application bits are set to “disable”
# 3 enables them always, whatever the application bits
#kernel.exec-shield = 2
kernel.randomize_va_space = 2

# Controls IP packet forwarding
net.ipv4.ip_forward = 0

# Controls source route verification
net.ipv4.conf.default.rp_filter = 1

# Do not accept source routing
net.ipv4.conf.default.accept_source_route = 0

# Ignoring broadcasts request
net.ipv4.icmp_echo_ignore_broadcasts = 1
net.ipv4.icmp_ignore_bogus_error_responses = 1

# Controls the System Request debugging functionality of the kernel
kernel.sysrq = 0

# Controls whether core dumps will append the PID to the core filename.
# Useful for debugging multi-threaded applications.
kernel.core_uses_pid = 1

# Controls the use of TCP syncookies
net.ipv4.tcp_syncookies = 1

# Disable netfilter on bridges.
#net.bridge.bridge-nf-call-ip6tables = 0
#net.bridge.bridge-nf-call-iptables = 0
#net.bridge.bridge-nf-call-arptables = 0

# Controls the default maxmimum size of a mesage queue
kernel.msgmnb = 65536

# Controls the maximum size of a message, in bytes
kernel.msgmax = 65536

# Controls the maximum shared segment size, in bytes
kernel.shmmax = 68719476736

# Controls the maximum number of shared memory segments, in pages
kernel.shmall = 4294967296


# See also http://www.nateware.com/linux-network-tuning-for-2013.html for
# an explanation about some of these parameters, and instructions for
# a few other tweaks outside this file.

# Protection from SYN flood attack.
net.ipv4.tcp_syncookies = 1

# See evil packets in your logs.
net.ipv4.conf.all.log_martians = 0

# Discourage Linux from swapping idle server processes to disk (default = 60)
vm.swappiness = 5

# Tweak how the flow of kernel messages is throttled.
#kernel.printk_ratelimit_burst = 10
#kernel.printk_ratelimit = 5

# --------------------------------------------------------------------
# The following allow the server to handle lots of connection requests
# --------------------------------------------------------------------

# Increase number of incoming connections that can queue up
# before dropping
net.core.somaxconn = 65535
#net.core.somaxconn = 9999999

# Handle SYN floods and large numbers of valid HTTPS connections
net.ipv4.tcp_max_syn_backlog = 30000

# Increase the length of the network device input queue
net.core.netdev_max_backlog = 20000

# Increase system file descriptor limit so we will (probably)
# never run out under lots of concurrent requests.
# (Per-process limit is set in /etc/security/limits.conf)
fs.file-max = 9999999

# Widen the port range used for outgoing connections
net.ipv4.ip_local_port_range = 500 65000

# If your servers talk UDP, also up these limits
net.ipv4.udp_rmem_min = 8192
net.ipv4.udp_wmem_min = 8192

# --------------------------------------------------------------------
# The following help the server efficiently pipe large amounts of data
# --------------------------------------------------------------------

# Disable source routing and redirects
net.ipv4.conf.all.send_redirects = 0
net.ipv4.conf.all.accept_redirects = 0
net.ipv4.conf.all.accept_source_route = 0

# Disable packet forwarding.
net.ipv4.ip_forward = 0
net.ipv6.conf.all.forwarding = 0

# Disable TCP slow start on idle connections
net.ipv4.tcp_slow_start_after_idle = 0

# Turn on the tcp_window_scaling
net.ipv4.tcp_window_scaling = 1

# Turn on the tcp_timestamps
net.ipv4.tcp_timestamps = 1

# Turn on the tcp_sack
net.ipv4.tcp_sack = 1

# Change Congestion Control (default: reno)
net.ipv4.tcp_congestion_control=htcp

# Increase Linux autotuning TCP buffer limits
# Set max to 16MB for 1GE and 32M (33554432) or 54M (56623104) for 10GE
# Don't set tcp_mem itself! Let the kernel scale it based on RAM.
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216
net.core.rmem_default = 16777216
net.core.wmem_default = 16777216
net.core.optmem_max = 40960
net.ipv4.tcp_rmem = 4096 87380 16777216
net.ipv4.tcp_wmem = 4096 87380 16777216


# --------------------------------------------------------------------
# The following allow the server to handle lots of connection churn
# --------------------------------------------------------------------

# Disconnect dead TCP connections after 1 minute
net.ipv4.tcp_keepalive_time = 60

# Wait a maximum of 5 * 2 = 10 seconds in the TIME_WAIT state after a FIN, to handle
# any remaining packets in the network.
net.netfilter.nf_conntrack_tcp_timeout_time_wait = 10

# How long to keep ESTABLISHED connections in conntrack table
# Should be higher than tcp_keepalive_time + tcp_keepalive_probes * tcp_keepalive_intvl )
net.netfilter.nf_conntrack_tcp_timeout_established = 300
net.netfilter.nf_conntrack_generic_timeout = 300

# Allow a high number of timewait sockets
net.ipv4.tcp_max_tw_buckets = 2000000

# Timeout broken connections faster (amount of time to wait for FIN)
net.ipv4.tcp_fin_timeout = 10

# Let the networking stack reuse TIME_WAIT connections when it thinks it's safe to do so
net.ipv4.tcp_tw_reuse = 1

# Determines the wait time between isAlive interval probes (reduce from 75 sec to 15)
net.ipv4.tcp_keepalive_intvl = 15

# Determines the number of probes before timing out (reduce from 9 sec to 5 sec)
net.ipv4.tcp_keepalive_probes = 5

net.ipv4.conf.lo.arp_ignore = 1
net.ipv4.conf.lo.arp_announce = 2

# -------------------------------------------------------------

Step 2 – limits.conf

$ sudo vi /etc/security/limits.conf

* soft nproc 20000500 
* hard nproc 20000500

* soft nofile 20000500 
* hard nofile 20000500

Step 3 – setup limit nginx

$ sudo vi /etc/nginx/nginx.conf 

user  nginx nginx;
worker_processes  auto;
pid        /var/run/nginx.pid;
worker_rlimit_nofile 999999;

events {
        worker_connections 8192; 
        multi_accept on;
        use epoll;
        accept_mutex_delay 100ms;
}

http {
        #include   /etc/nginx/naxsi_core.rules;
        include   /etc/nginx/mime.types;
        default_type  application/octet-stream;

        log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

        access_log /var/log/nginx/access.log main buffer=32k;
        error_log /var/log/nginx/error.log;

        client_max_body_size    128M;

        sendfile on;
        tcp_nopush on;
        tcp_nodelay on;
        keepalive_timeout 65;
        keepalive_requests 100000;
        aio        on;
        directio 512;
        types_hash_max_size 2048;

        fastcgi_send_timeout 14400s;
        fastcgi_read_timeout 14400s;

        server_tokens off;
        reset_timedout_connection on;

        add_header X-Frame-Options "SAMEORIGIN" always;
        add_header X-Content-Type-Options "nosniff" always;
        add_header X-Powered-By "AGINK";
        add_header rt-Fastcgi-Cache $upstream_cache_status;
        add_header X-XSS-Protection "1; mode=block";
        #add_header Content-Security-Policy "frame-ancestors 'self'";
        #add_header Content-Security-Policy "default-src 'self'";

        gzip  on;
        gzip_disable "msie6";
        gzip_http_version 1.1;
        gzip_vary on;
        gzip_comp_level 6;
        gzip_proxied any;
        gzip_buffers 16 8k;
        gzip_types
                application/atom+xml
                application/x-javascript
                application/javascript
                application/json
                application/rss+xml
                application/vnd.ms-fontobject
                application/x-font-opentype
                application/x-font-truetype
                application/x-font-ttf
                application/x-web-app-manifest+json
                application/xhtml+xml
                application/xml
                font/opentype
                font/otf
                font/eot
                image/svg+xml
                image/vnd.microsoft.icon
                image/x-icon
                text/css
                text/plain
                text/x-component
                text/xml
                # text/html
                text/javascript;

        # proxy cache
        proxy_set_header  Host               $host;
        proxy_set_header  X-Real-IP          $remote_addr;
        proxy_set_header  Remote-Addr        $remote_addr;
        proxy_set_header  X-Forwarded-Host   $host;
        proxy_set_header  X-Forwarded-Server $host;
        proxy_set_header  X-Forwarded-For    $proxy_add_x_forwarded_for;
        proxy_set_header  X-Forwarded-Proto  $scheme;
        #proxy_set_header  X-UA-Detect        $mobile;
        proxy_set_header  Accept-Encoding    "";
        proxy_set_header  Proxy              "";
        proxy_hide_header X-Pingback;
        proxy_hide_header Link;
        proxy_hide_header ETag;
        proxy_connect_timeout 5;
        proxy_send_timeout 14400;
        proxy_read_timeout 14400;
        proxy_cache_use_stale timeout invalid_header http_500 http_502 http_503 http_504;
        proxy_cache_lock on;
        proxy_cache_lock_timeout 5s;
        proxy_buffers 8 32k;
        proxy_buffer_size 64k;

        include conf.d/*;

        server {
                listen 8001;
                location /nginx_status {
                stub_status on;
                access_log   off;
                allow 127.0.0.1;
                deny all;
                }
        }
}

$ sudo vi /usr/lib/systemd/system/nginx.service 

[Service]
LimitNOFILE=2048000 # change it

$ sudo  systemctl --system daemon-reload
$ sudo  systemctl restart nginx

Step 4- setup limit php

$ sudo vi /etc/php-fpm.d/agink.id.conf

[9801-agink.id]
;--- unix socket ---
;listen = /var/run/agink.id.sock
;
;--- tcp socket --- 
listen = 127.0.0.1:9801
;
user = nginx 
group = nginx 
listen.owner = nginx 
listen.group = nginx 
listen.mode = 0660
request_slowlog_timeout = 5s
slowlog = /var/log/php-fpm/agink.id
listen.allowed_clients = 127.0.0.1
pm = ondemand
pm.max_children = 100
pm.process_idle_timeout = 10s
pm.max_requests = 200
;listen.backlog = -1
pm.status_path = /status
request_terminate_timeout = 1400s
rlimit_files = 131072
rlimit_core = unlimited
catch_workers_output = yes
env[HOSTNAME] = $HOSTNAME
env[TMP] = /tmp
env[TMPDIR] = /tmp
env[TEMP] = /tmp
;php_value[session.save_handler] = files
;php_value[session.save_path] = /var/lib/php/session
php_admin_value[error_log] = /var/log/php-fpm/error.log
php_admin_flag[log_errors] = on
php_admin_value[memory_limit] = -1
php_admin_flag[display_errors] = on
php_admin_flag[safe_mode] = on
php_admin_flag[expose_php] = off
php_admin_flag[allow_url_fopen] = on
php_admin_flag[register_globals] = off
php_admin_value[disable_functions] = phpinfo, system
php_admin_value[safe_mode_exec_dir] = php-required-executables-path
php_admin_value[safe_mode_allowed_env_vars] = PHP_
php_admin_value[session.cookie_httponly] = 1
php_admin_value[session.cookie_secure] = 1
php_admin_value[set_time_limit] = 0
php_admin_value[max_execution_time] = 14400 
php_admin_value[default_socket_timeout] = 14400 
security.limit_extensions = .php .php3 .php4 .php5 .html .htm .css

[9802-agink.id]
;--- unix socket ---
;listen = /var/run/agink.id.sock
;
;--- tcp socket --- 
listen = 127.0.0.1:9802
;
user = nginx 
group = nginx 
listen.owner = nginx 
listen.group = nginx 
listen.mode = 0660
request_slowlog_timeout = 5s
slowlog = /var/log/php-fpm/agink.id
listen.allowed_clients = 127.0.0.1
pm = ondemand
pm.max_children = 100
pm.process_idle_timeout = 10s
pm.max_requests = 200
;listen.backlog = -1
pm.status_path = /status
request_terminate_timeout = 1400s
rlimit_files = 131072
rlimit_core = unlimited
catch_workers_output = yes
env[HOSTNAME] = $HOSTNAME
env[TMP] = /tmp
env[TMPDIR] = /tmp
env[TEMP] = /tmp
;php_value[session.save_handler] = files
;php_value[session.save_path] = /var/lib/php/session
php_admin_value[error_log] = /var/log/php-fpm/error.log
php_admin_flag[log_errors] = on
php_admin_value[memory_limit] = -1
php_admin_flag[display_errors] = on
php_admin_flag[safe_mode] = on
php_admin_flag[expose_php] = off
php_admin_flag[allow_url_fopen] = on
php_admin_flag[register_globals] = off
php_admin_value[disable_functions] = phpinfo, system
php_admin_value[safe_mode_exec_dir] = php-required-executables-path
php_admin_value[safe_mode_allowed_env_vars] = PHP_
php_admin_value[session.cookie_httponly] = 1
php_admin_value[session.cookie_secure] = 1
php_admin_value[set_time_limit] = 0
php_admin_value[max_execution_time] = 14400 
php_admin_value[default_socket_timeout] = 14400 
security.limit_extensions = .php .php3 .php4 .php5 .html .htm .css

[9803-agink.id]
;--- unix socket ---
;listen = /var/run/agink.id.sock
;
;--- tcp socket --- 
listen = 127.0.0.1:9803
;
user = nginx 
group = nginx 
listen.owner = nginx 
listen.group = nginx 
listen.mode = 0660
request_slowlog_timeout = 5s
slowlog = /var/log/php-fpm/agink.id
listen.allowed_clients = 127.0.0.1
pm = ondemand
pm.max_children = 100
pm.process_idle_timeout = 10s
pm.max_requests = 200
;listen.backlog = -1
pm.status_path = /status
request_terminate_timeout = 1400s
rlimit_files = 131072
rlimit_core = unlimited
catch_workers_output = yes
env[HOSTNAME] = $HOSTNAME
env[TMP] = /tmp
env[TMPDIR] = /tmp
env[TEMP] = /tmp
;php_value[session.save_handler] = files
;php_value[session.save_path] = /var/lib/php/session
php_admin_value[error_log] = /var/log/php-fpm/error.log
php_admin_flag[log_errors] = on
php_admin_value[memory_limit] = -1
php_admin_flag[display_errors] = on
php_admin_flag[safe_mode] = on
php_admin_flag[expose_php] = off
php_admin_flag[allow_url_fopen] = on
php_admin_flag[register_globals] = off
php_admin_value[disable_functions] = phpinfo, system
php_admin_value[safe_mode_exec_dir] = php-required-executables-path
php_admin_value[safe_mode_allowed_env_vars] = PHP_
php_admin_value[session.cookie_httponly] = 1
php_admin_value[session.cookie_secure] = 1
php_admin_value[set_time_limit] = 0
php_admin_value[max_execution_time] = 14400 
php_admin_value[default_socket_timeout] = 14400 
security.limit_extensions = .php .php3 .php4 .php5 .html .htm .css

[9804-agink.id]
;--- unix socket ---
;listen = /var/run/agink.id.sock
;
;--- tcp socket --- 
listen = 127.0.0.1:9804
;
user = nginx 
group = nginx 
listen.owner = nginx 
listen.group = nginx 
listen.mode = 0660
request_slowlog_timeout = 5s
slowlog = /var/log/php-fpm/agink.id
listen.allowed_clients = 127.0.0.1
pm = ondemand
pm.max_children = 100
pm.process_idle_timeout = 10s
pm.max_requests = 200
;listen.backlog = -1
pm.status_path = /status
request_terminate_timeout = 1400s
rlimit_files = 131072
rlimit_core = unlimited
catch_workers_output = yes
env[HOSTNAME] = $HOSTNAME
env[TMP] = /tmp
env[TMPDIR] = /tmp
env[TEMP] = /tmp
;php_value[session.save_handler] = files
;php_value[session.save_path] = /var/lib/php/session
php_admin_value[error_log] = /var/log/php-fpm/error.log
php_admin_flag[log_errors] = on
php_admin_value[memory_limit] = -1
php_admin_flag[display_errors] = on
php_admin_flag[safe_mode] = on
php_admin_flag[expose_php] = off
php_admin_flag[allow_url_fopen] = on
php_admin_flag[register_globals] = off
php_admin_value[disable_functions] = phpinfo, system
php_admin_value[safe_mode_exec_dir] = php-required-executables-path
php_admin_value[safe_mode_allowed_env_vars] = PHP_
php_admin_value[session.cookie_httponly] = 1
php_admin_value[session.cookie_secure] = 1
php_admin_value[set_time_limit] = 0
php_admin_value[max_execution_time] = 14400 
php_admin_value[default_socket_timeout] = 14400 
security.limit_extensions = .php .php3 .php4 .php5 .html .htm .css


$ sudo vi /usr/lib/systemd/system/php-fpm.service 
[Service]
LimitNOFILE=2048000 # change it

$ sudo  systemctl --system daemon-reload
$ sudo systemctl restart php-fpm

Step 5 – setup limit mysql

$ sudo vi /etc/percona-server.conf.d/mysqld.cnf 

# Percona Server template configuration

[mysqld]
#
# Remove leading # and set to the amount of RAM for the most important data
# cache in MySQL. Start at 70% of total RAM for dedicated server, else 10%.
# innodb_buffer_pool_size = 128M
#
# Remove leading # to turn on a very important data integrity option: logging
# changes to the binary log between backups.
# log_bin
#
# Remove leading # to set options mainly useful for reporting servers.
# The server defaults are faster for transactions and fast SELECTs.
# Adjust sizes as needed, experiment to find the optimal values.
# join_buffer_size = 128M
# sort_buffer_size = 2M
# read_rnd_buffer_size = 2M
datadir=/var/lib/mysql
socket=/var/lib/mysql/mysql.sock

# Disabling symbolic-links is recommended to prevent assorted security risks
symbolic-links=0

log-error=/var/log/mysqld.log
pid-file=/var/run/mysqld/mysqld.pid

#----- TUNNING ----#

collation-server = utf8mb4_unicode_ci
character-set-server = utf8mb4

#--- tunning ---#
bind-address=127.0.0.1
innodb_flush_log_at_timeout = 3
innodb_read_io_threads = 32
innodb_write_io_threads = 16
innodb_buffer_pool_instances = 2
innodb_io_capacity = 5000
innodb_io_capacity_max = 10000
innodb_buffer_pool_size = 1908M
innodb_buffer_pool_instances = 2
join_buffer_size = 244M
tmp_table_size = 122M
max_heap_table_size = 122M

#allow_persistent = Off
open_files_limit=9999999
thread_pool_size=36
thread_pool_max_threads=9999999
max_allowed_packet=16777216
max_heap_table_size=102400
max_connections=100000
max_user_connections=100000
max_connect_errors=9999999
wait_timeout=1200
interactive_timeout=1200
tmp_table_size=102400
#query_cache_size=0
#query_cache_type=0
#query_cache_limit=102400

#sql-mode = ""
#sql-mode=ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION,NO_EN
GINE_SUBSTITUTION,ERROR_FOR_DIVISION_BY_ZERO,ALLOW_INVALID_DATES
sql_mode=STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION,STRICT_ALL_TABLES
default_authentication_plugin=mysql_native_password

$ sudo vi /usr/lib/systemd/system/mysql.service

# Sets open_files_limit
LimitNOFILE = 1024000 # change it

$ sudo systemctl --system daemon-reload
$ sudo systemctl restart mysql