部署

1
2
3
4
# 查看cpu个数
grep ^processor /proc/cpuinfo | wc -l
lshw -short -class cpu
cat /proc/cpuinfo | grep "model name"

nofile ulimit

/etc/security/limits.conf

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
# 查看系统资源限制
ulimit -a
ulimit -n
ulimit -Sn
ulimit -Hn
# ulimit -HSn 102400# vim /etc/security/limits.conf  //加入以下配置,重启即可生效
* soft nofile 65535
* hard nofile 65535

systemd

1
2
3
4
5
6
# systemd
[Service]
#LimitNOFILE=65536
LimitCORE=infinity
LimitNOFILE=512000
LimitNPROC=512000
1
2
3
4
5
cat /etc/systemd/system.conf |grep -E 'NOFILE|NPROC'
## DefaultLimitNOFILE=1024000
## DefaultLimitNPROC=1024000
## 查看生效
cat /proc/[PID]/limits

fs.file-max

1
2
3
4
5
6
7
sysctl -w net.ipv4.tcp_slow_start_after_idle=0
sysctl -w fs.file-max=6553560
# 查看分配状态,一共三个值
# 第一个代表全局已经分配的文件描述符数量
# 第二个代表自由的文件描述符(待重新分配的)
# 第三个代表总的文件描述符的数量
cat /proc/sys/fs/file-nr

sysctl.conf

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
fs.file-max = 6553560
net.ipv4.tcp_slow_start_after_idle = 0
net.core.default_qdisc = fq
net.ipv4.tcp_congestion_control = bbr
net.ipv4.tcp_fastopen = 3
########################################
vm.max_map_count = 262144
net.ipv4.tcp_max_tw_buckets = 6000
# timewait 的数量,默认是180000。
net.ipv4.ip_local_port_range = 1024 65000
# 允许系统打开的端口范围。
net.ipv4.tcp_tw_recycle = 0
# 关闭timewait 快速回收。
net.ipv4.tcp_tw_reuse = 1
# 开启重用。允许将TIME-WAIT sockets 重新用于新的TCP 连接。
net.ipv4.tcp_syncookies = 1
# 开启SYN Cookies,当出现SYN 等待队列溢出时,启用cookies 来处理。
net.core.somaxconn = 262144
# web 应用中listen 函数的backlog 默认会给我们内核参数的net.core.somaxconn 限制到128,而nginx 定义的NGX_LISTEN_BACKLOG 默认为511,所以有必要调整这个值。
net.core.netdev_max_backlog = 262144
# 每个网络接口接收数据包的速率比内核处理这些包的速率快时,允许送到队列的数据包的最大数目。
net.ipv4.tcp_max_orphans = 262144
# 系统中最多有多少个TCP 套接字不被关联到任何一个用户文件句柄上。如果超过这个数字,孤儿连接将即刻被复位并打印出警告信息。这个限制仅仅是为了防止简单的DoS 攻击,不能过分依靠它或者人为地减小这个值,更应该增加这个值(如果增加了内存之后)。
net.ipv4.tcp_max_syn_backlog = 262144
# 记录的那些尚未收到客户端确认信息的连接请求的最大值。对于有128M 内存的系统而言,缺省值是1024,小内存的系统则是128。
net.ipv4.tcp_timestamps = 0
# 时间戳可以避免序列号的卷绕。一个1Gbps 的链路肯定会遇到以前用过的序列号。时间戳能够让内核接受这种“异常”的数据包。这里需要将其关掉。
net.ipv4.tcp_synack_retries = 1
# 为了打开对端的连接,内核需要发送一个SYN 并附带一个回应前面一个SYN 的ACK。也就是所谓三次握手中的第二次握手。这个设置决定了内核放弃连接之前发送SYN+ACK 包的数量。
net.ipv4.tcp_syn_retries = 1
# 在内核放弃建立连接之前发送SYN 包的数量。
net.ipv4.tcp_fin_timeout = 1
# 如果套接字由本端要求关闭,这个参数决定了它保持在FIN-WAIT-2 状态的时间。对端可以出错并永远不关闭连接,甚至意外当机。缺省值是60 秒。2.2 内核的通常值是180 秒,3你可以按这个设置,但要记住的是,即使你的机器是一个轻载的WEB 服务器,也有因为大量的死套接字而内存溢出的风险,FIN- WAIT-2 的危险性比FIN-WAIT-1 要小,因为它最多只能吃掉1.5K 内存,但是它们的生存期长些。
net.ipv4.tcp_keepalive_time = 30
# 当keepalive 起用的时候,TCP 发送keepalive 消息的频度。缺省是2 小时。

build

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
sed -i.bak 's/CFLAGS="$CFLAGS -g"/#CFLAGS="$CFLAGS -g"/g' auto/cc/gcc
## gcc编译参数
--with-cc-opt='-O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector --param=ssp-buffer-size=4 -m64 -mtune=generic'
--with-cc-opt='-O2 -g -pipe -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector --param=ssp-buffer-size=4 -m64 -mtune=generic'
--with-ld-opt='-lrt -ljemalloc -Wl,-z,relro'
--with-ld-opt=-Wl,-E
--with-cc-opt='-m64 -mtune=native -g -O2 -fstack-protector --param=ssp-buffer-size=4 -Wformat -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -Wno-sign-compare -Wno-string-plus-int -Wno-deprecated-declarations -Wno-unused-parameter -Wno-unused-const-variable -Wno-conditional-uninitialized -Wno-mismatched-tags -Wno-c++11-extensions -Wno-sometimes-uninitialized -Wno-parentheses-equality -Wno-tautological-compare -Wno-self-assign -Wno-deprecated-register -Wno-deprecated -Wno-invalid-source-encoding -Wno-pointer-sign -Wno-parentheses -Wno-enum-conversion'
## configure example
/bin # nginx -V
nginx version: nginx/1.15.2
built by gcc 6.4.0 (Alpine 6.4.0) 
built with OpenSSL 1.0.2o  27 Mar 2018
TLS SNI support enabled
configure arguments: --prefix=/etc/nginx --sbin-path=/usr/sbin/nginx --modules-path=/usr/lib/nginx/modules --conf-path=/etc/nginx/nginx.conf --error-log-path=/var/log/nginx/error.log --http-log-path=/var/log/nginx/access.log --pid-path=/var/run/nginx.pid --lock-path=/var/run/nginx.lock --http-client-body-temp-path=/var/cache/nginx/client_temp --http-proxy-temp-path=/var/cache/nginx/proxy_temp --http-fastcgi-temp-path=/var/cache/nginx/fastcgi_temp --http-uwsgi-temp-path=/var/cache/nginx/uwsgi_temp --http-scgi-temp-path=/var/cache/nginx/scgi_temp --user=nginx --group=nginx --with-http_ssl_module --with-http_realip_module --with-http_addition_module --with-http_sub_module --with-http_dav_module --with-http_flv_module --with-http_mp4_module --with-http_gunzip_module --with-http_gzip_static_module --with-http_random_index_module --with-http_secure_link_module --with-http_stub_status_module --with-http_auth_request_module --with-http_xslt_module=dynamic --with-http_image_filter_module=dynamic --with-http_geoip_module=dynamic --with-threads --with-stream --with-stream_ssl_module --with-stream_ssl_preread_module --with-stream_realip_module --with-stream_geoip_module=dynamic --with-http_slice_module --with-mail --with-mail_ssl_module --with-compat --with-file-aio --with-http_v2_module

nginx.conf

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
# One worker per CPU-core.
worker_processes  4;
events {
  worker_connections  8096;
  multi_accept        on;
  use                 epoll;
}
worker_rlimit_nofile 40000;
pcre_jit on;

# Gzip Settings
gzip on;
gzip_disable "msie6";
gzip_vary on;
gzip_proxied any;
gzip_comp_level 6;
gzip_buffers 32 16k;
gzip_http_version 1.1;
gzip_min_length 250;
gzip_types image/jpeg image/bmp image/svg+xml text/plain text/css application/json application/javascript application/x-javascript text/xml application/xml application/xml+rss text/javascript image/x-icon;
# Brotli Settings
brotli on;
brotli_comp_level 4;
brotli_buffers 32 8k;
brotli_min_length 100;
brotli_static on;
brotli_types image/jpeg image/bmp image/svg+xml text/plain text/css application/json application/javascript application/x-javascript text/xml application/xml application/xml+rss text/javascript image/x-icon;


stream {
 server {
  listen 30094;
  proxy_connect_timeout 1s;
  proxy_timeout 5s;
  proxy_pass 192.168.33.27:30094;
 }
}

http {
  server_tokens off;
  sendfile           on;
  tcp_nopush         on;
  tcp_nodelay        on;
  keepalive_timeout  15;
  keepalive_requests 1000;
  open_file_cache max=200000 inactive=20s;
  open_file_cache_valid 30s;
  open_file_cache_min_uses 2;
  open_file_cache_errors on;
  server {
    # security headers
    add_header X-Frame-Options "SAMEORIGIN" always;
    add_header X-XSS-Protection "1; mode=block" always;
    add_header X-Content-Type-Options "nosniff" always;
    add_header Referrer-Policy "no-referrer-when-downgrade" always;
    add_header Content-Security-Policy "default-src * data: 'unsafe-eval' 'unsafe-inline'" always;
    add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" always;
    #add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload";
    #add_header X-Frame-Options DENY;
    #add_header X-Content-Type-Options nosniff;
    #add_header X-XSS-Protection "1; mode=block";
    add_header Set-Cookie "HttpOnly";
    add_header Set-Cookie "Secure";
    send_timeout 10;
    #client_max_body_size 10m;
    #client_body_buffer_size 128k;
    client_body_timeout 12;
    client_header_timeout 12;		
    client_body_buffer_size 10K;
    client_header_buffer_size 1k;
    client_max_body_size 8m;
    large_client_header_buffers 2 1k;
    proxy_redirect off;
    proxy_http_version 1.1;
    proxy_set_header Connection "";
    proxy_set_header Host $host;
    proxy_set_header X-Real-IP $remote_addr;
    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    proxy_connect_timeout 10;
    proxy_send_timeout 10;
    proxy_read_timeout 10;		
    proxy_buffer_size 4k;
    proxy_buffers 4 32k;
    proxy_busy_buffers_size 64k;        
    proxy_temp_file_write_size 64k;           
    proxy_ignore_client_abort on;
    location /{
      proxy_pass http://backend;
    }
  }
  upstream backend{
    server   127.0.0.1:1313  weight=1 max_fails=2 fail_timeout=10s;
    keepalive 300;
  }
}

nginx.service

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
[Unit]
Description=The NGINX HTTP and reverse proxy server
#After=syslog.target network.target remote-fs.target nss-lookup.target
After=syslog.target network.target

[Service]
LimitNOFILE=30000
Type=forking
PIDFile=/opt/nginx/logs/nginx.pid
ExecStartPre=/opt/nginx/sbin/nginx -t
ExecStart=/opt/nginx/sbin/nginx
ExecReload=/opt/nginx/sbin/nginx -s reload
ExecStop=/bin/kill -s QUIT $MAINPID
PrivateTmp=true

[Install]
WantedBy=multi-user.target

troubleshooting

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
netstat -n|awk '/^tcp/{++S[$NF]}END{for (key in S) print key,S[key]}'
# 返回结果如下:
LAST_ACK 14
SYN_RECV 348
ESTABLISHED 70
FIN_WAIT1 229
FIN_WAIT2 30
CLOSING 33
TIME_WAIT 18122
# 说明
# CLOSED:无连接是活动的或正在进行
# LISTEN:服务器在等待进入呼叫
# SYN_RECV:一个连接请求已经到达,等待确认
# SYN_SENT:应用已经开始,打开一个连接
# ESTABLISHED:正常数据传输状态
# FIN_WAIT1:应用说它已经完成
# FIN_WAIT2:另一边已同意释放
# ITMED_WAIT:等待所有分组死掉
# CLOSING:两边同时尝试关闭
# TIME_WAIT:另一边已初始化一个释放
# LAST_ACK:等待所有分组死掉

ss -tan state time-wait|wc -l
###
netstat -n | awk '/^tcp/ {++S[$NF]} END {for(a in S) print a, S[a]}'

参考