Ryfujk

We are load testing one of our application server using Gatling sending ~150 req/s. When we load test by calling the application server directly, everything works fine but when we are calling through nginx which act as a reverse proxy, there are a lot of timeouts (60s) at Gatling.

Looking at nginx logs we saw cases where upstream_connect_time would be '-'

{"log":""20/Mar/2019:08:26:16 +0000" client=172.30.14.21 method=POST request="POST /api/shopping/log HTTP/1.1" request_length=5672 status=499 bytes_sent=0 body_byt

es_sent=0 referer=- user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36" upstream_addr=172

.30.106.47:80 upstream_status=- request_time=60.000 upstream_response_time=- upstream_connect_time=- upstream_header_time=-n","stream":"stdout","time":"2019-03-20T08:

26:16.593007324Z"}

or in some cases, upstream_connect_time goes to 31s

{"log":""20/Mar/2019:08:26:00 +0000" client=172.30.14.21 method=POST request="POST /api/shopping/log HTTP/1.1" request_length=5672 status=200 bytes_sent=280 body_bytes_sent=12 referer=- user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36" upstream_addr=172.30.106.47:80 upstream_status=200 request_time=31.154 upstream_response_time=31.154 upstream_connect_time=31.080 upstream_header_time=31.154n","stream":"stdout","time":"2019-03-20T08:26:00.355564536Z"}

So we imagine this could be something related to connections to an upstream server. We've tried enabled keepalive but it doesn't help. When we tried lower the workers_connection to 512, we would get an error saying 'workers_connection are not enough'. Below is our nginx.conf. What could be the problem here?

user  nginx;

worker_processes  auto;

worker_rlimit_nofile 100000;



error_log  /var/log/nginx/error.log warn;

pid        /var/run/nginx.pid;



events {

  worker_connections 1000;

  use epoll;

  multi_accept on;

}



http {

  log_format main '"$time_local" client=$remote_addr '

               'method=$request_method request="$request" '

               'request_length=$request_length '

               'status=$status bytes_sent=$bytes_sent '

               'body_bytes_sent=$body_bytes_sent '

               'referer=$http_referer '

               'user_agent="$http_user_agent" '

               'upstream_addr=$upstream_addr '

               'upstream_status=$upstream_status '

               'request_time=$request_time '

               'upstream_response_time=$upstream_response_time '

               'upstream_connect_time=$upstream_connect_time '

               'upstream_header_time=$upstream_header_time';

  access_log  /var/log/nginx/access.log  main;

  open_file_cache max=200000 inactive=20s;

  open_file_cache_valid 30s;

  open_file_cache_min_uses 2;

  open_file_cache_errors on;

  sendfile on;

  tcp_nopush on;

  tcp_nodelay on;

  gzip on;

  gzip_min_length 10240;

  gzip_comp_level 1;

  gzip_vary on;

  gzip_disable msie6;

  gzip_proxied expired no-cache no-store private auth;

  gzip_types

      text/css

      text/javascript

      text/xml

      text/plain

      text/x-component

      application/javascript

      application/x-javascript

      application/json

      application/xml

      application/rss+xml

      application/atom+xml

      font/truetype

      font/opentype

      application/vnd.ms-fontobject

      image/svg+xml;



  reset_timedout_connection on;

  client_body_timeout 10;

  send_timeout 2;

  keepalive_requests 100000;



  upstream backend {

      server 172.30.106.47:80;

      keepalive 128;

  }



  server {

      listen 80;

      server_name localhost;

      location = /health {

          return 200;

      }

      location /status {

          stub_status on;

      }

      location / {

          proxy_buffers 32 4m;

          proxy_busy_buffers_size 25m;

          proxy_buffer_size 512k;

          proxy_max_temp_file_size 0;

          proxy_set_header Host $host;

          proxy_set_header X-Real-IP $remote_addr;

          proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

          client_max_body_size 1024m;

          client_body_buffer_size 4m;

          proxy_connect_timeout 300;

          proxy_read_timeout 300;

          proxy_send_timeout 300;

          proxy_intercept_errors off;

          stub_status on;

          proxy_pass http://backend;

      }

      location = /50x.html {

          root /usr/share/nginx/html;

      }

      error_page 500 502 503 504  /50x.html;

  }



}