src/http/ngx_http_upstream_round_robin.c - nginx-1.7.10

Functions defined

Macros defined

Source code


  1. /*
  2. * Copyright (C) Igor Sysoev
  3. * Copyright (C) Nginx, Inc.
  4. */


  5. #include <ngx_config.h>
  6. #include <ngx_core.h>
  7. #include <ngx_http.h>


  8. #define ngx_http_upstream_tries(p) ((p)->number                               \
  9.                                     + ((p)->next ? (p)->next->number : 0))


  10. static ngx_http_upstream_rr_peer_t *ngx_http_upstream_get_peer(
  11.     ngx_http_upstream_rr_peer_data_t *rrp);

  12. #if (NGX_HTTP_SSL)

  13. static ngx_int_t ngx_http_upstream_empty_set_session(ngx_peer_connection_t *pc,
  14.     void *data);
  15. static void ngx_http_upstream_empty_save_session(ngx_peer_connection_t *pc,
  16.     void *data);

  17. #endif


  18. ngx_int_t
  19. ngx_http_upstream_init_round_robin(ngx_conf_t *cf,
  20.     ngx_http_upstream_srv_conf_t *us)
  21. {
  22.     ngx_url_t                      u;
  23.     ngx_uint_t                     i, j, n, w;
  24.     ngx_http_upstream_server_t    *server;
  25.     ngx_http_upstream_rr_peer_t   *peer;
  26.     ngx_http_upstream_rr_peers_t  *peers, *backup;

  27.     us->peer.init = ngx_http_upstream_init_round_robin_peer;

  28.     if (us->servers) {
  29.         server = us->servers->elts;

  30.         n = 0;
  31.         w = 0;

  32.         for (i = 0; i < us->servers->nelts; i++) {
  33.             if (server[i].backup) {
  34.                 continue;
  35.             }

  36.             n += server[i].naddrs;
  37.             w += server[i].naddrs * server[i].weight;
  38.         }

  39.         if (n == 0) {
  40.             ngx_log_error(NGX_LOG_EMERG, cf->log, 0,
  41.                           "no servers in upstream \"%V\" in %s:%ui",
  42.                           &us->host, us->file_name, us->line);
  43.             return NGX_ERROR;
  44.         }

  45.         peers = ngx_pcalloc(cf->pool, sizeof(ngx_http_upstream_rr_peers_t)
  46.                               + sizeof(ngx_http_upstream_rr_peer_t) * (n - 1));
  47.         if (peers == NULL) {
  48.             return NGX_ERROR;
  49.         }

  50.         peers->single = (n == 1);
  51.         peers->number = n;
  52.         peers->weighted = (w != n);
  53.         peers->total_weight = w;
  54.         peers->name = &us->host;

  55.         n = 0;
  56.         peer = peers->peer;

  57.         for (i = 0; i < us->servers->nelts; i++) {
  58.             if (server[i].backup) {
  59.                 continue;
  60.             }

  61.             for (j = 0; j < server[i].naddrs; j++) {
  62.                 peer[n].sockaddr = server[i].addrs[j].sockaddr;
  63.                 peer[n].socklen = server[i].addrs[j].socklen;
  64.                 peer[n].name = server[i].addrs[j].name;
  65.                 peer[n].weight = server[i].weight;
  66.                 peer[n].effective_weight = server[i].weight;
  67.                 peer[n].current_weight = 0;
  68.                 peer[n].max_fails = server[i].max_fails;
  69.                 peer[n].fail_timeout = server[i].fail_timeout;
  70.                 peer[n].down = server[i].down;
  71.                 peer[n].server = server[i].name;
  72.                 n++;
  73.             }
  74.         }

  75.         us->peer.data = peers;

  76.         /* backup servers */

  77.         n = 0;
  78.         w = 0;

  79.         for (i = 0; i < us->servers->nelts; i++) {
  80.             if (!server[i].backup) {
  81.                 continue;
  82.             }

  83.             n += server[i].naddrs;
  84.             w += server[i].naddrs * server[i].weight;
  85.         }

  86.         if (n == 0) {
  87.             return NGX_OK;
  88.         }

  89.         backup = ngx_pcalloc(cf->pool, sizeof(ngx_http_upstream_rr_peers_t)
  90.                               + sizeof(ngx_http_upstream_rr_peer_t) * (n - 1));
  91.         if (backup == NULL) {
  92.             return NGX_ERROR;
  93.         }

  94.         peers->single = 0;
  95.         backup->single = 0;
  96.         backup->number = n;
  97.         backup->weighted = (w != n);
  98.         backup->total_weight = w;
  99.         backup->name = &us->host;

  100.         n = 0;
  101.         peer = backup->peer;

  102.         for (i = 0; i < us->servers->nelts; i++) {
  103.             if (!server[i].backup) {
  104.                 continue;
  105.             }

  106.             for (j = 0; j < server[i].naddrs; j++) {
  107.                 peer[n].sockaddr = server[i].addrs[j].sockaddr;
  108.                 peer[n].socklen = server[i].addrs[j].socklen;
  109.                 peer[n].name = server[i].addrs[j].name;
  110.                 peer[n].weight = server[i].weight;
  111.                 peer[n].effective_weight = server[i].weight;
  112.                 peer[n].current_weight = 0;
  113.                 peer[n].max_fails = server[i].max_fails;
  114.                 peer[n].fail_timeout = server[i].fail_timeout;
  115.                 peer[n].down = server[i].down;
  116.                 peer[n].server = server[i].name;
  117.                 n++;
  118.             }
  119.         }

  120.         peers->next = backup;

  121.         return NGX_OK;
  122.     }


  123.     /* an upstream implicitly defined by proxy_pass, etc. */

  124.     if (us->port == 0) {
  125.         ngx_log_error(NGX_LOG_EMERG, cf->log, 0,
  126.                       "no port in upstream \"%V\" in %s:%ui",
  127.                       &us->host, us->file_name, us->line);
  128.         return NGX_ERROR;
  129.     }

  130.     ngx_memzero(&u, sizeof(ngx_url_t));

  131.     u.host = us->host;
  132.     u.port = us->port;

  133.     if (ngx_inet_resolve_host(cf->pool, &u) != NGX_OK) {
  134.         if (u.err) {
  135.             ngx_log_error(NGX_LOG_EMERG, cf->log, 0,
  136.                           "%s in upstream \"%V\" in %s:%ui",
  137.                           u.err, &us->host, us->file_name, us->line);
  138.         }

  139.         return NGX_ERROR;
  140.     }

  141.     n = u.naddrs;

  142.     peers = ngx_pcalloc(cf->pool, sizeof(ngx_http_upstream_rr_peers_t)
  143.                               + sizeof(ngx_http_upstream_rr_peer_t) * (n - 1));
  144.     if (peers == NULL) {
  145.         return NGX_ERROR;
  146.     }

  147.     peers->single = (n == 1);
  148.     peers->number = n;
  149.     peers->weighted = 0;
  150.     peers->total_weight = n;
  151.     peers->name = &us->host;

  152.     peer = peers->peer;

  153.     for (i = 0; i < u.naddrs; i++) {
  154.         peer[i].sockaddr = u.addrs[i].sockaddr;
  155.         peer[i].socklen = u.addrs[i].socklen;
  156.         peer[i].name = u.addrs[i].name;
  157.         peer[i].weight = 1;
  158.         peer[i].effective_weight = 1;
  159.         peer[i].current_weight = 0;
  160.         peer[i].max_fails = 1;
  161.         peer[i].fail_timeout = 10;
  162.     }

  163.     us->peer.data = peers;

  164.     /* implicitly defined upstream has no backup servers */

  165.     return NGX_OK;
  166. }


  167. ngx_int_t
  168. ngx_http_upstream_init_round_robin_peer(ngx_http_request_t *r,
  169.     ngx_http_upstream_srv_conf_t *us)
  170. {
  171.     ngx_uint_t                         n;
  172.     ngx_http_upstream_rr_peer_data_t  *rrp;

  173.     rrp = r->upstream->peer.data;

  174.     if (rrp == NULL) {
  175.         rrp = ngx_palloc(r->pool, sizeof(ngx_http_upstream_rr_peer_data_t));
  176.         if (rrp == NULL) {
  177.             return NGX_ERROR;
  178.         }

  179.         r->upstream->peer.data = rrp;
  180.     }

  181.     rrp->peers = us->peer.data;
  182.     rrp->current = 0;

  183.     n = rrp->peers->number;

  184.     if (rrp->peers->next && rrp->peers->next->number > n) {
  185.         n = rrp->peers->next->number;
  186.     }

  187.     if (n <= 8 * sizeof(uintptr_t)) {
  188.         rrp->tried = &rrp->data;
  189.         rrp->data = 0;

  190.     } else {
  191.         n = (n + (8 * sizeof(uintptr_t) - 1)) / (8 * sizeof(uintptr_t));

  192.         rrp->tried = ngx_pcalloc(r->pool, n * sizeof(uintptr_t));
  193.         if (rrp->tried == NULL) {
  194.             return NGX_ERROR;
  195.         }
  196.     }

  197.     r->upstream->peer.get = ngx_http_upstream_get_round_robin_peer;
  198.     r->upstream->peer.free = ngx_http_upstream_free_round_robin_peer;
  199.     r->upstream->peer.tries = ngx_http_upstream_tries(rrp->peers);
  200. #if (NGX_HTTP_SSL)
  201.     r->upstream->peer.set_session =
  202.                                ngx_http_upstream_set_round_robin_peer_session;
  203.     r->upstream->peer.save_session =
  204.                                ngx_http_upstream_save_round_robin_peer_session;
  205. #endif

  206.     return NGX_OK;
  207. }


  208. ngx_int_t
  209. ngx_http_upstream_create_round_robin_peer(ngx_http_request_t *r,
  210.     ngx_http_upstream_resolved_t *ur)
  211. {
  212.     u_char                            *p;
  213.     size_t                             len;
  214.     socklen_t                          socklen;
  215.     ngx_uint_t                         i, n;
  216.     struct sockaddr                   *sockaddr;
  217.     ngx_http_upstream_rr_peer_t       *peer;
  218.     ngx_http_upstream_rr_peers_t      *peers;
  219.     ngx_http_upstream_rr_peer_data_t  *rrp;

  220.     rrp = r->upstream->peer.data;

  221.     if (rrp == NULL) {
  222.         rrp = ngx_palloc(r->pool, sizeof(ngx_http_upstream_rr_peer_data_t));
  223.         if (rrp == NULL) {
  224.             return NGX_ERROR;
  225.         }

  226.         r->upstream->peer.data = rrp;
  227.     }

  228.     peers = ngx_pcalloc(r->pool, sizeof(ngx_http_upstream_rr_peers_t)
  229.                      + sizeof(ngx_http_upstream_rr_peer_t) * (ur->naddrs - 1));
  230.     if (peers == NULL) {
  231.         return NGX_ERROR;
  232.     }

  233.     peers->single = (ur->naddrs == 1);
  234.     peers->number = ur->naddrs;
  235.     peers->name = &ur->host;

  236.     peer = peers->peer;

  237.     if (ur->sockaddr) {
  238.         peer[0].sockaddr = ur->sockaddr;
  239.         peer[0].socklen = ur->socklen;
  240.         peer[0].name = ur->host;
  241.         peer[0].weight = 1;
  242.         peer[0].effective_weight = 1;
  243.         peer[0].current_weight = 0;
  244.         peer[0].max_fails = 1;
  245.         peer[0].fail_timeout = 10;

  246.     } else {

  247.         for (i = 0; i < ur->naddrs; i++) {

  248.             socklen = ur->addrs[i].socklen;

  249.             sockaddr = ngx_palloc(r->pool, socklen);
  250.             if (sockaddr == NULL) {
  251.                 return NGX_ERROR;
  252.             }

  253.             ngx_memcpy(sockaddr, ur->addrs[i].sockaddr, socklen);

  254.             switch (sockaddr->sa_family) {
  255. #if (NGX_HAVE_INET6)
  256.             case AF_INET6:
  257.                 ((struct sockaddr_in6 *) sockaddr)->sin6_port = htons(ur->port);
  258.                 break;
  259. #endif
  260.             default: /* AF_INET */
  261.                 ((struct sockaddr_in *) sockaddr)->sin_port = htons(ur->port);
  262.             }

  263.             p = ngx_pnalloc(r->pool, NGX_SOCKADDR_STRLEN);
  264.             if (p == NULL) {
  265.                 return NGX_ERROR;
  266.             }

  267.             len = ngx_sock_ntop(sockaddr, socklen, p, NGX_SOCKADDR_STRLEN, 1);

  268.             peer[i].sockaddr = sockaddr;
  269.             peer[i].socklen = socklen;
  270.             peer[i].name.len = len;
  271.             peer[i].name.data = p;
  272.             peer[i].weight = 1;
  273.             peer[i].effective_weight = 1;
  274.             peer[i].current_weight = 0;
  275.             peer[i].max_fails = 1;
  276.             peer[i].fail_timeout = 10;
  277.         }
  278.     }

  279.     rrp->peers = peers;
  280.     rrp->current = 0;

  281.     if (rrp->peers->number <= 8 * sizeof(uintptr_t)) {
  282.         rrp->tried = &rrp->data;
  283.         rrp->data = 0;

  284.     } else {
  285.         n = (rrp->peers->number + (8 * sizeof(uintptr_t) - 1))
  286.                 / (8 * sizeof(uintptr_t));

  287.         rrp->tried = ngx_pcalloc(r->pool, n * sizeof(uintptr_t));
  288.         if (rrp->tried == NULL) {
  289.             return NGX_ERROR;
  290.         }
  291.     }

  292.     r->upstream->peer.get = ngx_http_upstream_get_round_robin_peer;
  293.     r->upstream->peer.free = ngx_http_upstream_free_round_robin_peer;
  294.     r->upstream->peer.tries = ngx_http_upstream_tries(rrp->peers);
  295. #if (NGX_HTTP_SSL)
  296.     r->upstream->peer.set_session = ngx_http_upstream_empty_set_session;
  297.     r->upstream->peer.save_session = ngx_http_upstream_empty_save_session;
  298. #endif

  299.     return NGX_OK;
  300. }


  301. ngx_int_t
  302. ngx_http_upstream_get_round_robin_peer(ngx_peer_connection_t *pc, void *data)
  303. {
  304.     ngx_http_upstream_rr_peer_data_t  *rrp = data;

  305.     ngx_int_t                      rc;
  306.     ngx_uint_t                     i, n;
  307.     ngx_http_upstream_rr_peer_t   *peer;
  308.     ngx_http_upstream_rr_peers_t  *peers;

  309.     ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0,
  310.                    "get rr peer, try: %ui", pc->tries);

  311.     pc->cached = 0;
  312.     pc->connection = NULL;

  313.     peers = rrp->peers;

  314.     /* ngx_lock_mutex(peers->mutex); */

  315.     if (peers->single) {
  316.         peer = &peers->peer[0];

  317.         if (peer->down) {
  318.             goto failed;
  319.         }

  320.     } else {

  321.         /* there are several peers */

  322.         peer = ngx_http_upstream_get_peer(rrp);

  323.         if (peer == NULL) {
  324.             goto failed;
  325.         }

  326.         ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
  327.                        "get rr peer, current: %ui %i",
  328.                        rrp->current, peer->current_weight);
  329.     }

  330.     pc->sockaddr = peer->sockaddr;
  331.     pc->socklen = peer->socklen;
  332.     pc->name = &peer->name;

  333.     /* ngx_unlock_mutex(peers->mutex); */

  334.     return NGX_OK;

  335. failed:

  336.     if (peers->next) {

  337.         /* ngx_unlock_mutex(peers->mutex); */

  338.         ngx_log_debug0(NGX_LOG_DEBUG_HTTP, pc->log, 0, "backup servers");

  339.         rrp->peers = peers->next;

  340.         n = (rrp->peers->number + (8 * sizeof(uintptr_t) - 1))
  341.                 / (8 * sizeof(uintptr_t));

  342.         for (i = 0; i < n; i++) {
  343.              rrp->tried[i] = 0;
  344.         }

  345.         rc = ngx_http_upstream_get_round_robin_peer(pc, rrp);

  346.         if (rc != NGX_BUSY) {
  347.             return rc;
  348.         }

  349.         /* ngx_lock_mutex(peers->mutex); */
  350.     }

  351.     /* all peers failed, mark them as live for quick recovery */

  352.     for (i = 0; i < peers->number; i++) {
  353.         peers->peer[i].fails = 0;
  354.     }

  355.     /* ngx_unlock_mutex(peers->mutex); */

  356.     pc->name = peers->name;

  357.     return NGX_BUSY;
  358. }


  359. static ngx_http_upstream_rr_peer_t *
  360. ngx_http_upstream_get_peer(ngx_http_upstream_rr_peer_data_t *rrp)
  361. {
  362.     time_t                        now;
  363.     uintptr_t                     m;
  364.     ngx_int_t                     total;
  365.     ngx_uint_t                    i, n;
  366.     ngx_http_upstream_rr_peer_t  *peer, *best;

  367.     now = ngx_time();

  368.     best = NULL;
  369.     total = 0;

  370.     for (i = 0; i < rrp->peers->number; i++) {

  371.         n = i / (8 * sizeof(uintptr_t));
  372.         m = (uintptr_t) 1 << i % (8 * sizeof(uintptr_t));

  373.         if (rrp->tried[n] & m) {
  374.             continue;
  375.         }

  376.         peer = &rrp->peers->peer[i];

  377.         if (peer->down) {
  378.             continue;
  379.         }

  380.         if (peer->max_fails
  381.             && peer->fails >= peer->max_fails
  382.             && now - peer->checked <= peer->fail_timeout)
  383.         {
  384.             continue;
  385.         }

  386.         peer->current_weight += peer->effective_weight;
  387.         total += peer->effective_weight;

  388.         if (peer->effective_weight < peer->weight) {
  389.             peer->effective_weight++;
  390.         }

  391.         if (best == NULL || peer->current_weight > best->current_weight) {
  392.             best = peer;
  393.         }
  394.     }

  395.     if (best == NULL) {
  396.         return NULL;
  397.     }

  398.     i = best - &rrp->peers->peer[0];

  399.     rrp->current = i;

  400.     n = i / (8 * sizeof(uintptr_t));
  401.     m = (uintptr_t) 1 << i % (8 * sizeof(uintptr_t));

  402.     rrp->tried[n] |= m;

  403.     best->current_weight -= total;

  404.     if (now - best->checked > best->fail_timeout) {
  405.         best->checked = now;
  406.     }

  407.     return best;
  408. }


  409. void
  410. ngx_http_upstream_free_round_robin_peer(ngx_peer_connection_t *pc, void *data,
  411.     ngx_uint_t state)
  412. {
  413.     ngx_http_upstream_rr_peer_data_t  *rrp = data;

  414.     time_t                       now;
  415.     ngx_http_upstream_rr_peer_t  *peer;

  416.     ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
  417.                    "free rr peer %ui %ui", pc->tries, state);

  418.     /* TODO: NGX_PEER_KEEPALIVE */

  419.     if (rrp->peers->single) {
  420.         pc->tries = 0;
  421.         return;
  422.     }

  423.     peer = &rrp->peers->peer[rrp->current];

  424.     if (state & NGX_PEER_FAILED) {
  425.         now = ngx_time();

  426.         /* ngx_lock_mutex(rrp->peers->mutex); */

  427.         peer->fails++;
  428.         peer->accessed = now;
  429.         peer->checked = now;

  430.         if (peer->max_fails) {
  431.             peer->effective_weight -= peer->weight / peer->max_fails;
  432.         }

  433.         ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
  434.                        "free rr peer failed: %ui %i",
  435.                        rrp->current, peer->effective_weight);

  436.         if (peer->effective_weight < 0) {
  437.             peer->effective_weight = 0;
  438.         }

  439.         /* ngx_unlock_mutex(rrp->peers->mutex); */

  440.     } else {

  441.         /* mark peer live if check passed */

  442.         if (peer->accessed < peer->checked) {
  443.             peer->fails = 0;
  444.         }
  445.     }

  446.     if (pc->tries) {
  447.         pc->tries--;
  448.     }

  449.     /* ngx_unlock_mutex(rrp->peers->mutex); */
  450. }


  451. #if (NGX_HTTP_SSL)

  452. ngx_int_t
  453. ngx_http_upstream_set_round_robin_peer_session(ngx_peer_connection_t *pc,
  454.     void *data)
  455. {
  456.     ngx_http_upstream_rr_peer_data_t  *rrp = data;

  457.     ngx_int_t                     rc;
  458.     ngx_ssl_session_t            *ssl_session;
  459.     ngx_http_upstream_rr_peer_t  *peer;

  460.     peer = &rrp->peers->peer[rrp->current];

  461.     /* TODO: threads only mutex */
  462.     /* ngx_lock_mutex(rrp->peers->mutex); */

  463.     ssl_session = peer->ssl_session;

  464.     rc = ngx_ssl_set_session(pc->connection, ssl_session);

  465.     ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0,
  466.                    "set session: %p", ssl_session);

  467.     /* ngx_unlock_mutex(rrp->peers->mutex); */

  468.     return rc;
  469. }


  470. void
  471. ngx_http_upstream_save_round_robin_peer_session(ngx_peer_connection_t *pc,
  472.     void *data)
  473. {
  474.     ngx_http_upstream_rr_peer_data_t  *rrp = data;

  475.     ngx_ssl_session_t            *old_ssl_session, *ssl_session;
  476.     ngx_http_upstream_rr_peer_t  *peer;

  477.     ssl_session = ngx_ssl_get_session(pc->connection);

  478.     if (ssl_session == NULL) {
  479.         return;
  480.     }

  481.     ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0,
  482.                    "save session: %p", ssl_session);

  483.     peer = &rrp->peers->peer[rrp->current];

  484.     /* TODO: threads only mutex */
  485.     /* ngx_lock_mutex(rrp->peers->mutex); */

  486.     old_ssl_session = peer->ssl_session;
  487.     peer->ssl_session = ssl_session;

  488.     /* ngx_unlock_mutex(rrp->peers->mutex); */

  489.     if (old_ssl_session) {

  490.         ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0,
  491.                        "old session: %p", old_ssl_session);

  492.         /* TODO: may block */

  493.         ngx_ssl_free_session(old_ssl_session);
  494.     }
  495. }


  496. static ngx_int_t
  497. ngx_http_upstream_empty_set_session(ngx_peer_connection_t *pc, void *data)
  498. {
  499.     return NGX_OK;
  500. }


  501. static void
  502. ngx_http_upstream_empty_save_session(ngx_peer_connection_t *pc, void *data)
  503. {
  504.     return;
  505. }

  506. #endif