One Level Up Top Level

src/http/ngx_http_upstream_round_robin.c - nginx-1.7.10

Functions defined

Macros defined

Source code


  2. /*
  3. * Copyright (C) Igor Sysoev
  4. * Copyright (C) Nginx, Inc.
  5. */


  8. #include <ngx_config.h>
  9. #include <ngx_core.h>
  10. #include <ngx_http.h>


  13. ‌#define ngx_http_upstream_tries(p) ((p)->number                               \
  14.                                     + ((p)->next ? (p)->next->number : 0))


  17. static ngx_http_upstream_rr_peer_t *ngx_http_upstream_get_peer(
  18.     ngx_http_upstream_rr_peer_data_t *rrp);

  20. #if (NGX_HTTP_SSL)

  22. static ngx_int_t ngx_http_upstream_empty_set_session(ngx_peer_connection_t *pc,
  23.     void *data);
  24. static void ngx_http_upstream_empty_save_session(ngx_peer_connection_t *pc,
  25.     void *data);

  27. #endif


  30. ngx_int_t
  31. ‌ngx_http_upstream_init_round_robin(ngx_conf_t *cf,
  32.     ngx_http_upstream_srv_conf_t *us)
  33. {
  34.     ngx_url_t                      u;
  35.     ngx_uint_t                     i, j, n, w;
  36.     ngx_http_upstream_server_t    *server;
  37.     ngx_http_upstream_rr_peer_t   *peer;
  38.     ngx_http_upstream_rr_peers_t  *peers, *backup;

  40.     us->peer.init = ngx_http_upstream_init_round_robin_peer;

  42.     if (us->servers) {
  43.         server = us->servers->elts;

  45.         n = 0;
  46.         w = 0;

  48.         for (i = 0; i < us->servers->nelts; i++) {
  49.             if (server[i].backup) {
  50.                 continue;
  51.             }

  53.             n += server[i].naddrs;
  54.             w += server[i].naddrs * server[i].weight;
  55.         }

  57.         if (n == 0) {
  58.             ngx_log_error(NGX_LOG_EMERG, cf->log, 0,
  59.                           "no servers in upstream \"%V\" in %s:%ui",
  60.                           &us->host, us->file_name, us->line);
  61.             return NGX_ERROR;
  62.         }

  64.         peers = ngx_pcalloc(cf->pool, sizeof(ngx_http_upstream_rr_peers_t)
  65.                               + sizeof(ngx_http_upstream_rr_peer_t) * (n - 1));
  66.         if (peers == NULL) {
  67.             return NGX_ERROR;
  68.         }

  70.         peers->single = (n == 1);
  71.         peers->number = n;
  72.         peers->weighted = (w != n);
  73.         peers->total_weight = w;
  74.         peers->name = &us->host;

  76.         n = 0;
  77.         peer = peers->peer;

  79.         for (i = 0; i < us->servers->nelts; i++) {
  80.             if (server[i].backup) {
  81.                 continue;
  82.             }

  84.             for (j = 0; j < server[i].naddrs; j++) {
  85.                 peer[n].sockaddr = server[i].addrs[j].sockaddr;
  86.                 peer[n].socklen = server[i].addrs[j].socklen;
  87.                 peer[n].name = server[i].addrs[j].name;
  88.                 peer[n].weight = server[i].weight;
  89.                 peer[n].effective_weight = server[i].weight;
  90.                 peer[n].current_weight = 0;
  91.                 peer[n].max_fails = server[i].max_fails;
  92.                 peer[n].fail_timeout = server[i].fail_timeout;
  93.                 peer[n].down = server[i].down;
  94.                 peer[n].server = server[i].name;
  95.                 n++;
  96.             }
  97.         }

  99.         us->peer.data = peers;

  101.         /* backup servers */

  103.         n = 0;
  104.         w = 0;

  106.         for (i = 0; i < us->servers->nelts; i++) {
  107.             if (!server[i].backup) {
  108.                 continue;
  109.             }

  111.             n += server[i].naddrs;
  112.             w += server[i].naddrs * server[i].weight;
  113.         }

  115.         if (n == 0) {
  116.             return NGX_OK;
  117.         }

  119.         backup = ngx_pcalloc(cf->pool, sizeof(ngx_http_upstream_rr_peers_t)
  120.                               + sizeof(ngx_http_upstream_rr_peer_t) * (n - 1));
  121.         if (backup == NULL) {
  122.             return NGX_ERROR;
  123.         }

  125.         peers->single = 0;
  126.         backup->single = 0;
  127.         backup->number = n;
  128.         backup->weighted = (w != n);
  129.         backup->total_weight = w;
  130.         backup->name = &us->host;

  132.         n = 0;
  133.         peer = backup->peer;

  135.         for (i = 0; i < us->servers->nelts; i++) {
  136.             if (!server[i].backup) {
  137.                 continue;
  138.             }

  140.             for (j = 0; j < server[i].naddrs; j++) {
  141.                 peer[n].sockaddr = server[i].addrs[j].sockaddr;
  142.                 peer[n].socklen = server[i].addrs[j].socklen;
  143.                 peer[n].name = server[i].addrs[j].name;
  144.                 peer[n].weight = server[i].weight;
  145.                 peer[n].effective_weight = server[i].weight;
  146.                 peer[n].current_weight = 0;
  147.                 peer[n].max_fails = server[i].max_fails;
  148.                 peer[n].fail_timeout = server[i].fail_timeout;
  149.                 peer[n].down = server[i].down;
  150.                 peer[n].server = server[i].name;
  151.                 n++;
  152.             }
  153.         }

  155.         peers->next = backup;

  157.         return NGX_OK;
  158.     }


  161.     /* an upstream implicitly defined by proxy_pass, etc. */

  163.     if (us->port == 0) {
  164.         ngx_log_error(NGX_LOG_EMERG, cf->log, 0,
  165.                       "no port in upstream \"%V\" in %s:%ui",
  166.                       &us->host, us->file_name, us->line);
  167.         return NGX_ERROR;
  168.     }

  170.     ngx_memzero(&u, sizeof(ngx_url_t));

  172.     u.host = us->host;
  173.     u.port = us->port;

  175.     if (ngx_inet_resolve_host(cf->pool, &u) != NGX_OK) {
  176.         if (u.err) {
  177.             ngx_log_error(NGX_LOG_EMERG, cf->log, 0,
  178.                           "%s in upstream \"%V\" in %s:%ui",
  179.                           u.err, &us->host, us->file_name, us->line);
  180.         }

  182.         return NGX_ERROR;
  183.     }

  185.     n = u.naddrs;

  187.     peers = ngx_pcalloc(cf->pool, sizeof(ngx_http_upstream_rr_peers_t)
  188.                               + sizeof(ngx_http_upstream_rr_peer_t) * (n - 1));
  189.     if (peers == NULL) {
  190.         return NGX_ERROR;
  191.     }

  193.     peers->single = (n == 1);
  194.     peers->number = n;
  195.     peers->weighted = 0;
  196.     peers->total_weight = n;
  197.     peers->name = &us->host;

  199.     peer = peers->peer;

  201.     for (i = 0; i < u.naddrs; i++) {
  202.         peer[i].sockaddr = u.addrs[i].sockaddr;
  203.         peer[i].socklen = u.addrs[i].socklen;
  204.         peer[i].name = u.addrs[i].name;
  205.         peer[i].weight = 1;
  206.         peer[i].effective_weight = 1;
  207.         peer[i].current_weight = 0;
  208.         peer[i].max_fails = 1;
  209.         peer[i].fail_timeout = 10;
  210.     }

  212.     us->peer.data = peers;

  214.     /* implicitly defined upstream has no backup servers */

  216.     return NGX_OK;
  217. }


  220. ngx_int_t
  221. ‌ngx_http_upstream_init_round_robin_peer(ngx_http_request_t *r,
  222.     ngx_http_upstream_srv_conf_t *us)
  223. {
  224.     ngx_uint_t                         n;
  225.     ngx_http_upstream_rr_peer_data_t  *rrp;

  227.     rrp = r->upstream->peer.data;

  229.     if (rrp == NULL) {
  230.         rrp = ngx_palloc(r->pool, sizeof(ngx_http_upstream_rr_peer_data_t));
  231.         if (rrp == NULL) {
  232.             return NGX_ERROR;
  233.         }

  235.         r->upstream->peer.data = rrp;
  236.     }

  238.     rrp->peers = us->peer.data;
  239.     rrp->current = 0;

  241.     n = rrp->peers->number;

  243.     if (rrp->peers->next && rrp->peers->next->number > n) {
  244.         n = rrp->peers->next->number;
  245.     }

  247.     if (n <= 8 * sizeof(uintptr_t)) {
  248.         rrp->tried = &rrp->data;
  249.         rrp->data = 0;

  251.     } else {
  252.         n = (n + (8 * sizeof(uintptr_t) - 1)) / (8 * sizeof(uintptr_t));

  254.         rrp->tried = ngx_pcalloc(r->pool, n * sizeof(uintptr_t));
  255.         if (rrp->tried == NULL) {
  256.             return NGX_ERROR;
  257.         }
  258.     }

  260.     r->upstream->peer.get = ngx_http_upstream_get_round_robin_peer;
  261.     r->upstream->peer.free = ngx_http_upstream_free_round_robin_peer;
  262.     r->upstream->peer.tries = ngx_http_upstream_tries(rrp->peers);
  263. #if (NGX_HTTP_SSL)
  264.     r->upstream->peer.set_session =
  265.                                ngx_http_upstream_set_round_robin_peer_session;
  266.     r->upstream->peer.save_session =
  267.                                ngx_http_upstream_save_round_robin_peer_session;
  268. #endif

  270.     return NGX_OK;
  271. }


  274. ngx_int_t
  275. ‌ngx_http_upstream_create_round_robin_peer(ngx_http_request_t *r,
  276.     ngx_http_upstream_resolved_t *ur)
  277. {
  278.     u_char                            *p;
  279.     size_t                             len;
  280.     socklen_t                          socklen;
  281.     ngx_uint_t                         i, n;
  282.     struct sockaddr                   *sockaddr;
  283.     ngx_http_upstream_rr_peer_t       *peer;
  284.     ngx_http_upstream_rr_peers_t      *peers;
  285.     ngx_http_upstream_rr_peer_data_t  *rrp;

  287.     rrp = r->upstream->peer.data;

  289.     if (rrp == NULL) {
  290.         rrp = ngx_palloc(r->pool, sizeof(ngx_http_upstream_rr_peer_data_t));
  291.         if (rrp == NULL) {
  292.             return NGX_ERROR;
  293.         }

  295.         r->upstream->peer.data = rrp;
  296.     }

  298.     peers = ngx_pcalloc(r->pool, sizeof(ngx_http_upstream_rr_peers_t)
  299.                      + sizeof(ngx_http_upstream_rr_peer_t) * (ur->naddrs - 1));
  300.     if (peers == NULL) {
  301.         return NGX_ERROR;
  302.     }

  304.     peers->single = (ur->naddrs == 1);
  305.     peers->number = ur->naddrs;
  306.     peers->name = &ur->host;

  308.     peer = peers->peer;

  310.     if (ur->sockaddr) {
  311.         peer[0].sockaddr = ur->sockaddr;
  312.         peer[0].socklen = ur->socklen;
  313.         peer[0].name = ur->host;
  314.         peer[0].weight = 1;
  315.         peer[0].effective_weight = 1;
  316.         peer[0].current_weight = 0;
  317.         peer[0].max_fails = 1;
  318.         peer[0].fail_timeout = 10;

  320.     } else {

  322.         for (i = 0; i < ur->naddrs; i++) {

  324.             socklen = ur->addrs[i].socklen;

  326.             sockaddr = ngx_palloc(r->pool, socklen);
  327.             if (sockaddr == NULL) {
  328.                 return NGX_ERROR;
  329.             }

  331.             ngx_memcpy(sockaddr, ur->addrs[i].sockaddr, socklen);

  333.             switch (sockaddr->sa_family) {
  334. #if (NGX_HAVE_INET6)
  335.             case AF_INET6:
  336.                 ((struct sockaddr_in6 *) sockaddr)->sin6_port = htons(ur->port);
  337.                 break;
  338. #endif
  339.             default: /* AF_INET */
  340.                 ((struct sockaddr_in *) sockaddr)->sin_port = htons(ur->port);
  341.             }

  343.             p = ngx_pnalloc(r->pool, NGX_SOCKADDR_STRLEN);
  344.             if (p == NULL) {
  345.                 return NGX_ERROR;
  346.             }

  348.             len = ngx_sock_ntop(sockaddr, socklen, p, NGX_SOCKADDR_STRLEN, 1);

  350.             peer[i].sockaddr = sockaddr;
  351.             peer[i].socklen = socklen;
  352.             peer[i].name.len = len;
  353.             peer[i].name.data = p;
  354.             peer[i].weight = 1;
  355.             peer[i].effective_weight = 1;
  356.             peer[i].current_weight = 0;
  357.             peer[i].max_fails = 1;
  358.             peer[i].fail_timeout = 10;
  359.         }
  360.     }

  362.     rrp->peers = peers;
  363.     rrp->current = 0;

  365.     if (rrp->peers->number <= 8 * sizeof(uintptr_t)) {
  366.         rrp->tried = &rrp->data;
  367.         rrp->data = 0;

  369.     } else {
  370.         n = (rrp->peers->number + (8 * sizeof(uintptr_t) - 1))
  371.                 / (8 * sizeof(uintptr_t));

  373.         rrp->tried = ngx_pcalloc(r->pool, n * sizeof(uintptr_t));
  374.         if (rrp->tried == NULL) {
  375.             return NGX_ERROR;
  376.         }
  377.     }

  379.     r->upstream->peer.get = ngx_http_upstream_get_round_robin_peer;
  380.     r->upstream->peer.free = ngx_http_upstream_free_round_robin_peer;
  381.     r->upstream->peer.tries = ngx_http_upstream_tries(rrp->peers);
  382. #if (NGX_HTTP_SSL)
  383.     r->upstream->peer.set_session = ngx_http_upstream_empty_set_session;
  384.     r->upstream->peer.save_session = ngx_http_upstream_empty_save_session;
  385. #endif

  387.     return NGX_OK;
  388. }


  391. ngx_int_t
  392. ‌ngx_http_upstream_get_round_robin_peer(ngx_peer_connection_t *pc, void *data)
  393. {
  394.     ngx_http_upstream_rr_peer_data_t  *rrp = data;

  396.     ngx_int_t                      rc;
  397.     ngx_uint_t                     i, n;
  398.     ngx_http_upstream_rr_peer_t   *peer;
  399.     ngx_http_upstream_rr_peers_t  *peers;

  401.     ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0,
  402.                    "get rr peer, try: %ui", pc->tries);

  404.     pc->cached = 0;
  405.     pc->connection = NULL;

  407.     peers = rrp->peers;

  409.     /* ngx_lock_mutex(peers->mutex); */

  411.     if (peers->single) {
  412.         peer = &peers->peer[0];

  414.         if (peer->down) {
  415.             goto failed;
  416.         }

  418.     } else {

  420.         /* there are several peers */

  422.         peer = ngx_http_upstream_get_peer(rrp);

  424.         if (peer == NULL) {
  425.             goto failed;
  426.         }

  428.         ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
  429.                        "get rr peer, current: %ui %i",
  430.                        rrp->current, peer->current_weight);
  431.     }

  433.     pc->sockaddr = peer->sockaddr;
  434.     pc->socklen = peer->socklen;
  435.     pc->name = &peer->name;

  437.     /* ngx_unlock_mutex(peers->mutex); */

  439.     return NGX_OK;

  441. failed:

  443.     if (peers->next) {

  445.         /* ngx_unlock_mutex(peers->mutex); */

  447.         ngx_log_debug0(NGX_LOG_DEBUG_HTTP, pc->log, 0, "backup servers");

  449.         rrp->peers = peers->next;

  451.         n = (rrp->peers->number + (8 * sizeof(uintptr_t) - 1))
  452.                 / (8 * sizeof(uintptr_t));

  454.         for (i = 0; i < n; i++) {
  455.              rrp->tried[i] = 0;
  456.         }

  458.         rc = ngx_http_upstream_get_round_robin_peer(pc, rrp);

  460.         if (rc != NGX_BUSY) {
  461.             return rc;
  462.         }

  464.         /* ngx_lock_mutex(peers->mutex); */
  465.     }

  467.     /* all peers failed, mark them as live for quick recovery */

  469.     for (i = 0; i < peers->number; i++) {
  470.         peers->peer[i].fails = 0;
  471.     }

  473.     /* ngx_unlock_mutex(peers->mutex); */

  475.     pc->name = peers->name;

  477.     return NGX_BUSY;
  478. }


  481. static ngx_http_upstream_rr_peer_t *
  482. ‌ngx_http_upstream_get_peer(ngx_http_upstream_rr_peer_data_t *rrp)
  483. {
  484.     time_t                        now;
  485.     uintptr_t                     m;
  486.     ngx_int_t                     total;
  487.     ngx_uint_t                    i, n;
  488.     ngx_http_upstream_rr_peer_t  *peer, *best;

  490.     now = ngx_time();

  492.     best = NULL;
  493.     total = 0;

  495.     for (i = 0; i < rrp->peers->number; i++) {

  497.         n = i / (8 * sizeof(uintptr_t));
  498.         m = (uintptr_t) 1 << i % (8 * sizeof(uintptr_t));

  500.         if (rrp->tried[n] & m) {
  501.             continue;
  502.         }

  504.         peer = &rrp->peers->peer[i];

  506.         if (peer->down) {
  507.             continue;
  508.         }

  510.         if (peer->max_fails
  511.             && peer->fails >= peer->max_fails
  512.             && now - peer->checked <= peer->fail_timeout)
  513.         {
  514.             continue;
  515.         }

  517.         peer->current_weight += peer->effective_weight;
  518.         total += peer->effective_weight;

  520.         if (peer->effective_weight < peer->weight) {
  521.             peer->effective_weight++;
  522.         }

  524.         if (best == NULL || peer->current_weight > best->current_weight) {
  525.             best = peer;
  526.         }
  527.     }

  529.     if (best == NULL) {
  530.         return NULL;
  531.     }

  533.     i = best - &rrp->peers->peer[0];

  535.     rrp->current = i;

  537.     n = i / (8 * sizeof(uintptr_t));
  538.     m = (uintptr_t) 1 << i % (8 * sizeof(uintptr_t));

  540.     rrp->tried[n] |= m;

  542.     best->current_weight -= total;

  544.     if (now - best->checked > best->fail_timeout) {
  545.         best->checked = now;
  546.     }

  548.     return best;
  549. }


  552. void
  553. ‌ngx_http_upstream_free_round_robin_peer(ngx_peer_connection_t *pc, void *data,
  554.     ngx_uint_t state)
  555. {
  556.     ngx_http_upstream_rr_peer_data_t  *rrp = data;

  558.     time_t                       now;
  559.     ngx_http_upstream_rr_peer_t  *peer;

  561.     ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
  562.                    "free rr peer %ui %ui", pc->tries, state);

  564.     /* TODO: NGX_PEER_KEEPALIVE */

  566.     if (rrp->peers->single) {
  567.         pc->tries = 0;
  568.         return;
  569.     }

  571.     peer = &rrp->peers->peer[rrp->current];

  573.     if (state & NGX_PEER_FAILED) {
  574.         now = ngx_time();

  576.         /* ngx_lock_mutex(rrp->peers->mutex); */

  578.         peer->fails++;
  579.         peer->accessed = now;
  580.         peer->checked = now;

  582.         if (peer->max_fails) {
  583.             peer->effective_weight -= peer->weight / peer->max_fails;
  584.         }

  586.         ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
  587.                        "free rr peer failed: %ui %i",
  588.                        rrp->current, peer->effective_weight);

  590.         if (peer->effective_weight < 0) {
  591.             peer->effective_weight = 0;
  592.         }

  594.         /* ngx_unlock_mutex(rrp->peers->mutex); */

  596.     } else {

  598.         /* mark peer live if check passed */

  600.         if (peer->accessed < peer->checked) {
  601.             peer->fails = 0;
  602.         }
  603.     }

  605.     if (pc->tries) {
  606.         pc->tries--;
  607.     }

  609.     /* ngx_unlock_mutex(rrp->peers->mutex); */
  610. }


  613. #if (NGX_HTTP_SSL)

  615. ngx_int_t
  616. ‌ngx_http_upstream_set_round_robin_peer_session(ngx_peer_connection_t *pc,
  617.     void *data)
  618. {
  619.     ngx_http_upstream_rr_peer_data_t  *rrp = data;

  621.     ngx_int_t                     rc;
  622.     ngx_ssl_session_t            *ssl_session;
  623.     ngx_http_upstream_rr_peer_t  *peer;

  625.     peer = &rrp->peers->peer[rrp->current];

  627.     /* TODO: threads only mutex */
  628.     /* ngx_lock_mutex(rrp->peers->mutex); */

  630.     ssl_session = peer->ssl_session;

  632.     rc = ngx_ssl_set_session(pc->connection, ssl_session);

  634.     ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0,
  635.                    "set session: %p", ssl_session);

  637.     /* ngx_unlock_mutex(rrp->peers->mutex); */

  639.     return rc;
  640. }


  643. void
  644. ‌ngx_http_upstream_save_round_robin_peer_session(ngx_peer_connection_t *pc,
  645.     void *data)
  646. {
  647.     ngx_http_upstream_rr_peer_data_t  *rrp = data;

  649.     ngx_ssl_session_t            *old_ssl_session, *ssl_session;
  650.     ngx_http_upstream_rr_peer_t  *peer;

  652.     ssl_session = ngx_ssl_get_session(pc->connection);

  654.     if (ssl_session == NULL) {
  655.         return;
  656.     }

  658.     ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0,
  659.                    "save session: %p", ssl_session);

  661.     peer = &rrp->peers->peer[rrp->current];

  663.     /* TODO: threads only mutex */
  664.     /* ngx_lock_mutex(rrp->peers->mutex); */

  666.     old_ssl_session = peer->ssl_session;
  667.     peer->ssl_session = ssl_session;

  669.     /* ngx_unlock_mutex(rrp->peers->mutex); */

  671.     if (old_ssl_session) {

  673.         ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0,
  674.                        "old session: %p", old_ssl_session);

  676.         /* TODO: may block */

  678.         ngx_ssl_free_session(old_ssl_session);
  679.     }
  680. }


  683. static ngx_int_t
  684. ‌ngx_http_upstream_empty_set_session(ngx_peer_connection_t *pc, void *data)
  685. {
  686.     return NGX_OK;
  687. }


  690. static void
  691. ‌ngx_http_upstream_empty_save_session(ngx_peer_connection_t *pc, void *data)
  692. {
  693.     return;
  694. }

  696. #endif

One Level Up Top Level