Haproxy map support to speed up vhost to backend lookup. (#229)

* Haproxy map support to speed up vhost to backend lookup. (#1)

* added support for haproxy maps with the help of --haproxy-map flag

* Updated read me for haproxy maps

* updated readme

* Updated readme

* changed list comparison as it was not compatible with python3

* added lua module to expose vhostmaps and changed test cases to reflect it. Changed haproxy_dir to use a variable instead of extracting the value every time

* changed to get path separator in getvhostmap.lua from library call instead of assuming a path separator

* updated config and test_cases to use vhostmap lua module, irrespective of haproxy-map flag, and changed lua module to handle the same, fixed few formatting issues with readme

* updated readme to reflect the change in _haproxy_getvhostmap behaviour

* fix for multiple vhosts with path which ensures it checks if path and auth are present before proceding with multiple vhosts acl

* regenerated the docs and added test case for haproxy map with http to https redirection

* fixed issue where acl for http to https was not getting generated when no path and no auth where specified and modified the test case to catch the same

Longhelp.md

@@ -23,8 +23,8 @@ usage: marathon_lb.py [-h] [--longhelp] [--marathon MARATHON [MARATHON ...]]
                       [--haproxy-config HAPROXY_CONFIG] [--group GROUP]
                       [--command COMMAND] [--sse] [--health-check]
                       [--lru-cache-capacity LRU_CACHE_CAPACITY]
-                      [--dont-bind-http-https] [--ssl-certs SSL_CERTS]
-                      [--skip-validation] [--dry]
+                      [--haproxy-map] [--dont-bind-http-https]
+                      [--ssl-certs SSL_CERTS] [--skip-validation] [--dry]
                       [--min-serv-port-ip-per-task MIN_SERV_PORT_IP_PER_TASK]
                       [--max-serv-port-ip-per-task MAX_SERV_PORT_IP_PER_TASK]
                       [--syslog-socket SYSLOG_SOCKET]
@@ -65,6 +65,8 @@ optional arguments:
                         LRU cache size (in number of items). This should be at
                         least as large as the number of tasks exposed via
                         marathon-lb. (default: 1000)
+  --haproxy-map         Use HAProxy maps for domain name to backendmapping.
+                        (default: False)
   --dont-bind-http-https
                         Don't bind to HTTP and HTTPS frontends. (default:
                         False)
@@ -370,6 +372,7 @@ global
   server-state-base /var/state/haproxy/
   lua-load /marathon-lb/getpids.lua
   lua-load /marathon-lb/getconfig.lua
+  lua-load /marathon-lb/getvhostmap.lua
 defaults
   load-server-state-from-file global
   log               global
@@ -395,6 +398,8 @@ listen stats
   monitor-uri /_haproxy_health_check
   acl getpid path /_haproxy_getpids
   http-request use-service lua.getpids if getpid
+  acl getvhostmap path /_haproxy_getvhostmap
+  http-request use-service lua.getvhostmap if getvhostmap
   acl getconfig path /_haproxy_getconfig
   http-request use-service lua.getconfig if getconfig
 ```
@@ -805,6 +810,58 @@ glues the acl names to the appropriate backend
   http-request auth realm "{realm}" if host_{cleanedUpHostname} path_{backend} !auth_{cleanedUpHostname}
   use_backend {backend} if host_{cleanedUpHostname} path_{backend}
 ```
+## `HAPROXY_MAP_HTTPS_FRONTEND_ACL`
+  *Overridable*
+
+Specified as `HAPROXY_MAP_HTTPS_FRONTEND_ACL` template or with label `HAPROXY_{n}_MAP_HTTPS_FRONTEND_ACL`.
+
+The ACL that performs the SNI based hostname matching
+for the `HAPROXY_HTTPS_FRONTEND_HEAD` template using haproxy maps
+
+
+**Default template for `HAPROXY_MAP_HTTPS_FRONTEND_ACL`:**
+```
+  use_backend %[ssl_fc_sni,lower,map_dom({haproxy_dir}/domain2backend.map)]
+```
+## `HAPROXY_MAP_HTTP_FRONTEND_ACL`
+  *Overridable*
+
+Specified as `HAPROXY_MAP_HTTP_FRONTEND_ACL` template or with label `HAPROXY_{n}_MAP_HTTP_FRONTEND_ACL`.
+
+The ACL that glues a backend to the corresponding virtual host
+of the `HAPROXY_HTTP_FRONTEND_HEAD` using haproxy maps.
+
+
+**Default template for `HAPROXY_MAP_HTTP_FRONTEND_ACL`:**
+```
+  use_backend %[req.hdr(host),lower,map_dom({haproxy_dir}/domain2backend.map)]
+```
+## `HAPROXY_MAP_HTTP_FRONTEND_ACL_ONLY`
+  *Overridable*
+
+Specified as `HAPROXY_MAP_HTTP_FRONTEND_ACL_ONLY` template or with label `HAPROXY_{n}_MAP_HTTP_FRONTEND_ACL_ONLY`.
+
+Define the ACL matching a particular hostname, This is useful only in the case
+ of multiple vhosts routing to the same backend in haproxy map.
+
+
+**Default template for `HAPROXY_MAP_HTTP_FRONTEND_ACL_ONLY`:**
+```
+  use_backend %[req.hdr(host),lower,map_dom({haproxy_dir}/domain2backend.map)]
+```
+## `HAPROXY_MAP_HTTP_FRONTEND_APPID_ACL`
+  *Overridable*
+
+Specified as `HAPROXY_MAP_HTTP_FRONTEND_APPID_ACL` template or with label `HAPROXY_{n}_MAP_HTTP_FRONTEND_APPID_ACL`.
+
+The ACL that glues a backend to the corresponding app
+of the `HAPROXY_HTTP_FRONTEND_APPID_HEAD` using haproxy maps.
+
+
+**Default template for `HAPROXY_MAP_HTTP_FRONTEND_APPID_ACL`:**
+```
+  use_backend %[req.hdr(x-marathon-app-id),lower,map_str({haproxy_dir}/domain2backend.map)]
+```
 ## `HAPROXY_TCP_BACKEND_ACL_ALLOW_DENY`
   *Global*
 

README.md

@@ -150,6 +150,14 @@ You can skip the configuration file validation (via calling HAProxy service) pro
 $ ./marathon_lb.py --marathon http://localhost:8080 --group external --skip-validation
 ```
 
+### Using HAProxy maps for backend lookup.
+You can use HAProxy maps to speed up web application (vhosts) to backend lookup. This is very useful for large installations where the traditional vhost to backend rules comparison takes considerable time since it sequentially compares each rule. HAProxy map creates a hash based lookup table so its fast compared to the other approach, this is supported in marathon-lb using `--haproxy-map` flag.
+
+```console 
+$ ./marathon_lb.py --marathon http://localhost:8080 --group external --haproxy-map
+```
+Currently it creates a lookup dictionary only for host header (both HTTP and HTTPS) and X-Marathon-App-Id header. But for path based routing and auth, it uses the usual backend rules comparison.
+
 ### API endpoints
 
 Marathon-lb exposes a few endpoints on port 9090 (by default). They are:
@@ -160,6 +168,7 @@ Marathon-lb exposes a few endpoints on port 9090 (by default). They are:
 | `:9090/haproxy?stats;csv`     | This is a CSV version of the stats above, which can be consumed by other tools. For example, it's used in the [`zdd.py`](zdd.py) script.                                                                                                                                                        |
 | `:9090/_haproxy_health_check` | HAProxy health check endpoint. Returns `200 OK` if HAProxy is healthy.                                                                                                                                                                                                                                                    |
 | `:9090/_haproxy_getconfig`    | Returns the HAProxy config file as it was when HAProxy was started. Implemented in [`getconfig.lua`](getconfig.lua).                                                                                                                                                                                                      |
+| `:9090/_haproxy_getvhostmap`    | Returns the HAProxy vhost to backend map. This endpoint returns HAProxy map file only when `--haproxy-map` flag is enabled, it returns a empty string otherwise. Implemented in [`getvhostmap.lua`](getvhostmap.lua).                                                                                                                                                                                                      |
 | `:9090/_haproxy_getpids`      | Returns the PIDs for all HAProxy instances within the current process namespace. This literally returns `$(pidof haproxy)`. Implemented in [`getpids.lua`](getpids.lua). This is also used by the [`zdd.py`](zdd.py) script to determine if connections have finished draining during a deploy. |
 
 

config.py

@@ -59,6 +59,7 @@ def load(self):
   server-state-base /var/state/haproxy/
   lua-load /marathon-lb/getpids.lua
   lua-load /marathon-lb/getconfig.lua
+  lua-load /marathon-lb/getvhostmap.lua
 defaults
   load-server-state-from-file global
   log               global
@@ -84,6 +85,8 @@ def load(self):
   monitor-uri /_haproxy_health_check
   acl getpid path /_haproxy_getpids
   http-request use-service lua.getpids if getpid
+  acl getvhostmap path /_haproxy_getvhostmap
+  http-request use-service lua.getvhostmap if getvhostmap
   acl getconfig path /_haproxy_getconfig
   http-request use-service lua.getconfig if getconfig
 ''',
@@ -220,6 +223,17 @@ def load(self):
                            description='''\
 The ACL that glues a backend to the corresponding virtual host
 of the `HAPROXY_HTTP_FRONTEND_HEAD`
+'''))
+
+        self.add_template(
+            ConfigTemplate(name='MAP_HTTP_FRONTEND_ACL',
+                           value='''\
+  use_backend %[req.hdr(host),lower,map_dom({haproxy_dir}/domain2backend.map)]
+''',
+                           overridable=True,
+                           description='''\
+The ACL that glues a backend to the corresponding virtual host
+of the `HAPROXY_HTTP_FRONTEND_HEAD` using haproxy maps.
 '''))
 
         self.add_template(
@@ -250,6 +264,17 @@ def load(self):
 vhosts routing to the same backend.
 '''))
 
+        self.add_template(
+            ConfigTemplate(name='MAP_HTTP_FRONTEND_ACL_ONLY',
+                           value='''\
+  use_backend %[req.hdr(host),lower,map_dom({haproxy_dir}/domain2backend.map)]
+''',
+                           overridable=True,
+                           description='''\
+Define the ACL matching a particular hostname, This is useful only in the case
+ of multiple vhosts routing to the same backend in haproxy map.
+'''))
+
         self.add_template(
             ConfigTemplate(name='HTTP_FRONTEND_ROUTING_ONLY',
                            value='''\
@@ -389,6 +414,18 @@ def load(self):
                            description='''\
 The ACL that glues a backend to the corresponding app
 of the `HAPROXY_HTTP_FRONTEND_APPID_HEAD`.
+'''))
+
+        self.add_template(
+            ConfigTemplate(name='MAP_HTTP_FRONTEND_APPID_ACL',
+                           value='''\
+  use_backend %[req.hdr(x-marathon-app-id),lower,\
+map_str({haproxy_dir}/domain2backend.map)]
+''',
+                           overridable=True,
+                           description='''\
+The ACL that glues a backend to the corresponding app
+of the `HAPROXY_HTTP_FRONTEND_APPID_HEAD` using haproxy maps.
 '''))
 
         self.add_template(
@@ -400,6 +437,17 @@ def load(self):
                            description='''\
 The ACL that performs the SNI based hostname matching
 for the `HAPROXY_HTTPS_FRONTEND_HEAD` template.
+'''))
+
+        self.add_template(
+            ConfigTemplate(name='MAP_HTTPS_FRONTEND_ACL',
+                           value='''\
+  use_backend %[ssl_fc_sni,lower,map_dom({haproxy_dir}/domain2backend.map)]
+''',
+                           overridable=True,
+                           description='''\
+The ACL that performs the SNI based hostname matching
+for the `HAPROXY_HTTPS_FRONTEND_HEAD` template using haproxy maps
 '''))
 
         self.add_template(
@@ -874,11 +922,21 @@ def haproxy_http_frontend_acl(self, app):
             return app.labels['HAPROXY_{0}_HTTP_FRONTEND_ACL']
         return self.t['HTTP_FRONTEND_ACL'].value
 
+    def haproxy_map_http_frontend_acl(self, app):
+        if 'HAPROXY_{0}_HTTP_FRONTEND_ACL' in app.labels:
+            return app.labels['HAPROXY_{0}_HTTP_FRONTEND_ACL']
+        return self.t['MAP_HTTP_FRONTEND_ACL'].value
+
     def haproxy_http_frontend_acl_only(self, app):
         if 'HAPROXY_{0}_HTTP_FRONTEND_ACL_ONLY' in app.labels:
             return app.labels['HAPROXY_{0}_HTTP_FRONTEND_ACL_ONLY']
         return self.t['HTTP_FRONTEND_ACL_ONLY'].value
 
+    def haproxy_map_http_frontend_acl_only(self, app):
+        if 'HAPROXY_{0}_HTTP_FRONTEND_ACL_ONLY' in app.labels:
+            return app.labels['HAPROXY_{0}_HTTP_FRONTEND_ACL_ONLY']
+        return self.t['MAP_HTTP_FRONTEND_ACL_ONLY'].value
+
     def haproxy_http_frontend_routing_only(self, app):
         if 'HAPROXY_{0}_HTTP_FRONTEND_ROUTING_ONLY' in app.labels:
             return app.labels['HAPROXY_{0}_HTTP_FRONTEND_ROUTING_ONLY']
@@ -933,11 +991,21 @@ def haproxy_http_frontend_appid_acl(self, app):
             return app.labels['HAPROXY_{0}_HTTP_FRONTEND_APPID_ACL']
         return self.t['HTTP_FRONTEND_APPID_ACL'].value
 
+    def haproxy_map_http_frontend_appid_acl(self, app):
+        if 'HAPROXY_{0}_HTTP_FRONTEND_APPID_ACL' in app.labels:
+            return app.labels['HAPROXY_{0}_HTTP_FRONTEND_APPID_ACL']
+        return self.t['MAP_HTTP_FRONTEND_APPID_ACL'].value
+
     def haproxy_https_frontend_acl(self, app):
         if 'HAPROXY_{0}_HTTPS_FRONTEND_ACL' in app.labels:
             return app.labels['HAPROXY_{0}_HTTPS_FRONTEND_ACL']
         return self.t['HTTPS_FRONTEND_ACL'].value
 
+    def haproxy_map_https_frontend_acl(self, app):
+        if 'HAPROXY_{0}_HTTPS_FRONTEND_ACL' in app.labels:
+            return app.labels['HAPROXY_{0}_HTTPS_FRONTEND_ACL']
+        return self.t['MAP_HTTPS_FRONTEND_ACL'].value
+
     def haproxy_https_frontend_acl_with_path(self, app):
         if 'HAPROXY_{0}_HTTPS_FRONTEND_ACL_WITH_PATH' in app.labels:
             return app.labels['HAPROXY_{0}_HTTPS_FRONTEND_ACL_WITH_PATH']
@@ -1349,9 +1417,15 @@ def __init__(self, name, func, description, perServicePort=True):
 labels.append(Label(name='HTTP_FRONTEND_ACL',
                     func=set_label,
                     description=''))
+labels.append(Label(name='MAP_HTTP_FRONTEND_ACL',
+                    func=set_label,
+                    description=''))
 labels.append(Label(name='HTTP_FRONTEND_ACL_ONLY',
                     func=set_label,
                     description=''))
+labels.append(Label(name='MAP_HTTP_FRONTEND_ACL_ONLY',
+                    func=set_label,
+                    description=''))
 labels.append(Label(name='HTTP_FRONTEND_ROUTING_ONLY',
                     func=set_label,
                     description=''))
@@ -1379,9 +1453,15 @@ def __init__(self, name, func, description, perServicePort=True):
 labels.append(Label(name='HTTP_FRONTEND_APPID_ACL',
                     func=set_label,
                     description=''))
+labels.append(Label(name='MAP_HTTP_FRONTEND_APPID_ACL',
+                    func=set_label,
+                    description=''))
 labels.append(Label(name='HTTPS_FRONTEND_ACL',
                     func=set_label,
                     description=''))
+labels.append(Label(name='MAP_HTTPS_FRONTEND_ACL',
+                    func=set_label,
+                    description=''))
 labels.append(Label(name='HTTPS_FRONTEND_ACL_WITH_AUTH',
                     func=set_label,
                     description=''))

getvhostmap.lua

@@ -0,0 +1,48 @@
+-- A simple Lua script which serves up the HAProxy
+-- vhost to backend map file.
+function check_file_exists(name)
+   local f=io.open(name,"r")
+   if f~=nil then io.close(f) return true else return false end
+end
+
+function read_vhostmap_file(cmdline)
+  local found = false
+  local filename = ''
+  for s in string.gmatch(cmdline, '%g+') do
+    if s == '-f' then
+      found = true
+    elseif found then
+      filename = s
+      sep = package.config:sub(1,1)
+      filename=filename:match("(.*"..sep..")").."domain2backend.map"
+      break
+    end
+  end
+
+  local map = ''
+  if check_file_exists(filename) then
+    local f = io.open(filename, "rb")
+    map = f:read("*all")
+    f:close()
+  else
+    map = ''
+  end
+  return map
+end
+
+function load_vhostmap()
+  local f = io.open('/proc/self/cmdline', "rb")
+  local cmdline = f:read("*all")
+  f:close()
+  return read_vhostmap_file(cmdline)
+end
+
+core.register_service("getvhostmap", "http", function(applet)
+  local haproxy_vhostmap = load_vhostmap()
+  applet:set_status(200)
+  applet:add_header("content-length", string.len(haproxy_vhostmap))
+  applet:add_header("content-type", "text/plain")
+  applet:start_response()
+  applet:send(haproxy_vhostmap)
+end)
+