Merge pull request #282 from Mashape/feature/path-resolver

[feature] Path Resolver

database/migrations/cassandra/2015-05-22-235608_plugins_fix.lua → database/migrations/cassandra/2015-05-22-235608_0.3.0.lua

@@ -1,15 +1,23 @@
 local Migration = {
-  name = "2015-05-22-235608_plugins_fix",
+  name = "2015-05-22-235608_0.3.0",
 
   up = function(options)
     return [[
       CREATE INDEX IF NOT EXISTS keyauth_consumer_id ON keyauth_credentials(consumer_id);
       CREATE INDEX IF NOT EXISTS basicauth_consumer_id ON basicauth_credentials(consumer_id);
+
+      ALTER TABLE apis ADD path text;
+      ALTER TABLE apis ADD strip_path boolean;
+      CREATE INDEX IF NOT EXISTS apis_path ON apis(path);
     ]]
   end,
 
   down = function(options)
     return [[
+      DROP INDEX apis_path;
+      ALTER TABLE apis DROP path;
+      ALTER TABLE apis DROP strip_path;
+
       DROP INDEX keyauth_consumer_id;
       DROP INDEX basicauth_consumer_id;
     ]]

kong-0.3.0-1.rockspec

@@ -70,7 +70,6 @@ build = {
     ["kong.resolver.access"] = "kong/resolver/access.lua",
     ["kong.resolver.header_filter"] = "kong/resolver/header_filter.lua",
     ["kong.resolver.certificate"] = "kong/resolver/certificate.lua",
-    ["kong.resolver.resolver_util"] = "kong/resolver/resolver_util.lua",
 
     ["kong.dao.error"] = "kong/dao/error.lua",
     ["kong.dao.schemas_validation"] = "kong/dao/schemas_validation.lua",

kong/constants.lua

@@ -37,14 +37,6 @@ return {
     RATELIMIT_LIMIT = "X-RateLimit-Limit",
     RATELIMIT_REMAINING = "X-RateLimit-Remaining"
   },
-  CACHE = {
-    APIS = "apis",
-    CONSUMERS = "consumers",
-    PLUGINS_CONFIGURATIONS = "plugins_configurations",
-    BASICAUTH_CREDENTIAL = "basicauth_credentials",
-    KEYAUTH_CREDENTIAL = "keyauth_credentials",
-    SSL = "ssl"
-  },
   AUTHENTICATION = {
     QUERY = "query",
     BASIC = "basic",

kong/dao/cassandra/apis.lua

@@ -9,13 +9,13 @@ function Apis:new(properties)
   self._schema = apis_schema
   self._queries = {
     insert = {
-      args_keys = { "id", "name", "public_dns", "target_url", "created_at" },
-      query = [[ INSERT INTO apis(id, name, public_dns, target_url, created_at)
-                  VALUES(?, ?, ?, ?, ?); ]]
+      args_keys = { "id", "name", "public_dns", "path", "strip_path", "target_url", "created_at" },
+      query = [[ INSERT INTO apis(id, name, public_dns, path, strip_path, target_url, created_at)
+                  VALUES(?, ?, ?, ?, ?, ?, ?); ]]
     },
     update = {
-      args_keys = { "name", "public_dns", "target_url", "id" },
-      query = [[ UPDATE apis SET name = ?, public_dns = ?, target_url = ? WHERE id = ?; ]]
+      args_keys = { "name", "public_dns", "path", "strip_path", "target_url", "id" },
+      query = [[ UPDATE apis SET name = ?, public_dns = ?, path = ?, strip_path = ?, target_url = ? WHERE id = ?; ]]
     },
     select = {
       query = [[ SELECT * FROM apis %s; ]]
@@ -33,6 +33,10 @@ function Apis:new(properties)
         args_keys = { "name" },
         query = [[ SELECT id FROM apis WHERE name = ?; ]]
       },
+      path = {
+        args_keys = { "path" },
+        query = [[ SELECT id FROM apis WHERE path = ?; ]]
+      },
       public_dns = {
         args_keys = { "public_dns" },
         query = [[ SELECT id FROM apis WHERE public_dns = ?; ]]
@@ -43,6 +47,21 @@ function Apis:new(properties)
   Apis.super.new(self, properties)
 end
 
+function Apis:find_all()
+  local apis = {}
+  for _, rows, page, err in Apis.super._execute_kong_query(self, self._queries.select.query, nil, {auto_paging=true}) do
+    if err then
+      return nil, err
+    end
+
+    for _, row in ipairs(rows) do
+      table.insert(apis, row)
+    end
+  end
+
+  return apis
+end
+
 -- @override
 function Apis:delete(api_id)
   local ok, err = Apis.super.delete(self, api_id)

kong/dao/schemas/apis.lua

@@ -1,4 +1,5 @@
 local url = require "socket.url"
+local stringy = require "stringy"
 local constants = require "kong.constants"
 
 local function validate_target_url(value)
@@ -16,11 +17,45 @@ local function validate_target_url(value)
   return false, "Invalid target URL"
 end
 
+local function check_public_dns_and_path(value, api_t)
+  local public_dns = type(api_t.public_dns) == "string" and stringy.strip(api_t.public_dns) or ""
+  local path = type(api_t.path) == "string" and stringy.strip(api_t.path) or ""
+
+  if path == "" and public_dns == "" then
+    return false, "At least a 'public_dns' or a 'path' must be specified"
+  end
+
+  return true
+end
+
+local function check_path(path, api_t)
+  local valid, err = check_public_dns_and_path(path, api_t)
+  if not valid then
+    return false, err
+  end
+
+  if path then
+    -- Prefix with `/` for the sake of consistency
+    local has_slash = string.match(path, "^/")
+    if not has_slash then api_t.path = "/"..path end
+    -- Check if characters are in RFC 3986 unreserved list
+    local is_alphanumeric = string.match(api_t.path, "^/[%w%.%-%_~]*/?$")
+    if not is_alphanumeric then
+      return false, "path must only contain alphanumeric and '. -, _, ~' characters"
+    end
+  end
+
+  return true
+end
+
 return {
   id = { type = constants.DATABASE_TYPES.ID },
   name = { type = "string", unique = true, queryable = true, default = function(api_t) return api_t.public_dns end },
-  public_dns = { type = "string", required = true, unique = true, queryable = true,
-                 regex = "([a-zA-Z0-9-]+(\\.[a-zA-Z0-9-]+)*)" },
+  public_dns = { type = "string", unique = true, queryable = true,
+                func = check_public_dns_and_path,
+                regex = "([a-zA-Z0-9-]+(\\.[a-zA-Z0-9-]+)*)" },
+  path = { type = "string", queryable = true, unique = true, func = check_path },
+  strip_path = { type = "boolean" },
   target_url = { type = "string", required = true, func = validate_target_url },
   created_at = { type = constants.DATABASE_TYPES.TIMESTAMP }
 }

kong/dao/schemas/consumers.lua

@@ -2,13 +2,13 @@ local stringy = require "stringy"
 local constants = require "kong.constants"
 
 local function check_custom_id_and_username(value, consumer_t)
-  local custom_id = consumer_t.custom_id
-  local username = consumer_t.username
+  local username = type(consumer_t.username) == "string" and stringy.strip(consumer_t.username) or ""
+  local custom_id = type(consumer_t.custom_id) == "string" and stringy.strip(consumer_t.custom_id) or ""
 
-  if (custom_id == nil or type(custom_id) == "string" and stringy.strip(custom_id) == "")
-    and (username == nil or type(username) == "string" and stringy.strip(username) == "") then
-      return false, "At least a 'custom_id' or a 'username' must be specified"
+  if custom_id == "" and username == "" then
+    return false, "At least a 'custom_id' or a 'username' must be specified"
   end
+
   return true
 end
 

kong/kong.lua

@@ -40,7 +40,7 @@ local _M = {}
 local function load_plugin_conf(api_id, consumer_id, plugin_name)
   local cache_key = cache.plugin_configuration_key(plugin_name, api_id, consumer_id)
 
-  local plugin = cache.get_and_set(cache_key, function()
+  local plugin = cache.get_or_set(cache_key, function()
     local rows, err = dao.plugins_configurations:find_by_keys {
         api_id = api_id,
         consumer_id = consumer_id ~= nil and consumer_id or constants.DATABASE_NULL_ID,

kong/plugins/basicauth/access.lua

@@ -74,7 +74,7 @@ function _M.execute(conf)
 
   -- Make sure we are not sending an empty table to find_by_keys
   if username then
-    credential = cache.get_and_set(cache.basicauth_credential_key(username), function()
+    credential = cache.get_or_set(cache.basicauth_credential_key(username), function()
       local credentials, err = dao.basicauth_credentials:find_by_keys { username = username }
       local result
       if err then
@@ -92,7 +92,7 @@ function _M.execute(conf)
   end
 
   -- Retrieve consumer
-  local consumer = cache.get_and_set(cache.consumer_key(credential.consumer_id), function()
+  local consumer = cache.get_or_set(cache.consumer_key(credential.consumer_id), function()
     local result, err = dao.consumers:find_one(credential.consumer_id)
     if err then
       return responses.send_HTTP_INTERNAL_SERVER_ERROR(err)

kong/plugins/keyauth/access.lua

@@ -124,7 +124,7 @@ function _M.execute(conf)
 
     -- Make sure we are not sending an empty table to find_by_keys
     if key then
-      credential = cache.get_and_set(cache.keyauth_credential_key(key), function()
+      credential = cache.get_or_set(cache.keyauth_credential_key(key), function()
         local credentials, err = dao.keyauth_credentials:find_by_keys { key = key }
         local result
         if err then
@@ -145,7 +145,7 @@ function _M.execute(conf)
   end
 
   -- Retrieve consumer
-  local consumer = cache.get_and_set(cache.consumer_key(credential.consumer_id), function()
+  local consumer = cache.get_or_set(cache.consumer_key(credential.consumer_id), function()
     local result, err = dao.consumers:find_one(credential.consumer_id)
     if err then
       return responses.send_HTTP_INTERNAL_SERVER_ERROR(err)

kong/plugins/ssl/certificate.lua

@@ -6,7 +6,7 @@ function _M.execute(conf)
   local ssl = require "ngx.ssl"
   ssl.clear_certs()
 
-  local data = cache.get_and_set(cache.ssl_data(ngx.ctx.api.id), function()
+  local data = cache.get_or_set(cache.ssl_data(ngx.ctx.api.id), function()
     local result = {
       cert_der = ngx.decode_base64(conf._cert_der_cache),
       key_der = ngx.decode_base64(conf._key_der_cache)

kong/resolver/access.lua

@@ -1,7 +1,7 @@
 local url = require "socket.url"
+local cache = require "kong.tools.database_cache"
 local constants = require "kong.constants"
 local responses = require "kong.tools.responses"
-local resolver_util = require "kong.resolver.resolver_util"
 
 local _M = {}
 
@@ -32,31 +32,106 @@ local function get_host_from_url(val)
   return parsed_url.host..(port and ":"..port or "")
 end
 
--- Retrieve the API from the Host that has been requested
-function _M.execute(conf)
-  local hosts_headers = {}
+-- Find an API from a request made to nginx. Either from one of the Host or X-Host-Override headers
+-- matching the API's `public_dns`, either from the `request_uri` matching the API's `path`.
+--
+-- To perform this, we need to query _ALL_ APIs in memory. It is the only way to compare the `request_uri`
+-- as a regex to the values set in DB. We keep APIs in the database cache for a longer time than usual.
+-- @see https://github.com/Mashape/kong/issues/15 for an improvement on this.
+--
+-- @param  `request_uri` The URI for this request.
+-- @return `err`         Any error encountered during the retrieval.
+-- @return `api`         The retrieved API, if any.
+-- @return `hosts`       The list of headers values found in Host and X-Host-Override.
+-- @return `request_uri` The URI for this request.
+-- @return `by_path`     If the API was retrieved by path, will be true, false if by Host.
+local function find_api(request_uri)
+  local retrieved_api
+
+  -- retrieve all APIs
+  local apis_dics, err = cache.get_or_set("ALL_APIS_BY_DIC", function()
+    local apis, err = dao.apis:find_all()
+    if err then
+      return nil, err
+    end
+
+    -- build dictionnaries of public_dns:api and path:apis for efficient lookup.
+    local dns_dic, path_dic = {}, {}
+    for _, api in ipairs(apis) do
+      if api.public_dns then
+        dns_dic[api.public_dns] = api
+      end
+      if api.path then
+        path_dic[api.path] = api
+      end
+    end
+    return {dns = dns_dic, path = path_dic}
+  end, 60) -- 60 seconds cache
+
+  if err then
+    return err
+  end
+
+  -- find by Host header
+  local all_hosts = {}
   for _, header_name in ipairs({"Host", constants.HEADERS.HOST_OVERRIDE}) do
-    local host = ngx.req.get_headers()[header_name]
-    if type(host) == "string" then -- single header
-      table.insert(hosts_headers, host)
-    elseif type(host) == "table" then -- multiple headers
-      for _, v in ipairs(host) do
-        table.insert(hosts_headers, v)
+    local hosts = ngx.req.get_headers()[header_name]
+    if hosts then
+      if type(hosts) == "string" then
+        hosts = {hosts}
+      end
+      -- for all values of this header, try to find an API using the apis_by_dns dictionnary
+      for _, host in ipairs(hosts) do
+        table.insert(all_hosts, host)
+        if apis_dics.dns[host] then
+          retrieved_api = apis_dics.dns[host]
+          break
+        end
       end
     end
   end
 
-  -- Find the API
-  local api, err = resolver_util.find_api(hosts_headers)
+  -- If it was found by Host, return.
+  if retrieved_api then
+    return nil, retrieved_api, all_hosts
+  end
+
+  -- Otherwise, we look for it by path. We have to loop over all APIs and compare the requested URI.
+  for path, api in pairs(apis_dics.path) do
+    local m, err = ngx.re.match(request_uri, "^"..path)
+    if err then
+      ngx.log(ngx.ERR, "[resolver] error matching requested path: "..err)
+    elseif m then
+      retrieved_api = api
+    end
+  end
 
+  -- Return the retrieved_api or nil
+  return nil, retrieved_api, all_hosts, true
+end
+
+-- Retrieve the API from the Host that has been requested
+function _M.execute(conf)
+  local request_uri = ngx.var.request_uri
+  local err, api, hosts, by_path = find_api(request_uri)
   if err then
     return responses.send_HTTP_INTERNAL_SERVER_ERROR(err)
   elseif not api then
-    return responses.send_HTTP_NOT_FOUND("API not found with Host: "..table.concat(hosts_headers, ","))
+    return responses.send_HTTP_NOT_FOUND {
+      message = "API not found with these values",
+      public_dns = hosts,
+      path = request_uri
+    }
+  end
+
+  -- If API was retrieved by path
+  if by_path and api.strip_path then
+    -- All paths are '/path', so let's replace it with a single '/'
+    request_uri = string.gsub(request_uri, api.path, "/")
   end
 
   -- Setting the backend URL for the proxy_pass directive
-  ngx.var.backend_url = get_backend_url(api)..ngx.var.request_uri
+  ngx.var.backend_url = get_backend_url(api)..request_uri
   ngx.req.set_header("Host", get_host_from_url(ngx.var.backend_url))
 
   ngx.ctx.api = api

kong/resolver/certificate.lua

@@ -1,12 +1,33 @@
-local resolver_util = require("kong.resolver.resolver_util")
+local cache = require "kong.tools.database_cache"
+local stringy = require "stringy"
 
 local _M = {}
 
+local function find_api(hosts)
+  local retrieved_api, err
+  for _, host in ipairs(hosts) do
+    local sanitized_host = stringy.split(host, ":")[1]
+
+    retrieved_api, err = cache.get_or_set(cache.api_key(sanitized_host), function()
+      local apis, err = dao.apis:find_by_keys { public_dns = sanitized_host }
+      if err then
+        return nil, err
+      elseif apis and #apis == 1 then
+        return apis[1]
+      end
+    end)
+
+    if err or retrieved_api then
+      return retrieved_api, err
+    end
+  end
+end
+
 function _M.execute(conf)
   local ssl = require "ngx.ssl"
   local server_name = ssl.server_name()
   if server_name then -- Only support SNI requests
-    local api, err = resolver_util.find_api({server_name})
+    local api, err = find_api({server_name})
     if not err and api then
       ngx.ctx.api = api
     end