By adding a new service- an API your app is using- you will be able to track, transform, and analyze the traffic between your app and this API on Apitools.
+
+ You can change the name of your service as well as delete it from here.
+
+
diff --git a/config/nginx.conf b/config/nginx.conf
index 87419d5..1ed5993 100644
--- a/config/nginx.conf
+++ b/config/nginx.conf
@@ -14,6 +14,27 @@ http {
include mime.types;
+ real_ip_header X-Forwarded-For;
+ set_real_ip_from 0.0.0.0/0;
+
+ # set log format for logstash
+ log_format logstash '{ "@timestamp": "$time_iso8601", '
+ '"@fields": { '
+ '"vhost": "$host:$server_port", '
+ '"port": "$server_port", '
+ '"remote_addr": "$remote_addr", '
+ '"remote_user": "$remote_user", '
+ '"body_bytes_sent": "$body_bytes_sent", '
+ '"request_time": "$request_time", '
+ '"status": "$status", '
+ '"request": "$request", '
+ '"hostname": "$hostname", '
+ '"request_method": "$request_method", '
+ '"http_referrer": "$http_referer", '
+ '"request_id": "$http_x_request_id", '
+ '"http_user_agent": "$http_user_agent" } }';
+ access_log /var/nginx/logstash.log logstash;
+
lua_package_path ";;${prefix}lua/?.lua;";
diff --git a/config/supervisor.conf b/config/supervisor.conf
index 3285aa2..6c27d4b 100644
--- a/config/supervisor.conf
+++ b/config/supervisor.conf
@@ -1,3 +1,7 @@
[program:openresty]
command=/opt/openresty/nginx/sbin/nginx -p /var/www/brainslug/ -c config/nginx.conf
autorestart=true
+stdout_logfile_backups=2
+stdout_logfile_maxbytes=50MB
+stderr_logfile_backups=2
+stderr_logfile_maxbytes=50MB
diff --git a/lua/apps/api.lua b/lua/apps/api.lua
index fdec80c..e879e45 100644
--- a/lua/apps/api.lua
+++ b/lua/apps/api.lua
@@ -97,6 +97,7 @@ router:get( '/api/services/:service_id/pipeline' , r('pipelines.show
router:post( '/api/services/:service_id/pipeline' , r('pipelines.update'))
router:get( '/api/traces' , r('traces.index'))
+router:get( '/api/traces/:uuid/find' , r('traces.uuid'))
router:get( '/api/traces/search' , r('traces.search'))
router:get( '/api/traces/search_for_index' , r('traces.search_for_index'))
router:get( '/api/traces/last_id' , r('traces.last_id'))
@@ -112,9 +113,11 @@ router:get( '/api/traces/saved' , r('traces.index_s
router:get( '/api/traces/count' , r('traces.count'))
+router:get( '/api/services/:service_id/traces' , r('service_traces.index'))
router:get( '/api/services/:service_id/traces/search' , r('service_traces.search'))
router:get( '/api/services/:service_id/traces/search_for_index' , r('service_traces.search_for_index'))
router:get( '/api/services/:service_id/traces/count' , r('service_traces.count'))
+router:post( '/api/services/:service_id/traces/:id/redo' , r('traces.redo'))
router:delete( '/api/services/:service_id/traces/:id' , r('traces.delete'))
router:delete( '/api/services/:service_id/traces/all' , r('service_traces.delete_all'))
@@ -198,6 +201,9 @@ router:post( '/api/brain/register' , r('brain.register
router:post( '/api/brain/link' , r('brain.link'))
router:post( '/api/brain/unlink' , r('brain.unlink'))
+router:get( '/api/brain/middleware_specs/search/' , r('brain_middleware_specs.search'))
+router:get( '/api/brain/middleware_specs/:id/' , r('brain_middleware_specs.show'))
+
router:get( '/api/jor/dump/export' , r('backups.export'))
router:post( '/api/jor/dump/import' , r('backups.import'))
router:get( '/api/redis/stats' , r('redis.stats'))
diff --git a/lua/brain.lua b/lua/brain.lua
index 58f172b..7e66fc2 100644
--- a/lua/brain.lua
+++ b/lua/brain.lua
@@ -7,6 +7,8 @@ local json = require 'json'
local http = require 'http'
local resty_http = require 'resty.http'
+local inspect = require'inspect'
+
local ten_minutes, five_seconds = 60*10, 5
local Brain = {}
@@ -16,6 +18,32 @@ Brain.url = function(path)
return Brain.host .. (path or '')
end
+
+local get = function(url, params)
+ if params then
+ url = string.format("%s%s?%s", Brain.host, url, ngx.encode_args(params))
+ else
+ url = string.format("%s%s", Brain.host, url)
+ end
+ print(url)
+ return http.simple(url)
+end
+
+
+Brain.search_middleware = function(endpoint, query, per_page, page)
+ return get('/api/middleware/search', {
+ endpoint = endpoint,
+ query = query,
+ per_page = per_page,
+ page = page
+
+ })
+end
+
+Brain.show_middleware = function(id)
+ return get(string.format('/api/middleware/%s', id))
+end
+
local extract_middlewares = function(service)
local pipeline = Pipeline:get(service)
local middlewares = {}
diff --git a/lua/cachejor.lua b/lua/cachejor.lua
index 4ebbefe..f9e56a8 100644
--- a/lua/cachejor.lua
+++ b/lua/cachejor.lua
@@ -1,5 +1,5 @@
local redis = require('concurredis')
-local luajson = require('json')
+local json = require('cjson')
local inspect = require 'inspect'
local cachejor = {}
@@ -10,17 +10,20 @@ local store = redisstore
-- redis backend store
function redisstore:encode(...)
- return luajson.encode({value = ... })
+ return json.encode({value = ... })
end
function redisstore:decode(str)
- local obj = luajson.decode(str)
+ local obj = json.decode(str)
local value = obj.value
return value
end
function redisstore:save(value, ...)
+ if type(value) == "function" then
+ return
+ end
local key, field = cache:key(...)
local value = redisstore:encode(value)
@@ -77,7 +80,7 @@ function cache:key(collection, ...)
local key = ""
for i,part in ipairs({...}) do
- part = luajson.encode(part)
+ part = json.encode(part)
key = key .. "/" .. part
end
diff --git a/lua/concurredis.lua b/lua/concurredis.lua
index 82fc635..cfc2737 100644
--- a/lua/concurredis.lua
+++ b/lua/concurredis.lua
@@ -68,8 +68,32 @@ concurredis.execute = function(f)
end
end
+concurredis.disable_bgsave = function(fun)
+ return concurredis.execute(function(red)
+ local save = red:config('get', 'save')
+
+ local res, err = pcall(fun, red)
+
+ assert(red:config('set', unpack(save)))
+ assert(res, err)
+ end)
+
+end
+
concurredis.save = function()
- concurredis.execute(function(red) assert(red:save()) end)
+ concurredis.disable_bgsave(function(red)
+ local backoff = 0.01
+ local total = 0
+
+ while not red:save() do
+ total = total + backoff
+ ngx.sleep(backoff)
+
+ if total > 15 then
+ assert(red:save())
+ end
+ end
+ end)
end
concurredis.config = function(key, value)
diff --git a/lua/console.lua b/lua/console.lua
index e385c0d..947cc6f 100644
--- a/lua/console.lua
+++ b/lua/console.lua
@@ -44,16 +44,16 @@ function Console_methods:error(...)
log(self, 'error', ...)
end
-function Console_methods:get_latest_messages(how_many)
+function Console_methods:get_latest_messages(since_when)
local mt = getmetatable(self)
local events = Event:all({
channel = 'middleware',
service_id = mt.service_id,
- middleware_uuid = mt.middleware_uuid
+ middleware_uuid = mt.middleware_uuid,
+ _created_at = { ['$gt'] = since_when }
}, {
- reversed = true,
- max_documents = how_many
+ reversed = true
})
local messages = {}
diff --git a/lua/controller_helpers.lua b/lua/controller_helpers.lua
index 0ed2225..ff99b8c 100644
--- a/lua/controller_helpers.lua
+++ b/lua/controller_helpers.lua
@@ -28,7 +28,34 @@ end
helpers.send_json = function(object_to_serialize, status)
ngx.status = status or ngx.HTTP_OK
- helpers.print(helpers.json_response(object_to_serialize))
+
+ if ngx.var.arg_download then
+ ngx.header['Content-Disposition'] = 'attachment; filename=' ..
+ tostring(ngx.var.arg_filename or "")
+ end
+
+ if type(object_to_serialize) == 'function' then
+ helpers.stream(object_to_serialize)
+ else
+ helpers.print(helpers.json_response(object_to_serialize))
+ end
+end
+
+helpers.stream = function(iterator)
+ local head_request = ngx.req.get_method() == 'HEAD'
+
+ ngx.header['Transfer-Encoding'] = 'chunked'
+
+ if head_request then
+ return ngx.eof()
+ end
+
+ ngx.print('[')
+ for line in iterator do
+ ngx.print(line)
+ end
+ ngx.print(']')
+ ngx.eof()
end
helpers.json_response = function(object_to_serialize, status)
@@ -36,6 +63,8 @@ helpers.json_response = function(object_to_serialize, status)
if type(object_to_serialize) == 'table' and (next(object_to_serialize) == nil) then
return '[]'
+ elseif type(object_to_serialize) == 'string' then
+ return object_to_serialize
else
return luajson.encode(object_to_serialize, {preProcess = remove_binary_strings})
end
@@ -54,8 +83,12 @@ helpers.jor_options = function(params)
local per_page = params and tonumber(params.per_page) or 20
local max_per_page = 100
- options.reversed = not not params.reversed
- options.max_documents = per_page > max_per_page and max_per_page or per_page
+ if params.download then
+ options.iterator = true
+ else
+ options.reversed = not not params.reversed
+ options.max_documents = per_page > max_per_page and max_per_page or per_page
+ end
return options
end
@@ -64,8 +97,12 @@ helpers.jor_conditions_and_options = function(params, ordering)
local conditions = helpers.decode_json_or_error(params.query or '{}')
local options = helpers.jor_options(params)
local last_id = params.last_id
+ local first_id = params.first_id
+ local download = params.download
- if last_id then
+ if last_id and first_id then
+ conditions._id = { ['$gte'] = tonumber(first_id), ['$lte'] = tonumber(last_id) }
+ elseif last_id and not download then
ordering = ordering or options.reversed and '$lt' or '$gt'
conditions._id = { [ordering] = tonumber(last_id) }
end
diff --git a/lua/controllers/brain_middleware_specs_controller.lua b/lua/controllers/brain_middleware_specs_controller.lua
new file mode 100644
index 0000000..2a407e0
--- /dev/null
+++ b/lua/controllers/brain_middleware_specs_controller.lua
@@ -0,0 +1,19 @@
+local h = require 'controller_helpers'
+local Brain = require 'brain'
+
+return {
+ search = function(params)
+ local results = Brain.search_middleware(
+ params.endpoint,
+ params.query,
+ params.per_page,
+ params.page
+ )
+ h.send_json(results)
+ end,
+
+ show = function(params)
+ local middleware = Brain.show_middleware(params.id)
+ h.send_json(middleware)
+ end
+}
diff --git a/lua/controllers/console_controller.lua b/lua/controllers/console_controller.lua
index 7ef0ed7..2a5ed51 100644
--- a/lua/controllers/console_controller.lua
+++ b/lua/controllers/console_controller.lua
@@ -5,10 +5,10 @@ local console_controller = {
index = function(params)
local service_id = params.service_id
local uuid = params.uuid
- local how_many = params.how_many or 20
+ local since_when = params.since
local console = Console.new(service_id, uuid)
- local messages = console.get_latest_messages(tonumber(how_many))
+ local messages = console.get_latest_messages(tonumber(since_when))
h.send_json(messages)
end
diff --git a/lua/controllers/service_traces_controller.lua b/lua/controllers/service_traces_controller.lua
index f667fda..81f0c35 100644
--- a/lua/controllers/service_traces_controller.lua
+++ b/lua/controllers/service_traces_controller.lua
@@ -1,25 +1,20 @@
local h = require 'controller_helpers'
-local m = require 'model_helpers'
-local Event = require 'models.event'
local Trace = require 'models.trace'
-local fn = require 'functional'
-local map = fn.map
-
-local inspect = require 'inspect'
-
-local function search(params)
- local conditions, options = h.jor_conditions_and_options(params)
+local function search(params, order)
+ local conditions, options = h.jor_conditions_and_options(params, order)
conditions.service_id = tonumber(params.service_id)
return conditions, options
end
local service_traces = {
+ index = function(params)
+ h.send_json(Trace:all(search(params)))
+ end,
+
count = function(params)
- local conditions, options = h.jor_conditions_and_options(params, '$gt')
- conditions.service_id = tonumber(params.service_id)
- local count = Trace:count(conditions, options)
+ local count = Trace:count(search(params, '$gt'))
h.send_json({document_count = count})
end,
diff --git a/lua/controllers/services_controller.lua b/lua/controllers/services_controller.lua
index 1dc5666..f7dd6c7 100644
--- a/lua/controllers/services_controller.lua
+++ b/lua/controllers/services_controller.lua
@@ -11,7 +11,8 @@ local Pipeline = require 'models.pipeline'
local Brain = require 'brain'
local has_valid_endpoint_characters = function(endpoint)
- return not endpoint.code:find('-')
+-- allowed are lowercase letters, alphanuberic characters and a dash
+ return not endpoint.code:find('[^%l%w-]')
end
local has_service_already = function(endpoint)
diff --git a/lua/controllers/traces_controller.lua b/lua/controllers/traces_controller.lua
index ae5d477..4bf4c1b 100644
--- a/lua/controllers/traces_controller.lua
+++ b/lua/controllers/traces_controller.lua
@@ -19,6 +19,11 @@ local traces = {
h.send_json({document_count = count})
end,
+ uuid = function(params)
+ local trace = Trace:find_or_error({uuid = params.uuid}, NOT_FOUND)
+ h.send_json(trace)
+ end,
+
index = function(params)
params.last_id = params.last_id or Trace:get_last_id()
local conditions, options = h.jor_conditions_and_options(params)
diff --git a/lua/http.lua b/lua/http.lua
index 525fc6a..d163dfa 100644
--- a/lua/http.lua
+++ b/lua/http.lua
@@ -24,7 +24,7 @@ local url_escape = function(s)
return string.gsub(s, "([^A-Za-z0-9_])", char_escape)
end
-local encode_query_string = function(t, sep)
+http.encode_query_string = function(t, sep)
if sep == nil then
sep = "&"
end
@@ -66,7 +66,7 @@ local init_req = function(r)
r.body = r.body or ''
if type(r.body) == 'table' then
- r.body = encode_query_string(r.body)
+ r.body = http.encode_query_string(r.body)
r.headers["Content-type"] = "application/x-www-form-urlencoded"
r.headers["content-length"] = #r.body
end
@@ -98,7 +98,7 @@ function http.simple(req, body)
end
if type(req.body) == "table" then
- req.body = encode_query_string(req.body)
+ req.body = http.encode_query_string(req.body)
req.headers["Content-type"] = "application/x-www-form-urlencoded"
req.headers["content-length"] = #req.body
end
diff --git a/lua/jor.lua b/lua/jor.lua
index 82faddc..9a10141 100644
--- a/lua/jor.lua
+++ b/lua/jor.lua
@@ -70,6 +70,8 @@ add_jor_timed_method('find', function(self, collection, doc, options)
local results = jor.driver:find(collection, doc, options or {})
+ if type(results) == 'function' then return results end
+
if not results or #results == 1 and results[1] == 'null' or results[1] == 'none' then
return {}
end
diff --git a/lua/models/default_middleware_specs.lua b/lua/models/default_middleware_specs.lua
index a451a6a..302aa5b 100644
--- a/lua/models/default_middleware_specs.lua
+++ b/lua/models/default_middleware_specs.lua
@@ -56,15 +56,15 @@ return function (request, next_middleware)
-- initialize cache store
local threshold = 60 -- 60 seconds
- local key = 'cache=' .. request.uri_full
+ local key = "cache=" .. request.uri_full
if request.method == "GET" then
local stored = bucket.middleware.get(key)
if stored then
- local expires = stored.headers['X-Expires']
+ local expires = stored.headers["X-Expires"]
if expires and expires > time.now() then -- not expired yet
-- send.event({channel = "cache", msg = "returned cached content", level = "debug", key = key, content = stored, expires = expires, now = time.now() })
- stored.headers['Expires'] = time.http(expires)
+ stored.headers["Expires"] = time.http(expires)
end
return stored
end
@@ -74,9 +74,9 @@ return function (request, next_middleware)
-- if content is not cached, do the real request & get response
local response = next_middleware()
- if request.method == 'GET' then
+ if request.method == "GET" then
local expires = time.now() + threshold
- response.headers['X-Expires'] = expires
+ response.headers["X-Expires"] = expires
bucket.middleware.set(key, response, expires)
-- send.event({channel = "cache", msg = "stored cached content", level = "debug", content = response })
end
diff --git a/lua/models/trace.lua b/lua/models/trace.lua
index 5ede170..db423d2 100644
--- a/lua/models/trace.lua
+++ b/lua/models/trace.lua
@@ -3,7 +3,9 @@ local Service = require 'models.service'
local Event = require 'models.event'
local fn = require 'functional'
local http = require 'http'
+local uuid4 = require 'uuid'
local inspect = require 'inspect'
+local Config = require 'models.config'
local Trace = Model:new()
@@ -14,11 +16,25 @@ Trace.excluded_fields_to_index = {res = { body = true } }
Trace.keep = 1000
-Trace_mt = {__index = Trace}
+local function link(uuid)
+ local slug_name = Config.get_slug_name()
+ return "https://" .. slug_name .. ".my.apitools.com/app/traces/find/" .. uuid
+end
+
+local Trace_mt = {
+ __index = function(table, key)
+ if key == 'link' then
+ return link(table.uuid)
+ else
+ return rawget(table, key)
+ end
+ end
+}
function Trace:new(req)
Model:check_dots(self, Trace, 'new')
- return { req = req }
+ local trace = { req = req, uuid = uuid4.getUUID() }
+ return setmetatable(trace, Trace_mt)
end
function Trace:delete_expired(to_keep)
diff --git a/lua/ngxjor.lua b/lua/ngxjor.lua
index 24afb7a..92a0a2c 100644
--- a/lua/ngxjor.lua
+++ b/lua/ngxjor.lua
@@ -282,6 +282,50 @@ local function _add_index(collection, path, id)
end)
end
+local function _load_documents(collection, ids)
+ return redis.execute(function(red)
+
+ red:init_pipeline()
+ for _, id in ipairs(ids) do
+ red:get(_doc_key(collection, tonumber(id)))
+ end
+ local res, err = red:commit_pipeline()
+ if not res then return nil, err end
+
+ local docs = {}
+
+ for i=1, #res do
+ -- ngx.null is equivalent to null and can't be decoded to json
+ if ngx.null == res[i] then
+ table.insert(docs, 'null')
+ else
+ table.insert(docs, luajson.decode(res[i]))
+ end
+ end
+
+ return docs
+ end)
+end
+
+local function _documents_iterator(collection, ids)
+ local i = 0
+ local n = table.getn(ids)
+ return function ()
+ return redis.execute(function(red)
+ i = i + 1
+ if i <= n then
+ local id = ids[i]
+ local res = red:get(_doc_key(collection, tonumber(id)))
+
+ if res == ngx.null then
+ return 'null'
+ else
+ return res
+ end
+ end
+ end)
+ end
+end
function ngxjor.get_doc_paths(self, path, doc)
if type(path) == 'string' then
@@ -743,28 +787,12 @@ function ngxjor.find_with_options(self, collection, doc, options)
return ids
end
- return redis.execute(function(red)
-
- red:init_pipeline()
- for _, id in ipairs(ids) do
- red:get(_doc_key(collection, tonumber(id)))
- end
- local res, err = red:commit_pipeline()
- if not res then return nil, err end
-
- local docs = {}
- for i=1, #res do
- -- ngx.null is equivalent to null and can't be decoded to json
- if ngx.null == res[i] then
- table.insert(docs, 'null')
- else
- table.insert(docs, luajson.decode(res[i]))
- end
- end
-
- return docs
- end)
+ if options.iterator then
+ return _documents_iterator(collection, ids)
+ else
+ return _load_documents(collection, ids)
+ end
end
-- Used only in tests
diff --git a/lua/rack.lua b/lua/rack.lua
index 5c123e3..47e0b36 100644
--- a/lua/rack.lua
+++ b/lua/rack.lua
@@ -44,12 +44,6 @@ end
local function check_response(status, body)
rack_assert(status, "Rack returned with no status. Ensure that you set res.status to something in at least one of your middlewares.")
-
- -- If we have a 5xx or a 3/4xx and no body entity, exit allowing nginx config
- -- to generate a response.
- if status >= 500 or (status >= 300 and body == nil) then
- ngx.exit(status)
- end
end
local function copy(src, dest)
diff --git a/lua/statsd_wrapper.lua b/lua/statsd_wrapper.lua
index 6bb848c..3c51382 100644
--- a/lua/statsd_wrapper.lua
+++ b/lua/statsd_wrapper.lua
@@ -16,7 +16,7 @@ local function get_instance()
if statsd_server then
local statsd_port = os.getenv('SLUG_STATSD_PORT')
local Config = require 'models.config'
- local instance_id = Config.get_slug_name(true) or 'no_host'
+ local instance_id = Config and Config.get_slug_name(true) or 'no_host'
local env = os.getenv('SLUG_ENV') or 'dev'
local namespace = 'brainslug.' .. env .. '.' .. instance_id