bw - init work on metrics (wip)

src/bw/lua/bunkerweb/datastore.lua

@@ -15,8 +15,10 @@ if not lru then
 	logger:log(ERR, "failed to instantiate LRU cache : " .. err_lru)
 end
 
-function datastore:initialize()
-	if subsystem == "http" then
+function datastore:initialize(dict)
+	if dict then
+		self.dict = dict
+	elseif subsystem == "http" then
 		self.dict = shared.datastore
 	else
 		self.dict = shared.datastore_stream
@@ -112,4 +114,32 @@ function datastore:flush_lru()
 	lru:flush_all()
 end
 
+function datastore:safe_rpush(key, value)
+	local length, err = self.dict:rpush(key, value)
+	if not length and err == "no memory" then
+		local i = 0
+		while i < 5 do
+			local val
+			val, err = self.dict:lpop(key)
+			if not val then
+				return val, err
+			end
+			length, err = self.dict:rpush(key, value)
+			if not length and err ~= "no memory" then
+				return length, err
+			end
+			i = i + 1
+		end
+	end
+	return length, err
+end
+
+function datastore:lpop(key)
+	return self.dict:lpop(key)
+end
+
+function datastore:llen(key)
+	return self.dict:llen(key)
+end
+
 return datastore

src/bw/lua/bunkerweb/helpers.lua

@@ -185,6 +185,7 @@ helpers.fill_ctx = function(no_ref)
 			end
 			data.remote_addr = var.remote_addr
 			data.server_name = var.server_name
+			data.local_time = var.local_time
 			if data.kind == "http" then
 				data.uri = var.uri
 				data.request_uri = var.request_uri

src/common/core/metrics/confs/http/metrics.conf

@@ -0,0 +1 @@
+lua_shared_dict metrics_datastore {{ METRICS_MEMORY_SIZE }};

src/common/core/metrics/confs/stream/metrics.conf

@@ -0,0 +1 @@
+lua_shared_dict metrics_datastore_stream {{ METRICS_MEMORY_SIZE }};

src/common/core/metrics/metrics.lua

@@ -0,0 +1,108 @@
+local class = require "middleclass"
+local plugin = require "bunkerweb.plugin"
+local utils = require "bunkerweb.utils"
+local cjson = require "cjson"
+local datastore = require "datastore"
+
+local metrics = class("metrics", plugin)
+
+local ngx = ngx
+local shared = ngx.shared
+local subsystem = ngx.config.subsystem
+local ERR = ngx.ERR
+local HTTP_INTERNAL_SERVER_ERROR = ngx.HTTP_INTERNAL_SERVER_ERROR
+local HTTP_OK = ngx.HTTP_OK
+
+local get_reason = utils.get_reason
+local get_country = utils.get_country
+local encode = cjson.encode
+local decode = cjson.decode
+
+local match = string.match
+
+function metrics:initialize(ctx)
+	-- Call parent initialize
+	plugin.initialize(self, "metrics", ctx)
+	local dict
+	if subsystem == "http" then
+		dict = shared.metrics_datastore
+	else
+		dict = shared.metrics_datastore_stream
+	end
+	self.metrics_datastore = datastore:new(dict)
+end
+
+function metrics:log()
+	-- Don't go further if metrics is not enabled
+	if self.variables["USE_METRICS"] == "no" then
+		return self:ret(true, "metrics are disabled")
+	end
+	-- Store blocked requests
+	local reason, data = get_reason(self.ctx)
+	if reason then
+		local country = "local"
+		local err
+		if self.ctx.bw.ip_is_global then
+			country, err = get_country(self.ctx.bw.remote_addr)
+			if not country then
+				country = "unknown"
+				self.logger:log(ERR, "can't get country code " .. err)
+			end
+		end
+		local request = {
+			date = self.ctx.bw.local_time,
+			ip = self.ctx.bw.remote_addr,
+			country = country,
+			method = self.ctx.bw.request_method,
+			url = self.ctx.bw.request_uri,
+			code = ngx.status,
+			["user-agent"] = self.ctx.bw.http_user_agent or "",
+			reason = reason,
+			data = data
+		}
+		local ok
+		ok, err = self.metrics_datastore:safe_rpush("metrics_requests", encode(request))
+		if not ok then
+			self.logger:log(ERR, "can't save request to datastore : " .. err)
+		end
+	end
+	return self:ret(true, "success")
+end
+
+function metrics:log_default()
+	return self:log()
+end
+
+function metrics:api()
+
+	-- Match request
+	if not match(self.ctx.bw.uri, "^/metrics/requests$") or self.ctx.bw.request_method ~= "GET" then
+		return self:ret(false, "success")
+	end
+
+	-- Get requests metrics
+	local len, err = self.metrics_datastore:llen("metrics_requests")
+	if not len then
+		return self:ret(true, "error while getting length of metrics_requests : " .. err, HTTP_INTERNAL_SERVER_ERROR)
+	end
+	local i = 0
+	local data = {}
+	while i < len do
+		local request
+		request, err = self.metrics_datastore:lpop("metrics_requests")
+		if request then
+			table.insert(data, decode(request))
+		else
+			return self:ret(true, "error while getting metrics_requests : " .. err, HTTP_INTERNAL_SERVER_ERROR)
+		end
+		local ok
+		ok, err = self.metrics_datastore:safe_rpush("metrics_requests", request)
+		if not ok then
+			self.logger:log(ERR, "can't save request to datastore : " .. err)
+		end
+		i = i + 1 
+	end
+	return self:ret(true, data, HTTP_OK)
+end
+
+return metrics

src/common/core/metrics/plugin.json

@@ -0,0 +1,27 @@
+{
+  "id": "metrics",
+  "name": "Metrics",
+  "description": "Metrics collection and retrieve.",
+  "version": "1.0",
+  "stream": "partial",
+  "settings": {
+    "USE_METRICS": {
+      "context": "global",
+      "default": "yes",
+      "help": "Enable collection and retrieval of internal metrics.",
+      "id": "use-metrics",
+      "label": "Use metrics",
+      "regex": "^(yes|no)$",
+      "type": "check"
+    },
+    "METRICS_MEMORY_SIZE": {
+      "context": "global",
+      "default": "16m",
+      "help": "Size of the internal storage for metrics.",
+      "id": "metrics-memory-size",
+      "label": "Metrics memory size",
+      "regex": "^\\d+[kKmMgG]?$",
+      "type": "text"
+    }
+  }
+}

src/common/core/order.json

@@ -32,7 +32,7 @@
     "antibot"
   ],
   "headers": ["headers", "cors", "reverseproxy", "clientcache", "antibot"],
-  "log": ["badbehavior", "bunkernet"],
+  "log": ["badbehavior", "bunkernet", "metrics"],
   "preread": [
     "whitelist",
     "blacklist",
@@ -42,5 +42,5 @@
     "reversescan"
   ],
   "log_stream": ["badbehavior", "bunkernet"],
-  "log_default": ["badbehavior", "bunkernet"]
+  "log_default": ["badbehavior", "bunkernet", "metrics"]
 }