bw - add basic counter mechanism for metrics

src/bw/lua/bunkerweb/plugin.lua

@@ -93,4 +93,19 @@ function plugin:ret(ret, msg, status, redirect, data)
 	return { ret = ret, msg = msg, status = status, redirect = redirect, data = data }
 end
 
+function plugin:set_metric(kind, key, value)
+	if self.ctx and self.ctx.bw then
+		if not self.ctx.bw.metrics then
+			self.ctx.bw.metrics = {}
+		end
+		if not self.ctx.bw.metrics[self.id] then
+			self.ctx.bw.metrics[self.id] = {}
+		end
+		if not self.ctx.bw.metrics[self.id][kind] then
+			self.ctx.bw.metrics[self.id][kind] = {}
+		end
+		self.ctx.bw.metrics[self.id][kind][key] = value
+	end
+end
+
 return plugin

src/common/core/antibot/antibot.lua

@@ -152,6 +152,7 @@ function antibot:access()
 		if ok == nil then
 			return self:ret(false, "check challenge error : " .. err, HTTP_INTERNAL_SERVER_ERROR)
 		elseif not ok then
+			self:set_metric("counters", "failed_challenges", 1)
 			self.logger:log(ngx.WARN, "client failed challenge : " .. err)
 		end
 		if redirect then

src/common/core/badbehavior/badbehavior.lua

@@ -49,6 +49,7 @@ function badbehavior:log()
 	if not ok then
 		return self:ret(false, "can't create increase timer : " .. err)
 	end
+	self:set_metric("counters", tostring(ngx.status), 1)
 	return self:ret(true, "success")
 end
 

src/common/core/blacklist/blacklist.lua

@@ -132,12 +132,14 @@ function blacklist:access()
 		if not ok then
 			self.logger:log(ERR, "error while checking cache : " .. cached)
 		elseif cached and cached ~= "ok" then
+			local data = self:get_data(cached)
+			self:set_metric("counters", "failed_" .. data.id, 1)
 			return self:ret(
 				true,
 				k .. " is in cached blacklist (info : " .. cached .. ")",
 				get_deny_status(),
 				nil,
-				self:get_data(cached)
+				data
 			)
 		end
 		if ok and cached then
@@ -161,12 +163,14 @@ function blacklist:access()
 					self.logger:log(ERR, "error while adding element to cache : " .. err)
 				end
 				if blacklisted ~= "ok" then
+					local data = self:get_data(blacklisted)
+					self:set_metric("counters", "failed_" .. data.id, 1)
 					return self:ret(
 						true,
 						k .. " is blacklisted (info : " .. blacklisted .. ")",
 						get_deny_status(),
 						nil,
-						self:get_data(blacklisted)
+						data
 					)
 				end
 			end

src/common/core/bunkernet/bunkernet.lua

@@ -10,6 +10,8 @@ local ngx = ngx
 local ERR = ngx.ERR
 local NOTICE = ngx.NOTICE
 local WARN = ngx.WARN
+local HTTP_INTERNAL_SERVER_ERROR = ngx.HTTP_INTERNAL_SERVER_ERROR
+local HTTP_OK = ngx.HTTP_OK
 local timer_at = ngx.timer.at
 local get_phase = ngx.get_phase
 local get_version = utils.get_version
@@ -28,6 +30,7 @@ local open = io.open
 local encode = cjson.encode
 local decode = cjson.decode
 local http_new = http.new
+local match = string.match
 
 function bunkernet:initialize(ctx)
 	-- Call parent initialize
@@ -308,4 +311,28 @@ function bunkernet:report(ip, reason, reason_data, method, url, headers)
 	return self:request("POST", "/report", data)
 end
 
+function bunkernet:api()
+	-- Match request
+	if not match(self.ctx.bw.uri, "^/bunkernet/ping$") or self.ctx.bw.request_method ~= "POST" then
+		return self:ret(false, "success")
+	end
+	-- Check id
+	if not self.bunkernet_id then
+		return self:ret(true, "missing instance ID", HTTP_INTERNAL_SERVER_ERROR)
+	end
+	-- Send ping request
+	local ok, err, status, _ = self:ping()
+	if not ok then
+		return self:ret(true, "error while sending request to API : " .. err, HTTP_INTERNAL_SERVER_ERROR)
+	end
+	if status ~= 200 then
+		return self:ret(
+			true,
+			"received status " .. tostring(status) .. " from API using instance ID " .. self.bunkernet_id,
+			HTTP_INTERNAL_SERVER_ERROR
+		)
+	end
+	return self:ret(true, "connectivity with API using instance ID " .. self.bunkernet_id .. " is successful", HTTP_OK)
+end
+
 return bunkernet

src/common/core/cors/cors.lua

@@ -56,6 +56,7 @@ function cors:header()
 		and self.variables["CORS_ALLOW_ORIGIN"] ~= "*"
 		and not regex_match(self.ctx.bw.http_origin, self.variables["CORS_ALLOW_ORIGIN"])
 	then
+		self:set_metric("counters", "failed_cors", 1)
 		self.logger:log(WARN, "origin " .. self.ctx.bw.http_origin .. " is not allowed")
 		return self:ret(true, "origin " .. self.ctx.bw.http_origin .. " is not allowed")
 	end

src/common/core/country/country.lua

@@ -34,6 +34,7 @@ function country:access()
 					.. ")"
 			)
 		end
+		self:set_metric("counters", "failed_country", 1)
 		return self:ret(
 			true,
 			"client IP "
@@ -85,6 +86,7 @@ function country:access()
 		if not ok then
 			return self:ret(false, "error while adding item to cache : " .. err)
 		end
+		self:set_metric("counters", "failed_country", 1)
 		return self:ret(
 			true,
 			"client IP " .. self.ctx.bw.remote_addr .. " is not whitelisted (country = " .. country_data .. ")",
@@ -105,6 +107,7 @@ function country:access()
 				if not ok then
 					return self:ret(false, "error while adding item to cache : " .. err)
 				end
+				self:set_metric("counters", "failed_country", 1)
 				return self:ret(
 					true,
 					"client IP " .. self.ctx.bw.remote_addr .. " is blacklisted (country = " .. country_data .. ")",

src/common/core/dnsbl/dnsbl.lua

@@ -91,6 +91,7 @@ function dnsbl:access()
 		if cached == "ok" then
 			return self:ret(true, "client IP " .. self.ctx.bw.remote_addr .. " is in DNSBL cache (not blacklisted)")
 		end
+		self:set_metric("counters", "failed_dnsbl", 1)
 		return self:ret(
 			true,
 			"client IP " .. self.ctx.bw.remote_addr .. " is in DNSBL cache (server = " .. cached .. ")",

src/common/core/errors/errors.lua

@@ -3,6 +3,7 @@ local plugin = require "bunkerweb.plugin"
 
 local ngx = ngx
 local subsystem = ngx.config.subsystem
+local tostring = tostring
 
 local template
 local render = nil
@@ -69,6 +70,11 @@ function errors:initialize(ctx)
 	}
 end
 
+function errors:log()
+	self:set_metric("counters", tostring(ngx.status), 1)
+	return self:ret(true, "success")
+end
+
 function errors:render_template(code)
 	-- Render template
 	render("error.html", {

src/common/core/greylist/greylist.lua

@@ -151,6 +151,7 @@ function greylist:access()
 	end
 
 	-- Return
+	self:set_metric("counters", "failed_greylist" .. data.id, 1)
 	return self:ret(true, "not in greylist", get_deny_status())
 end
 

src/common/core/limit/limit.lua

@@ -158,6 +158,7 @@ function limit:access()
 	end
 	-- Limit reached
 	if limited then
+		self:set_metric("counters", "limited_uri_" .. self.ctx.bw.uri, 1)
 		return self:ret(
 			true,
 			"client IP "

src/common/core/metrics/metrics.lua

@@ -90,6 +90,29 @@ function metrics:log(bypass_checks)
 		-- Update worker cache
 		lru:set("requests", requests)
 	end
+	-- Get metrics from plugins
+	local all_metrics = self.ctx.bw.metrics
+	if all_metrics then
+		-- Loop on plugins
+		for plugin, plugin_metrics in pairs(all_metrics) do
+			-- Loop on kinds
+			for kind, kind_metrics in pairs(plugin_metrics) do
+				-- Increment counters
+				if kind == "counters" then
+					for metric_key, metric_value in pairs(kind_metrics) do
+						local lru_key = plugin .. "_counter_" .. metric_key
+						local metric_counter = lru:get(lru_key)
+						if not metric_counter then
+							metric_counter = metric_value
+						else
+							metric_counter = metric_counter + metric_value
+						end
+						lru:set(lru_key, metric_counter)
+					end
+				end
+			end
+		end
+	end
 	return self:ret(true, "success")
 end
 
@@ -113,53 +136,97 @@ function metrics:timer()
 	if not is_needed then
 		return self:ret(true, "metrics not used")
 	end
+
+	local ret = true
+	local ret_err = "metrics updated"
+	local wid = tostring(worker_id())
+
 	-- Purpose of following code is to populate the LRU cache.
 	-- In case of a reload, everything in LRU cache is removed
 	-- so we need to copy it from SHM cache if it exists.
 	local setup = lru:get("setup")
 	if not setup then
-		local requests, err = self.metrics_datastore:get("requests_" .. tostring(worker_id()))
-		if not requests and err ~= "not found" then
-			self.logger:log(ERR, "error while checking datastore : " .. err)
-		end
-		if requests then
-			lru:set("requests", decode(requests))
+		for _, key in ipairs(self.metrics_datastore:keys()) do
+			if key:match("_" .. wid .. "$") then
+				local value, err = self.metrics_datastore:get(key)
+				if not value and err ~= "not found" then
+					ret = false
+					ret_err = err
+					self.logger:log(ERR, "error while checking " .. key .. " : " .. err)
+				end
+				if value then
+					local ok, decoded = pcall(decode, value)
+					if ok then
+						value = decoded
+					end
+					lru:set(key:gsub("_" .. wid .. "$", ""), value)
+				end
+			end
 		end
 		lru:set("setup", true)
 	end
-	-- Get worker requests
-	local requests = lru:get("requests")
-	if not requests then
-		requests = {}
+	-- Loop on all keys
+	for _, key in ipairs(lru:get_keys()) do
+		-- Get LRU data
+		local value = lru:get(key)
+		if type(value) == "table" then
+			value = encode(value)
+		end
+		-- Push to dict
+		local ok, err = self.metrics_datastore:set(key .. "_" .. wid, value)
+		if not ok then
+			ret = false
+			ret_err = err
+			self.logger:log(ERR, "can't update " .. key .. "_" .. wid .. " : " .. err)
+		end
 	end
-	-- Push to dict
-	local ok, err = self.metrics_datastore:set("requests_" .. tostring(worker_id()), encode(requests))
-	if not ok then
-		return self:ret(false, "can't update requests : " .. err)
-	end
-	return self:ret(true, "metrics updated")
+	-- Done
+	return self:ret(ret, ret_err)
 end
 
 function metrics:api()
 	-- Match request
-	if not match(self.ctx.bw.uri, "^/metrics/requests$") or self.ctx.bw.request_method ~= "GET" then
+	if not match(self.ctx.bw.uri, "^/metrics/.+$") or self.ctx.bw.request_method ~= "GET" then
 		return self:ret(false, "success")
 	end
-	-- Get requests metrics
-	local keys = self.metrics_datastore:keys()
-	local requests = {}
-	for _, key in ipairs(keys) do
-		if key:match("^requests_") then
+	-- Extract filter parameter
+	local filter = self.ctx.bw.uri:gsub("^/metrics/", "")
+	-- Loop on keys
+	local metrics_data = {}
+	for _, key in ipairs(self.metrics_datastore:keys()) do
+		-- Check if key starts with our filter
+		if key:match("^" .. filter .. "_") then
+			-- Get the value
 			local data, err = self.metrics_datastore:get(key)
 			if not data then
 				return self:ret(true, "error while fetching requests : " .. err, HTTP_INTERNAL_SERVER_ERROR)
 			end
-			for _, request in ipairs(decode(data)) do
-				table_insert(requests, request)
+			local metric_key = key:gsub("_[0-9]+$", ""):gsub("^" .. filter .. "_", "")
+			if metric_key == "" then
+				metric_key = filter
+			end
+			-- Table case
+			local ok, decoded = pcall(decode, data)
+			if ok then
+				data = decoded
+			end
+			if type(data) == "table" then
+				if not metrics_data[metric_key] then
+					metrics_data[metric_key] = {}
+				end
+				for _, metric_value in ipairs(data) do
+					table_insert(metrics_data[metric_key], metric_value)
+				end
+			-- Counter case
+			else
+				if not metrics_data[metric_key] then
+					metrics_data[metric_key] = 0
+				end
+				metrics_data[metric_key] = metrics_data[metric_key] + data
 			end
 		end
 	end
-	return self:ret(true, requests, HTTP_OK)
+	return self:ret(true, metrics_data, HTTP_OK)
 end
 
 return metrics

src/common/core/misc/misc.lua

@@ -23,10 +23,18 @@ function misc:access()
 	-- Check if method is allowed
 	for allowed_method in self.variables["ALLOWED_METHODS"]:gmatch("[^|]+") do
 		if method == allowed_method then
+			self:set_metric("counters", "failed_method", 1)
 			return self:ret(true, "method " .. method .. " is allowed")
 		end
 	end
 	return self:ret(true, "method " .. method .. " is not allowed", HTTP_NOT_ALLOWED)
 end
 
+function misc:log_default()
+	if self.variables["DISABLE_DEFAULT_SERVER"] == "yes" then
+		self:set_metric("counters", "failed_default", 1)
+	end
+	return self:ret(true, "success")
+end
+
 return misc

src/common/core/order.json

@@ -32,7 +32,7 @@
     "antibot"
   ],
   "headers": ["headers", "cors", "reverseproxy", "clientcache", "antibot"],
-  "log": ["badbehavior", "bunkernet", "metrics"],
+  "log": ["badbehavior", "bunkernet", "errors", "metrics"],
   "preread": [
     "whitelist",
     "blacklist",
@@ -42,6 +42,6 @@
     "reversescan"
   ],
   "log_stream": ["badbehavior", "bunkernet"],
-  "log_default": ["badbehavior", "bunkernet", "metrics"],
+  "log_default": ["badbehavior", "bunkernet", "errors", "misc", "metrics"],
   "timer": ["metrics"]
 }

src/common/core/redis/redis.lua

@@ -5,6 +5,9 @@ local redis = class("redis", plugin)
 
 local ngx = ngx
 local NOTICE = ngx.NOTICE
+local HTTP_INTERNAL_SERVER_ERROR = ngx.HTTP_INTERNAL_SERVER_ERROR
+local HTTP_OK = ngx.HTTP_OK
+local match = string.match
 
 function redis:initialize(ctx)
 	-- Call parent initialize
@@ -34,4 +37,26 @@ function redis:init_worker()
 	return self:ret(true, "success")
 end
 
+function redis:api()
+	-- Match request
+	if not match(self.ctx.bw.uri, "^/redis/ping$") or self.ctx.bw.request_method ~= "POST" then
+		return self:ret(false, "success")
+	end
+	-- Check redis connection
+	local ok, err = self.clusterstore:connect(true)
+	if not ok then
+		return self:ret(true, "redis connect error : " .. err, HTTP_INTERNAL_SERVER_ERROR)
+	end
+	-- Send ping
+	local ok, err = self.clusterstore:call("ping")
+	self.clusterstore:close()
+	if err then
+		return self:ret(true, "error while sending ping command to redis server : " .. err, HTTP_INTERNAL_SERVER_ERROR)
+	end
+	if not ok then
+		return self:ret(true, "redis ping command failed", HTTP_INTERNAL_SERVER_ERROR)
+	end
+	return self:ret(true, "success", HTTP_OK)
+end
+
 return redis

src/common/core/reversescan/reversescan.lua

@@ -37,6 +37,7 @@ function reversescan:access()
 		elseif cached == "open" then
 			ret_threads = true
 			ret_err = "port " .. port .. " is opened for IP " .. self.ctx.bw.remote_addr
+			self:set_metric("counters", "failed_" .. port, 1)
 			break
 			-- Perform scan in a thread
 		elseif not cached then
@@ -99,6 +100,7 @@ function reversescan:access()
 		if open then
 			ret_threads = true
 			ret_err = "port " .. port .. " is opened for IP " .. self.ctx.bw.remote_addr
+			self:set_metric("counters", "failed_" .. port, 1)
 			break
 		end
 	end

src/common/core/whitelist/whitelist.lua

@@ -137,6 +137,7 @@ function whitelist:access()
 		ngx_var.is_whitelisted = "yes"
 		self.ctx.bw.is_whitelisted = "yes"
 		env_set("is_whitelisted", "yes")
+		self:set_metric("counters", "passed_whitelist", 1)
 		return self:ret(true, err, OK)
 	end
 	-- Perform checks
@@ -155,7 +156,8 @@ function whitelist:access()
 					ngx_var.is_whitelisted = "yes"
 					self.ctx.bw.is_whitelisted = "yes"
 					env_set("is_whitelisted", "yes")
-					return self:ret(true, k .. " is whitelisted (info : " .. whitelisted .. ")", ngx.OK)
+					self:set_metric("counters", "passed_whitelist", 1)
+					return self:ret(true, k .. " is whitelisted (info : " .. whitelisted .. ")", OK)
 				end
 			end
 		end

src/ui/src/Instances.py

@@ -349,7 +349,7 @@ class Instances:
     def get_reports(self, _id: Optional[int] = None) -> List[dict[str, Any]]:
         if _id:
             instance = self.__instance_from_id(_id)
-            resp, instance_reports = instance.reports()
+            resp, instance_reports = instance.reports()["requests"]
             if not resp:
                 return []
             return instance_reports[instance.name if instance.name != "local" else "127.0.0.1"].get("msg", [])
@@ -357,7 +357,7 @@ class Instances:
         reports: List[dict[str, Any]] = []
         for instance in self.get_instances():
             try:
-                resp, instance_reports = instance.reports()
+                resp, instance_reports = instance.reports()["requests"]
             except :
                 continue