fix: enhance metrics logging by adding request ID and add Redis requests handling

src/bw/lua/bunkerweb/helpers.lua

@@ -188,6 +188,7 @@ helpers.fill_ctx = function(no_ref)
 			data.time_local = var.time_local
 			if data.kind == "http" then
 				data.uri = var.uri
+				data.request_id = var.request_id
 				data.request_uri = var.request_uri
 				data.request_method = var.request_method
 				data.http_user_agent = var.http_user_agent

src/common/core/metrics/metrics.lua

@@ -64,6 +64,7 @@ function metrics:log(bypass_checks)
 			end
 		end
 		local request = {
+			id = self.ctx.bw.request_id,
 			date = self.ctx.bw.start_time or time(),
 			ip = self.ctx.bw.remote_addr,
 			country = country,
@@ -75,20 +76,20 @@ function metrics:log(bypass_checks)
 			server_name = self.ctx.bw.server_name,
 			data = data,
 			security_mode = security_mode,
+			synced = not self.use_redis,
 		}
-		-- Get current requests
-		local requests = lru:get("requests")
-		if not requests then
-			requests = {}
-		end
-		-- Add current request
+		-- Get requests from LRU
+		local requests = lru:get("requests") or {}
+
+		-- Add to LRU
 		table_insert(requests, request)
-		-- Remove old requests
-		local nb_delete = #requests - tonumber(self.variables["METRICS_MAX_BLOCKED_REQUESTS"])
-		while nb_delete > 0 do
+
+		-- Remove old requests if needed
+		local max_requests = tonumber(self.variables["METRICS_MAX_BLOCKED_REQUESTS"])
+		while #requests > max_requests do
 			table_remove(requests, 1)
-			nb_delete = nb_delete - 1
 		end
+
 		-- Update worker cache
 		lru:set("requests", requests)
 	end
@@ -168,10 +169,43 @@ function metrics:timer()
 		end
 		lru:set("setup", true)
 	end
+
+	local clusterstore_ok = nil
+	if self.use_redis then
+		clusterstore_ok, err = self.clusterstore:connect()
+		if not clusterstore_ok then
+			self.logger:log(ERR, "Can't connect to Redis server: " .. err .. " - requests will be stored in datastore")
+		end
+	end
+
 	-- Loop on all keys
 	for _, key in ipairs(lru:get_keys()) do
 		-- Get LRU data
 		local value = lru:get(key)
+		if key == "requests" and clusterstore_ok then
+			for _, request in ipairs(value) do
+				if not request.synced then
+					-- Add only unsynced requests
+					local ok
+					ok, err = self.clusterstore:call("rpush", "requests", encode(request))
+					if not ok then
+						self.logger:log(ERR, "Can't sync request to Redis: " .. err)
+						break
+					end
+					request.synced = true -- Mark as synced
+				end
+			end
+
+			-- Remove old requests if needed
+			local max_requests = tonumber(self.variables["METRICS_MAX_BLOCKED_REQUESTS_REDIS"])
+			local nb_requests = self.clusterstore:call("llen", "requests")
+			if nb_requests and nb_requests > max_requests then
+				self.clusterstore:call("ltrim", "requests", -max_requests, -1)
+			end
+
+			-- Update LRU cache
+			lru:set("requests", value)
+		end
 		if type(value) == "table" then
 			value = encode(value)
 		end
@@ -184,6 +218,11 @@ function metrics:timer()
 			self.logger:log(ERR, "can't update " .. key .. "_" .. wid .. " : " .. err)
 		end
 	end
+
+	if clusterstore_ok then
+		self.clusterstore:close()
+	end
+
 	-- Done
 	return self:ret(ret, ret_err)
 end
@@ -203,7 +242,7 @@ function metrics:api()
 			-- Get the value
 			local data, err = self.metrics_datastore:get(key)
 			if not data then
-				return self:ret(true, "error while fetching requests : " .. err, HTTP_INTERNAL_SERVER_ERROR)
+				return self:ret(true, "error while fetching metric : " .. err, HTTP_INTERNAL_SERVER_ERROR)
 			end
 			local metric_key = key:gsub("_[0-9]+$", ""):gsub("^" .. filter .. "_", "")
 			if metric_key == "" then

src/common/core/metrics/plugin.json

@@ -25,12 +25,21 @@
     },
     "METRICS_MAX_BLOCKED_REQUESTS": {
       "context": "global",
-      "default": "100",
+      "default": "1000",
       "help": "Maximum number of blocked requests to store (per worker).",
       "id": "metrics-max-blocked-requests",
       "label": "Metrics max blocked requests",
       "regex": "^\\d+$",
       "type": "text"
+    },
+    "METRICS_MAX_BLOCKED_REQUESTS_REDIS": {
+      "context": "global",
+      "default": "100000",
+      "help": "Maximum number of blocked requests to store in Redis.",
+      "id": "metrics-max-blocked-requests-redis",
+      "label": "Metrics max blocked requests Redis",
+      "regex": "^\\d+$",
+      "type": "text"
     }
   }
 }

src/ui/app/routes/reports.py

@@ -1,25 +1,41 @@
-from datetime import datetime, timedelta
+from datetime import datetime
+from itertools import chain
+from json import loads
 
 from flask import Blueprint, render_template
 from flask_login import login_required
 
 from app.dependencies import BW_INSTANCES_UTILS
 
+from app.routes.utils import get_redis_client
+
 reports = Blueprint("reports", __name__)
 
 
 @reports.route("/reports", methods=["GET"])
 @login_required
 def reports_page():
-    reports = BW_INSTANCES_UTILS.get_reports()
-    current_date = datetime.now().astimezone()
-    for i in range(len(reports)):
-        date = datetime.fromtimestamp(reports[i]["date"]).astimezone()
-        if date < current_date - timedelta(days=7):
-            break
-        reports[i]["date"] = date.isoformat()
+    redis_client = get_redis_client()
+
+    # Generator for Redis reports
+    redis_reports = (loads(report) for report in redis_client.lrange("requests", 0, -1)) if redis_client else iter([])
+
+    # Combine Redis and instance reports into a single generator
+    reports = chain(redis_reports, BW_INSTANCES_UTILS.get_reports())
+
+    # Set to track seen IDs
+    seen_ids = set()
 
-    # Filter reports based on status code OR security_mode="detect"
     return render_template(
-        "reports.html", reports=[report for report in reports if (400 <= report.get("status", 0) < 500) or (report.get("security_mode") == "detect")]
+        "reports.html",
+        reports=(
+            {
+                **report,
+                "date": datetime.fromtimestamp(report["date"]).astimezone().isoformat(),
+            }
+            for report in reports
+            if report.get("id") not in seen_ids
+            and not seen_ids.add(report["id"])  # Add to seen_ids if not already present
+            and (400 <= report.get("status", 0) < 500 or report.get("security_mode") == "detect")
+        ),
     )

src/ui/app/templates/reports.html

@@ -3,7 +3,6 @@
     <!-- Content -->
     <div class="card table-responsive text-nowrap p-4 pb-8 min-vh-70">
         {% set base_flags_url = url_for('static', filename='img/flags') %}
-        <input type="hidden" id="reports_number" value="{{ reports|length }}" />
         <input type="hidden" id="base_flags_url" value="{{ base_flags_url }}" />
         <textarea type="hidden"
                   id="columns_preferences_defaults"
@@ -58,8 +57,9 @@
                 </tr>
             </thead>
             <tbody>
-                {% set ns = namespace(countries=[]) %}
+                {% set ns = namespace(countries=[], reports_number=0) %}
                 {% for report in reports %}
+                    {% set ns.reports_number = ns.reports_number + 1 %}
                     {% if report["country"] not in ns.countries %}
                         {% set ns.countries = ns.countries + [report["country"].lower()] %}
                     {% endif %}
@@ -91,6 +91,7 @@
                     </tr>
                 {% endfor %}
             </tbody>
+            <input type="hidden" id="reports_number" value="{{ ns.reports_number }}" />
             <input type="hidden" id="countries" value="{{ ns.countries|join(',') }}" />
             <span class="position-absolute bottom-0 start-50 translate-middle badge rounded-pill bg-secondary">
                 TZ: <script nonce="{{ script_nonce }}">document.write(Intl.DateTimeFormat().resolvedOptions().timeZone);</script>