fix: use serverSide processing for reports

This commit is contained in:
Théophile Diot 2024-12-02 18:21:09 +01:00
parent 818270384c
commit b7f3974fe8
No known key found for this signature in database
GPG key ID: FA995104A0BA376A
5 changed files with 497 additions and 155 deletions

View file

@ -62,7 +62,7 @@ repos:
- id: codespell
name: Codespell Spell Checker
exclude: (^src/(ui/templates|common/core/.+/files|bw/loading)/.+.html|modsecurity-rules.conf.*|src/ui/app/static/(fonts|libs)/.+)$
entry: codespell --ignore-regex="(tabEl|Widgits)" --skip CHANGELOG.md,CODE_OF_CONDUCT.md,src/ui/client/build.py,src/ui/app/static/json/countries.geojson,src/ui/app/static/js/pages/reports.js,src/ui/app/static/json/periscop.min.json,src/ui/app/static/json/blockhaus.min.json
entry: codespell --ignore-regex="(tabEl|Widgits)" --skip CHANGELOG.md,CODE_OF_CONDUCT.md,src/ui/client/build.py,src/ui/app/static/json/countries.geojson,src/ui/app/static/js/pages/reports.js,src/ui/app/static/json/periscop.min.json,src/ui/app/static/json/blockhaus.min.json,src/ui/app/routes/reports.py
language: python
types: [text]

View file

@ -1,41 +1,412 @@
from collections import defaultdict
from datetime import datetime
from itertools import chain
from json import loads
from json import dumps, loads
from flask import Blueprint, render_template
from flask import Blueprint, jsonify, render_template, request, url_for
from flask_login import login_required
from app.dependencies import BW_INSTANCES_UTILS
from app.routes.utils import get_redis_client
from app.routes.utils import cors_required, get_redis_client
reports = Blueprint("reports", __name__)
COUNTRIES_DATA_NAMES = {
"ad": "Andorra",
"ae": "United Arab Emirates",
"af": "Afghanistan",
"ag": "Antigua and Barbuda",
"ai": "Anguilla",
"al": "Albania",
"am": "Armenia",
"ao": "Angola",
"aq": "Antarctica",
"ar": "Argentina",
"as": "American Samoa",
"at": "Austria",
"au": "Australia",
"aw": "Aruba",
"ax": "Åland Islands",
"az": "Azerbaijan",
"ba": "Bosnia and Herzegovina",
"bb": "Barbados",
"bd": "Bangladesh",
"be": "Belgium",
"bf": "Burkina Faso",
"bg": "Bulgaria",
"bh": "Bahrain",
"bi": "Burundi",
"bj": "Benin",
"bl": "Saint Barthélemy",
"bm": "Bermuda",
"bn": "Brunei Darussalam",
"bo": "Bolivia, Plurinational State of",
"bq": "Caribbean Netherlands",
"br": "Brazil",
"bs": "Bahamas",
"bt": "Bhutan",
"bv": "Bouvet Island",
"bw": "Botswana",
"by": "Belarus",
"bz": "Belize",
"ca": "Canada",
"cc": "Cocos (Keeling) Islands",
"cd": "Congo, the Democratic Republic of the",
"cf": "Central African Republic",
"cg": "Republic of the Congo",
"ch": "Switzerland",
"ci": "Côte d'Ivoire",
"ck": "Cook Islands",
"cl": "Chile",
"cm": "Cameroon",
"cn": "China (People's Republic of China)",
"co": "Colombia",
"cr": "Costa Rica",
"cu": "Cuba",
"cv": "Cape Verde",
"cw": "Curaçao",
"cx": "Christmas Island",
"cy": "Cyprus",
"cz": "Czech Republic",
"de": "Germany",
"dj": "Djibouti",
"dk": "Denmark",
"dm": "Dominica",
"do": "Dominican Republic",
"dz": "Algeria",
"ec": "Ecuador",
"ee": "Estonia",
"eg": "Egypt",
"eh": "Western Sahara",
"er": "Eritrea",
"es": "Spain",
"et": "Ethiopia",
"eu": "Europe",
"fi": "Finland",
"fj": "Fiji",
"fk": "Falkland Islands (Malvinas)",
"fm": "Micronesia, Federated States of",
"fo": "Faroe Islands",
"fr": "France",
"ga": "Gabon",
"gb": "United Kingdom",
"gd": "Grenada",
"ge": "Georgia",
"gf": "French Guiana",
"gg": "Guernsey",
"gh": "Ghana",
"gi": "Gibraltar",
"gl": "Greenland",
"gm": "Gambia",
"gn": "Guinea",
"gp": "Guadeloupe",
"gq": "Equatorial Guinea",
"gr": "Greece",
"gs": "South Georgia and the South Sandwich Islands",
"gt": "Guatemala",
"gu": "Guam",
"gw": "Guinea-Bissau",
"gy": "Guyana",
"hk": "Hong Kong",
"hm": "Heard Island and McDonald Islands",
"hn": "Honduras",
"hr": "Croatia",
"ht": "Haiti",
"hu": "Hungary",
"id": "Indonesia",
"ie": "Ireland",
"il": "Israel",
"im": "Isle of Man",
"in": "India",
"io": "British Indian Ocean Territory",
"iq": "Iraq",
"ir": "Iran, Islamic Republic of",
"is": "Iceland",
"it": "Italy",
"je": "Jersey",
"jm": "Jamaica",
"jo": "Jordan",
"jp": "Japan",
"ke": "Kenya",
"kg": "Kyrgyzstan",
"kh": "Cambodia",
"ki": "Kiribati",
"km": "Comoros",
"kn": "Saint Kitts and Nevis",
"kp": "Korea, Democratic People's Republic of",
"kr": "Korea, Republic of",
"kw": "Kuwait",
"ky": "Cayman Islands",
"kz": "Kazakhstan",
"la": "Laos (Lao People's Democratic Republic)",
"lb": "Lebanon",
"lc": "Saint Lucia",
"li": "Liechtenstein",
"lk": "Sri Lanka",
"lr": "Liberia",
"ls": "Lesotho",
"lt": "Lithuania",
"lu": "Luxembourg",
"lv": "Latvia",
"ly": "Libya",
"ma": "Morocco",
"mc": "Monaco",
"md": "Moldova, Republic of",
"me": "Montenegro",
"mf": "Saint Martin",
"mg": "Madagascar",
"mh": "Marshall Islands",
"mk": "North Macedonia",
"ml": "Mali",
"mm": "Myanmar",
"mn": "Mongolia",
"mo": "Macao",
"mp": "Northern Mariana Islands",
"mq": "Martinique",
"mr": "Mauritania",
"ms": "Montserrat",
"mt": "Malta",
"mu": "Mauritius",
"mv": "Maldives",
"mw": "Malawi",
"mx": "Mexico",
"my": "Malaysia",
"mz": "Mozambique",
"na": "Namibia",
"nc": "New Caledonia",
"ne": "Niger",
"nf": "Norfolk Island",
"ng": "Nigeria",
"ni": "Nicaragua",
"nl": "Netherlands",
"no": "Norway",
"np": "Nepal",
"nr": "Nauru",
"nu": "Niue",
"nz": "New Zealand",
"om": "Oman",
"pa": "Panama",
"pe": "Peru",
"pf": "French Polynesia",
"pg": "Papua New Guinea",
"ph": "Philippines",
"pk": "Pakistan",
"pl": "Poland",
"pm": "Saint Pierre and Miquelon",
"pn": "Pitcairn",
"pr": "Puerto Rico",
"ps": "Palestine",
"pt": "Portugal",
"pw": "Palau",
"py": "Paraguay",
"qa": "Qatar",
"re": "Réunion",
"ro": "Romania",
"rs": "Serbia",
"ru": "Russian Federation",
"rw": "Rwanda",
"sa": "Saudi Arabia",
"sb": "Solomon Islands",
"sc": "Seychelles",
"sd": "Sudan",
"se": "Sweden",
"sg": "Singapore",
"sh": "Saint Helena, Ascension and Tristan da Cunha",
"si": "Slovenia",
"sj": "Svalbard and Jan Mayen Islands",
"sk": "Slovakia",
"sl": "Sierra Leone",
"sm": "San Marino",
"sn": "Senegal",
"so": "Somalia",
"sr": "Suriname",
"ss": "South Sudan",
"st": "Sao Tome and Principe",
"sv": "El Salvador",
"sx": "Sint Maarten (Dutch part)",
"sy": "Syrian Arab Republic",
"sz": "Swaziland",
"tc": "Turks and Caicos Islands",
"td": "Chad",
"tf": "French Southern Territories",
"tg": "Togo",
"th": "Thailand",
"tj": "Tajikistan",
"tk": "Tokelau",
"tl": "Timor-Leste",
"tm": "Turkmenistan",
"tn": "Tunisia",
"to": "Tonga",
"tr": "Turkey",
"tt": "Trinidad and Tobago",
"tv": "Tuvalu",
"tw": "Taiwan (Republic of China)",
"tz": "Tanzania, United Republic of",
"ua": "Ukraine",
"ug": "Uganda",
"um": "US Minor Outlying Islands",
"us": "United States",
"uy": "Uruguay",
"uz": "Uzbekistan",
"va": "Holy See (Vatican City State)",
"vc": "Saint Vincent and the Grenadines",
"ve": "Venezuela, Bolivarian Republic of",
"vg": "Virgin Islands, British",
"vi": "Virgin Islands, U.S.",
"vn": "Vietnam",
"vu": "Vanuatu",
"wf": "Wallis and Futuna Islands",
"ws": "Samoa",
"xk": "Kosovo",
"ye": "Yemen",
"yt": "Mayotte",
"za": "South Africa",
"zm": "Zambia",
"zw": "Zimbabwe",
}
@reports.route("/reports", methods=["GET"])
@login_required
def reports_page():
return render_template("reports.html")
@reports.route("/reports/fetch", methods=["POST"])
@login_required
@cors_required
def reports_fetch():
redis_client = get_redis_client()
# Generator for Redis reports
redis_reports = (loads(report) for report in redis_client.lrange("requests", 0, -1)) if redis_client else iter([])
# Fetch reports
def fetch_reports():
if redis_client:
redis_reports = redis_client.lrange("requests", 0, -1)
redis_reports = (loads(report_raw) for report_raw in redis_reports)
else:
redis_reports = []
instance_reports = BW_INSTANCES_UTILS.get_reports() if BW_INSTANCES_UTILS else []
return chain(redis_reports, instance_reports)
# Combine Redis and instance reports into a single generator
reports = chain(redis_reports, BW_INSTANCES_UTILS.get_reports())
# Set to track seen IDs
# Filter valid and unique reports
seen_ids = set()
return render_template(
"reports.html",
reports=(
{
**report,
"date": datetime.fromtimestamp(report["date"]).astimezone().isoformat(),
}
for report in reports
if report.get("id") not in seen_ids
and not seen_ids.add(report["id"]) # Add to seen_ids if not already present
and (400 <= report.get("status", 0) < 500 or report.get("security_mode") == "detect")
),
all_reports = list(
report
for report in fetch_reports()
if report.get("id") not in seen_ids
and (400 <= report.get("status", 0) < 500 or report.get("security_mode") == "detect")
and not seen_ids.add(report.get("id"))
)
# Extract DataTables parameters
draw = int(request.form.get("draw", 1))
start = int(request.form.get("start", 0))
length = int(request.form.get("length", 10))
search_value = request.form.get("search[value]", "").lower()
order_column_index = int(request.form.get("order[0][column]", 0)) - 1
order_direction = request.form.get("order[0][dir]", "desc")
search_panes = defaultdict(list)
for key, value in request.form.items():
if key.startswith("searchPanes["):
field = key.split("[")[1].split("]")[0]
search_panes[field].append(value)
columns = ["date", "ip", "country", "method", "url", "status", "user_agent", "reason", "server_name", "data", "security_mode"]
# Apply searchPanes filters
def filter_by_search_panes(reports):
for field, selected_values in search_panes.items():
reports = [report for report in reports if report.get(field, "N/A") in selected_values]
return reports
# Global search filtering
def global_search_filter(report):
return any(search_value in str(report.get(col, "")).lower() for col in columns)
# Sort reports
def sort_reports(reports):
if 0 <= order_column_index < len(columns):
sort_key = columns[order_column_index]
reports.sort(key=lambda x: x.get(sort_key, ""), reverse=(order_direction == "desc"))
# Apply filters and sort
filtered_reports = filter(global_search_filter, all_reports) if search_value else all_reports
filtered_reports = list(filter_by_search_panes(filtered_reports))
sort_reports(filtered_reports)
# Pagination
paginated_reports = filtered_reports[start : start + length] # noqa: E203
# Format reports for the response
def format_report(report):
return {
"date": datetime.fromtimestamp(report.get("date", 0)).isoformat() if report.get("date") else "N/A",
"ip": report.get("ip", "N/A"),
"country": report.get("country", "N/A"),
"method": report.get("method", "N/A"),
"url": report.get("url", "N/A"),
"status": report.get("status", "N/A"),
"user_agent": report.get("user_agent", "N/A"),
"reason": report.get("reason", "N/A"),
"server_name": report.get("server_name", "N/A"),
"data": dumps(report.get("data", {})),
"security_mode": report.get("security_mode", "N/A"),
}
formatted_reports = [format_report(report) for report in paginated_reports]
# Calculate pane counts
pane_counts = defaultdict(lambda: defaultdict(lambda: {"total": 0, "count": 0}))
filtered_ids = {report["id"] for report in filtered_reports}
for report in all_reports:
for field in columns[1:]: # Skip date field
value = report.get(field, "N/A")
# Ensure value is hashable (convert dicts or lists to strings if necessary)
if isinstance(value, (dict, list)):
value = str(value)
pane_counts[field][value]["total"] += 1
if report["id"] in filtered_ids:
pane_counts[field][value]["count"] += 1
# Prepare SearchPanes options
base_flags_url = url_for("static", filename="img/flags")
search_panes_options = {}
for field, values in pane_counts.items():
if field == "country":
search_panes_options["country"] = []
for code, counts in values.items():
country_code = code.lower()
country_name = COUNTRIES_DATA_NAMES.get(country_code, "N/A")
search_panes_options["country"].append(
{
"label": f"""<img src="{base_flags_url}/{'zz' if code == 'local' else country_code}.svg" class="border border-1 p-0 me-1" height="17" />&nbsp;&nbsp;{'N/A' if code == 'local' else country_name}""",
"value": code,
"total": counts["total"],
"count": counts["count"],
}
)
else:
search_panes_options[field] = [
{
"label": value,
"value": value,
"total": counts["total"],
"count": counts["count"],
}
for value, counts in values.items()
]
# Response
return jsonify(
{
"draw": draw,
"recordsTotal": len(all_reports),
"recordsFiltered": len(filtered_reports),
"data": formatted_reports,
"searchPanes": {"options": search_panes_options},
}
)

View file

@ -1347,3 +1347,7 @@ html.dark-style div.dts div.dataTables_scrollBody table {
border-color: #fff;
color: #000;
}
.dark-style div.dt-processing > div:last-child > div {
background-color: #fff;
}

View file

@ -1,8 +1,4 @@
$(function () {
const reportsNumber = parseInt($("#reports_number").val(), 10) || 0;
const dataCountries = ($("#countries").val() || "")
.split(",")
.filter((code) => code && code !== "local");
$(document).ready(function () {
const baseFlagsUrl = $("#base_flags_url").val().trim();
const countriesDataNames = {
@ -259,26 +255,6 @@ $(function () {
zw: "Zimbabwe",
};
// Precompute filtered options using jQuery's map for efficiency
const countriesSearchPanesOptions = $.map(dataCountries, function (code) {
if (countriesDataNames[code]) {
return {
label: `<img src="${baseFlagsUrl}/${code}.svg" class="border border-1 p-0 me-1" height="17" />&nbsp;&nbsp;${countriesDataNames[code]}`,
value: function (rowData) {
return rowData[3].indexOf(`${code}.svg`) !== -1;
},
};
}
});
// Append the "N/A" option first
countriesSearchPanesOptions.unshift({
label: `<img src="${baseFlagsUrl}/zz.svg" class="border border-1 p-0 me-1" height="17" />&nbsp;&nbsp;N/A`,
value: function (rowData) {
return rowData[3].indexOf("N/A") !== -1;
},
});
// Batch update tooltips
const updateCountryTooltips = () => {
$("[data-bs-original-title]").each(function () {
@ -318,26 +294,12 @@ $(function () {
search: true,
},
bottomStart: {
info: true,
},
bottomEnd: {},
};
// Adjust page length options based on reports number
if (reportsNumber > 10) {
const menu = [10];
if (reportsNumber > 25) menu.push(25);
if (reportsNumber > 50) menu.push(50);
if (reportsNumber > 100) menu.push(100);
menu.push({ label: "All", value: -1 });
layout.bottomStart = {
pageLength: {
menu: menu,
menu: [10, 25, 50, 100, { label: "All", value: -1 }],
},
info: true,
};
layout.bottomEnd.paging = true;
}
},
};
// Define DataTable buttons
layout.topStart.buttons = [
@ -407,7 +369,7 @@ $(function () {
if (!autoRefresh) {
clearInterval(interval);
} else {
window.location.reload();
reports_table.ajax.reload(null, false);
}
}, 10000); // 10 seconds
} else {
@ -433,11 +395,6 @@ $(function () {
columnVisibilityCondition: (column) => column > 2 && column < 12,
dataTableOptions: {
columnDefs: [
{
orderable: false,
className: "dtr-control",
targets: 0,
},
{ orderable: false, targets: -1 },
{ visible: false, targets: [4, 5, 6, 7, 10] },
{ type: "ip-address", targets: 2 },
@ -457,9 +414,22 @@ $(function () {
searchPanes: {
show: true,
combiner: "or",
options: countriesSearchPanesOptions,
// options: countriesSearchPanesOptions,
},
targets: 3,
render: function (data) {
const countryCode = data.toLowerCase();
const tooltipContent = countriesDataNames[countryCode] || "N/A";
return `
<span data-bs-toggle="tooltip" data-bs-original-title="${tooltipContent}">
<img src="${baseFlagsUrl}/${
countryCode === "local" ? "zz" : countryCode
}.svg"
class="border border-1 p-0 me-1"
height="17" />
&nbsp;&nbsp;${countryCode === "local" ? "N/A" : data}
</span>`;
},
},
{
searchPanes: { show: true },
@ -477,12 +447,84 @@ $(function () {
lengthMenu: "Display _MENU_ reports",
zeroRecords: "No matching reports found",
},
processing: true,
serverSide: true,
ajax: {
url: `${window.location.pathname}/fetch`,
type: "POST",
data: function (d) {
d.csrf_token = $("#csrf_token").val(); // Add CSRF token if needed
return d;
},
},
initComplete: () => {
$("#reports_wrapper")
.find(".btn-secondary")
.removeClass("btn-secondary");
updateCountryTooltips();
},
columns: [
{
data: null,
defaultContent: "",
orderable: false,
className: "dtr-control",
},
{ data: "date", title: "Date" },
{ data: "ip", title: "IP Address" },
{ data: "country", title: "Country" },
{ data: "method", title: "Method" },
{ data: "url", title: "URL" },
{ data: "status", title: "Status Code" },
{ data: "user_agent", title: "User-Agent" },
{ data: "reason", title: "Reason" },
{ data: "server_name", title: "Server name" },
{ data: "data", title: "Data" },
{ data: "security_mode", title: "Security mode" },
],
headerCallback: function (thead) {
const headers = [
{
title: "Date",
tooltip: "The date and time when the Report was created",
},
{ title: "IP Address", tooltip: "The reported IP address" },
{
title: "Country",
tooltip: "The country of the reported IP address",
},
{ title: "Method", tooltip: "The method used by the attacker" },
{
title: "URL",
tooltip: "The URL that was targeted by the attacker",
},
{
title: "Status Code",
tooltip: "The HTTP status code returned by BunkerWeb",
},
{ title: "User-Agent", tooltip: "The User-Agent of the attacker" },
{ title: "Reason", tooltip: "The reason why the Report was created" },
{
title: "Server name",
tooltip: "The Server name that created the report",
},
{ title: "Data", tooltip: "Additional data about the Report" },
{ title: "Security mode", tooltip: "Security mode" },
];
// Apply tooltips to column headers
$(thead)
.find("th")
.each(function (index) {
const header = headers[index - 1]; // Adjust index to skip expandable column
if (header) {
$(this)
.attr("data-bs-toggle", "tooltip")
.attr("data-bs-placement", "bottom")
.attr("title", header.tooltip);
}
});
// Initialize Bootstrap tooltips
$('[data-bs-toggle="tooltip"]').tooltip();
},
},
});
@ -490,6 +532,8 @@ $(function () {
toggleAutoRefresh();
}
$("#reports_wrapper").find(".btn-secondary").removeClass("btn-secondary");
// Update tooltips after table draw
reports_table.on("draw.dt", updateCountryTooltips);

View file

@ -16,87 +16,10 @@
class="text-center relative w-full p-2 text-primary rounded-lg fw-bold">Loading reports...</p>
<table id="reports"
class="table responsive nowrap position-relative w-100 d-none">
<thead>
<tr>
<th data-bs-toggle="tooltip"
data-bs-placement="bottom"
data-bs-original-title="Show the Reports' details"></th>
<th data-bs-toggle="tooltip"
data-bs-placement="bottom"
data-bs-original-title="The date and time when the Report was created">Date</th>
<th data-bs-toggle="tooltip"
data-bs-placement="bottom"
data-bs-original-title="The reported IP address">IP Address</th>
<th data-bs-toggle="tooltip"
data-bs-placement="bottom"
data-bs-original-title="The country of the reported IP address">Country</th>
<th data-bs-toggle="tooltip"
data-bs-placement="bottom"
data-bs-original-title="The method used by the attacker">Method</th>
<th data-bs-toggle="tooltip"
data-bs-placement="bottom"
data-bs-original-title="The URL that was targeted by the attacker">URL</th>
<th data-bs-toggle="tooltip"
data-bs-placement="bottom"
data-bs-original-title="The HTTP status code returned by BunkerWeb">Status Code</th>
<th data-bs-toggle="tooltip"
data-bs-placement="bottom"
data-bs-original-title="The User-Agent of the attacker">User-Agent</th>
<th data-bs-toggle="tooltip"
data-bs-placement="bottom"
data-bs-original-title="The reason why the Report was created">Reason</th>
<th data-bs-toggle="tooltip"
data-bs-placement="bottom"
data-bs-original-title="The Server name that created the report">Server name</th>
<th data-bs-toggle="tooltip"
data-bs-placement="bottom"
data-bs-original-title="Additional data about the Report">Data</th>
<th data-bs-toggle="tooltip"
data-bs-placement="bottom"
data-bs-original-title="Security mode">Security mode</th>
</tr>
</thead>
<tbody>
{% set ns = namespace(countries=[], reports_number=0) %}
{% for report in reports %}
{% set ns.reports_number = ns.reports_number + 1 %}
{% if report["country"] not in ns.countries %}
{% set ns.countries = ns.countries + [report["country"].lower()] %}
{% endif %}
<tr>
<td></td>
<td class="report-date">{{ report["date"] }}</td>
<td>{{ report["ip"] }}</td>
<td data-bs-toggle="tooltip"
data-bs-placement="top"
data-bs-original-title="{% if report['country'] == "local" %}N/A{% else %}{{ report["country"]|lower }}{% endif %}">
<img src="{{ base_flags_url }}/{% if report['country'] == "local" %}zz{% else %}{{ report['country']|lower }}{% endif %}.svg"
class="border border-1 p-0 me-1"
height="17" />
&nbsp;&nbsp;
{% if report['country'] == "local" %}
N/A
{% else %}
{{ report["country"] }}
{% endif %}
</td>
<td>{{ report["method"] }}</td>
<td>{{ report["url"] }}</td>
<td>{{ report["status"] }}</td>
<td>{{ report["user_agent"] }}</td>
<td>{{ report["reason"] }}</td>
<td>{{ report["server_name"] }}</td>
<td>{{ report["data"] }}</td>
<td>{{ report["security_mode"] }}</td>
</tr>
{% endfor %}
</tbody>
<input type="hidden" id="reports_number" value="{{ ns.reports_number }}" />
<input type="hidden" id="countries" value="{{ ns.countries|join(',') }}" />
<span class="position-absolute bottom-0 start-50 translate-middle badge rounded-pill bg-secondary">
TZ: <script nonce="{{ script_nonce }}">document.write(Intl.DateTimeFormat().resolvedOptions().timeZone);</script>
</span>
</table>
<span class="position-absolute bottom-0 start-50 translate-middle badge rounded-pill bg-secondary">
TZ: <script nonce="{{ script_nonce }}">document.write(Intl.DateTimeFormat().resolvedOptions().timeZone);</script>
</span>
</div>
<!-- / Content -->
{% endblock %}