refactor - start to use ngx.ctx for per-request data

This commit is contained in:
bunkerity 2023-04-18 18:23:08 +02:00
parent 29cb6fe161
commit 28ef546a9a
5 changed files with 182 additions and 130 deletions

View file

@ -94,18 +94,31 @@ helpers.fill_ctx = function()
-- Instantiate bw table
local data = {}
-- Common vars
data.kind = "http"
if not ngx.shared.cachestore then
data.kind = "stream"
end
data.ip = ngx.var.remote_addr
data.uri = ngx.var.uri
data.original_uri = ngx.var.original_uri
data.user_agent = ngx.var.http_user_agent
-- Global IP
-- IP data : global
local ip_is_global, err = utils.ip_is_global(data.ip)
if ip_is_global == nil then
table.insert(errors, "can't check if IP is global : " .. err)
else
data.ip_is_global = ip_is_global
end
-- ctx filled
-- IP data : v4 / v6
data.ip_is_ipv4 = utils.is_ipv4(data.ip)
data.ip_is_ipv6 = utils.is_ipv6(data.ip)
-- Misc info
data.integration = utils.get_integration()
data.version = utils.get_version()
-- Plugins
data.plugins = {}
-- Fill ctx
ngx.ctx.bw = data
return true, "ctx filled", errors
end

View file

@ -1,67 +1,36 @@
local cdatastore = require "bunkerweb.datastore"
local ipmatcher = require "resty.ipmatcher"
local cjson = require "cjson"
local resolver = require "resty.dns.resolver"
local mmdb = require "bunkerweb.mmdb"
local clogger = require "bunkerweb.logger"
local ipmatcher = require "resty.ipmatcher"
local resolver = require "resty.dns.resolver"
local session = require "resty.session"
local cjson = require "cjson"
local logger = clogger:new("UTILS")
local datastore = cdatastore:new()
local logger = clogger:new("UTILS")
local datastore = cdatastore:new()
local cachestore = ccachestore:new()
local utils = {}
utils.set_values = function()
local reserved_ips = {
"0.0.0.0/8",
"10.0.0.0/8",
"100.64.0.0/10",
"127.0.0.0/8",
"169.254.0.0/16",
"172.16.0.0/12",
"192.0.0.0/24",
"192.88.99.0/24",
"192.168.0.0/16",
"198.18.0.0/15",
"198.51.100.0/24",
"203.0.113.0/24",
"224.0.0.0/4",
"233.252.0.0/24",
"240.0.0.0/4",
"255.255.255.255/32"
}
local ok, err = datastore:set("misc_reserved_ips", cjson.encode({ data = reserved_ips }))
if not ok then
return false, err
end
local var_resolvers, err = datastore:get("variable_DNS_RESOLVERS")
if not var_resolvers then
return false, err
end
local list_resolvers = {}
for str_resolver in var_resolvers:gmatch("%S+") do
table.insert(list_resolvers, str_resolver)
end
ok, err = datastore:set("misc_resolvers", cjson.encode(list_resolvers))
if not ok then
return false, err
end
return true, "success"
end
local utils = {}
utils.get_variable = function(var, site_search)
-- Default site search to true
if site_search == nil then
site_search = true
end
-- Get global value
local value, err = datastore:get("variable_" .. var)
if not value then
return nil, "Can't access variable " .. var .. " from datastore : " .. err
return nil, "can't access variable " .. var .. " from datastore : " .. err
end
-- Site search case
if site_search then
-- Check if multisite is set to yes
local multisite, err = datastore:get("variable_MULTISITE")
if not multisite then
return nil, "Can't access variable MULTISITE from datastore : " .. err
return nil, "can't access variable MULTISITE from datastore : " .. err
end
-- Multisite case
if multisite == "yes" and ngx.var.server_name then
local value_site, err = datastore:get("variable_" .. ngx.var.server_name .. "_" .. var)
if value_site then
@ -73,19 +42,23 @@ utils.get_variable = function(var, site_search)
end
utils.has_variable = function(var, value)
-- Get global variable
local check_value, err = datastore:get("variable_" .. var)
if not value then
return nil, "Can't access variable " .. var .. " from datastore : " .. err
end
-- Check if multisite is set to yes
local multisite, err = datastore:get("variable_MULTISITE")
if not multisite then
return nil, "Can't access variable MULTISITE from datastore : " .. err
end
-- Multisite case
if multisite == "yes" then
local servers, err = datastore:get("variable_SERVER_NAME")
if not servers then
return nil, "Can't access variable SERVER_NAME from datastore : " .. err
end
-- Check each server
for server in servers:gmatch("%S+") do
local check_value_site, err = datastore:get("variable_" .. server .. "_" .. var)
if check_value_site and check_value_site == value then
@ -98,19 +71,23 @@ utils.has_variable = function(var, value)
end
utils.has_not_variable = function(var, value)
-- Get global variable
local check_value, err = datastore:get("variable_" .. var)
if not value then
return nil, "Can't access variable " .. var .. " from datastore : " .. err
end
-- Check if multisite is set to yes
local multisite, err = datastore:get("variable_MULTISITE")
if not multisite then
return nil, "Can't access variable MULTISITE from datastore : " .. err
end
-- Multisite case
if multisite == "yes" then
local servers, err = datastore:get("variable_SERVER_NAME")
if not servers then
return nil, "Can't access variable SERVER_NAME from datastore : " .. err
end
-- Check each server
for server in servers:gmatch("%S+") do
local check_value_site, err = datastore:get("variable_" .. server .. "_" .. var)
if check_value_site and check_value_site ~= value then
@ -122,11 +99,15 @@ utils.has_not_variable = function(var, value)
return check_value ~= value, "success"
end
function utils.get_multiple_variables(vars)
utils.get_multiple_variables = function(vars)
-- Get all keys
local keys = datastore:keys()
local result = {}
-- Loop on keys
for i, key in ipairs(keys) do
-- Loop on vars
for j, var in ipairs(vars) do
-- Filter on good ones
local _, _, server, subvar = key:find("variable_(.*)_?(" .. var .. "_?%d*)")
if subvar then
if not server or server == "" then
@ -149,10 +130,12 @@ function utils.get_multiple_variables(vars)
end
utils.is_ip_in_networks = function(ip, networks)
-- Instantiate ipmatcher
local ipm, err = ipmatcher.new(networks)
if not ipm then
return nil, "can't instantiate ipmatcher : " .. err
end
-- Match
local matched, err = ipm:match(ip)
if err then
return nil, "can't check ip : " .. err
@ -169,18 +152,31 @@ utils.is_ipv6 = function(ip)
end
utils.ip_is_global = function(ip)
local data, err = datastore:get("misc_reserved_ips")
if not data then
return nil, "can't get reserved ips : " .. err
end
local ok, reserved_ips = pcall(cjson.decode, data)
if not ok then
return nil, "can't decode json : " .. reserved_ips
end
-- Reserved, non public IPs
local reserved_ips = {
"0.0.0.0/8",
"10.0.0.0/8",
"100.64.0.0/10",
"127.0.0.0/8",
"169.254.0.0/16",
"172.16.0.0/12",
"192.0.0.0/24",
"192.88.99.0/24",
"192.168.0.0/16",
"198.18.0.0/15",
"198.51.100.0/24",
"203.0.113.0/24",
"224.0.0.0/4",
"233.252.0.0/24",
"240.0.0.0/4",
"255.255.255.255/32"
}
-- Instantiate ipmatcher
local ipm, err = ipmatcher.new(reserved_ips)
if not ipm then
return nil, "can't instantiate ipmatcher : " .. err
end
-- Match
local matched, err = ipm:match(ip)
if err then
return nil, "can't check ip : " .. err
@ -189,32 +185,49 @@ utils.ip_is_global = function(ip)
end
utils.get_integration = function()
-- Check if already in datastore
local integration, err = datastore:get("misc_integration")
if integration then
return integration
end
-- Swarm
local var, err = datastore:get("variable_SWARM_MODE")
if var == "yes" then
integration = "swarm"
else
-- Kubernetes
local var, err = datastore:get("variable_KUBERNETES_MODE")
if var == "yes" then
integration = "kubernetes"
else
local f, err = io.open("/etc/os-release", "r")
if f then
local data = f:read("*a")
if data:find("Alpine") then
integration = "docker"
else
integration = "unknown"
end
f:close()
-- Autoconf
local var, err = datastore:get("variable_AUTOCONF_MODE")
if var == "yes" then
integration = "autoconf"
else
integration = "unknown"
-- Already present (e.g. : linux)
local f, err = io.open("/usr/share/bunkerweb/INTEGRATION", "r")
if f then
integration = f:read("*a"):gsub("[\n\r]", "")
f:close()
else
local f, err = io.open("/etc/os-release", "r")
if f then
local data = f:read("*a")
f:close()
-- Docker
if data:find("Alpine") then
integration = "docker"
end
-- Strange case ...
else
integration = "unknown"
end
end
end
end
end
-- Save integration
local ok, err = datastore:set("misc_integration", integration)
if not ok then
logger:log(ngx.ERR, "can't cache integration to datastore : " .. err)
@ -223,17 +236,20 @@ utils.get_integration = function()
end
utils.get_version = function()
-- Check if already in datastore
local version, err = datastore:get("misc_version")
if version then
return version
end
-- Read VERSION file
local f, err = io.open("/usr/share/bunkerweb/VERSION", "r")
if not f then
logger:log(ngx.ERR, "can't read VERSION file : " .. err)
return "unknown"
return nil
end
version = f:read("*a")
version = f:read("*a"):gsub("[\n\r]", "")
f:close()
-- Save it to datastore
local ok, err = datastore:set("misc_version", version)
if not ok then
logger:log(ngx.ERR, "can't cache version to datastore : " .. err)
@ -242,28 +258,62 @@ utils.get_version = function()
end
utils.get_reason = function()
-- ngx.ctx
if ngx.ctx.reason then
return ngx.ctx.reason
end
-- ngx.var
if ngx.var.reason and ngx.var.reason ~= "" then
return ngx.var.reason
end
-- os.getenv
if os.getenv("REASON") == "modsecurity" then
return "modsecurity"
end
-- datastore ban
local banned, err = datastore:get("bans_ip_" .. ngx.var.remote_addr)
if banned then
return banned
end
-- unknown
if ngx.status == utils.get_deny_status() then
return "unknown"
end
return nil
end
utils.get_rdns = function(ip)
utils.get_resolvers = function()
-- Get resolvers from datastore if existing
local str_resolvers, err = datastore:get("misc_resolvers")
if not str_resolvers then
if str_resolvers then
return cjson.decode(str_resolvers)
end
-- Otherwise extract DNS_RESOLVERS variable
local var_resolvers, err = datastore:get("variable_DNS_RESOLVERS")
if not var_resolvers then
logger:log(ngx.ERR, "can't get variable DNS_RESOLVERS from datastore : " .. err)
return nil, err
end
-- Make table for resolver1 resolver2 ... string
local resolvers = {}
for str_resolver in var_resolvers:gmatch("%S+") do
table.insert(resolvers, str_resolver)
end
-- Add it to the datastore
local ok, err = datastore:set("misc_resolvers", cjson.encode(resolvers))
if not ok then
logger:log(ngx.ERR, "can't save misc_resolvers to datastore : " .. err)
end
return resolvers
end
utils.get_rdns = function(ip)
-- Get resolvers
local resolvers, err = utils.get_resolvers()
if not resolvers then
return false, err
end
local resolvers = cjson.decode(str_resolvers)
-- Instantiate resolver
local rdns, err = resolver:new {
nameservers = resolvers,
retrans = 1,
@ -272,6 +322,7 @@ utils.get_rdns = function(ip)
if not rdns then
return false, err
end
-- Do rDNS query
local answers, err = rdns:reverse_query(ip)
if not answers then
return false, err
@ -279,6 +330,7 @@ utils.get_rdns = function(ip)
if answers.errcode then
return false, answers.errstr
end
-- Return first element
for i, answer in ipairs(answers) do
if answer.ptrdname then
return answer.ptrdname, "success"
@ -288,11 +340,12 @@ utils.get_rdns = function(ip)
end
utils.get_ips = function(fqdn, resolvers)
local str_resolvers, err = datastore:get("misc_resolvers")
if not str_resolvers then
-- Get resolvers
local resolvers, err = utils.get_resolvers()
if not resolvers then
return false, err
end
local resolvers = cjson.decode(str_resolvers)
-- Instantiante resolver
local rdns, err = resolver:new {
nameservers = resolvers,
retrans = 1,
@ -301,6 +354,7 @@ utils.get_ips = function(fqdn, resolvers)
if not rdns then
return false, err
end
-- Query FQDN
local answers, err = rdns:query(fqdn, nil, {})
if not answers then
return false, err
@ -308,6 +362,7 @@ utils.get_ips = function(fqdn, resolvers)
if answers.errcode then
return {}, answers.errstr
end
-- Return all IPs
local ips = {}
for i, answer in ipairs(answers) do
if answer.address then
@ -318,9 +373,11 @@ utils.get_ips = function(fqdn, resolvers)
end
utils.get_country = function(ip)
-- Check if mmdb is loaded
if not mmdb.country_db then
return false, "mmdb country not loaded"
end
-- Perform lookup
local ok, result, err = pcall(mmdb.country_db.lookup, mmdb.country_db, ip)
if not ok then
return nil, result
@ -332,9 +389,11 @@ utils.get_country = function(ip)
end
utils.get_asn = function(ip)
-- Check if mmdp is loaded
if not mmdb.asn_db then
return false, "mmdb asn not loaded"
end
-- Perform lookup
local ok, result, err = pcall(mmdb.asn_db.lookup, mmdb.asn_db, ip)
if not ok then
return nil, result
@ -347,6 +406,7 @@ end
utils.rand = function(nb)
local charset = {}
-- lowers, uppers and numbers
for i = 48, 57 do table.insert(charset, string.char(i)) end
for i = 65, 90 do table.insert(charset, string.char(i)) end
for i = 97, 122 do table.insert(charset, string.char(i)) end
@ -358,9 +418,11 @@ utils.rand = function(nb)
end
utils.get_deny_status = function()
if ngx.var.is_stream == "yes" then
return 403
-- Stream case
if ngx.ctx.bw and ngx.ctx.bw.kind == "stream" then
return 444
end
-- http case
local status, err = datastore:get("variable_DENY_HTTP_STATUS")
if not status then
logger:log(ngx.ERR, "can't get DENY_HTTP_STATUS variable " .. err)
@ -370,13 +432,16 @@ utils.get_deny_status = function()
end
utils.get_session = function()
if ngx.ctx.session then
return ngx.ctx.session, ngx.ctx.session_err, ngx.ctx.session_exists
-- Session already in context
if ngx.ctx.bw.session then
return ngx.ctx.bw.session, ngx.ctx.bw.session_err, ngx.ctx.bw.session_exists
end
-- Open session
local _session, err, exists = session.start()
if err then
logger:log(ngx.ERR, "UTILS", "can't start session : " .. err)
logger:log(ngx.ERR, "can't start session : " .. err)
end
-- Fill ctx
ngx.ctx.session = _session
ngx.ctx.session_err = err
ngx.ctx.session_exists = exists
@ -389,6 +454,7 @@ utils.get_session = function()
end
utils.save_session = function()
-- Check if save is needed
if ngx.ctx.session and not ngx.ctx.session_err and not ngx.ctx.session_saved then
ngx.ctx.session:set_data(ngx.ctx.session_data)
local ok, err = ngx.ctx.session:save()
@ -405,6 +471,7 @@ utils.save_session = function()
end
utils.set_session = function(key, value)
-- Set new data
if ngx.ctx.session and not ngx.ctx.session_err then
ngx.ctx.session_data[key] = value
return true, "value set"
@ -413,6 +480,7 @@ utils.set_session = function(key, value)
end
utils.get_session = function(key)
-- Get data
if ngx.ctx.session and not ngx.ctx.session_err then
return true, "value get", ngx.ctx.session_data[key]
end

View file

@ -43,47 +43,6 @@ for line in io.lines("/etc/nginx/variables.env") do
end
init_logger:log(ngx.NOTICE, "saved variables into datastore")
-- Set misc values into the datastore
init_logger:log(ngx.NOTICE, "saving misc values into datastore ...")
local miscs = {
reserved_ips = {
"0.0.0.0/8",
"10.0.0.0/8",
"100.64.0.0/10",
"127.0.0.0/8",
"169.254.0.0/16",
"172.16.0.0/12",
"192.0.0.0/24",
"192.88.99.0/24",
"192.168.0.0/16",
"198.18.0.0/15",
"198.51.100.0/24",
"203.0.113.0/24",
"224.0.0.0/4",
"233.252.0.0/24",
"240.0.0.0/4",
"255.255.255.255/32"
},
resolvers = {}
}
local var_resolvers, err = ds:get("variable_DNS_RESOLVERS")
if not var_resolvers then
init_logger:log(ngx.ERR, "can't get variable DNS_RESOLVERS from datastore : " .. err)
return false
end
for str_resolver in var_resolvers:gmatch("%S+") do
table.insert(miscs.resolvers, str_resolver)
end
for k, v in pairs(miscs) do
local ok, err = ds:set("misc_" .. k, cjson.encode(v))
if not ok then
init_logger:log(ngx.ERR, "can't save misc " .. k .. " into datastore : " .. err)
return false
end
init_logger:log(ngx.INFO, "saved misc " .. k .. " into datastore")
end
init_logger:log(ngx.NOTICE, "saved misc values into datastore")
-- Set API values into the datastore
init_logger:log(ngx.NOTICE, "saving API values into datastore ...")
local value, err = ds:get("variable_USE_API")

View file

@ -30,6 +30,18 @@ if not ok then
logger:log(ngx.ERR, "can't update cachestore : " .. err)
end
-- Fill ctx
logger:log(ngx.INFO, "filling ngx.ctx ...")
local ok, ret, errors = helpers.fill_ctx()
if not ok then
logger:log(ngx.ERR, "fill_ctx() failed : " .. ret)
elseif errors then
for i, error in ipairs(errors) do
logger:log(ngx.ERR, "fill_ctx() error " .. tostring(i) .. " : " .. error)
end
end
logger:log(ngx.INFO, "ngx.ctx filled (ret = " .. ret .. ")")
-- Process bans as soon as possible
local ok, reason = datastore:get("bans_ip_" .. ngx.var.remote_addr)
if not ok and reason ~= "not found" then

View file

@ -25,8 +25,8 @@ if not plugins then
end
plugins = cjson.decode(plugins)
-- Call head() methods
logger:log(ngx.INFO, "calling head() methods of plugins ...")
-- Call header() methods
logger:log(ngx.INFO, "calling header() methods of plugins ...")
for i, plugin in ipairs(plugins) do
-- Require call
local plugin_lua, err = helpers.require_plugin(plugin.id)
@ -35,28 +35,28 @@ for i, plugin in ipairs(plugins) do
elseif plugin_lua == nil then
logger:log(ngx.INFO, err)
else
-- Check if plugin has head method
if plugin_lua.head ~= nil then
-- Check if plugin has header method
if plugin_lua.header ~= nil then
-- New call
local ok, plugin_obj = helpers.new_plugin(plugin_lua)
if not ok then
logger:log(ngx.ERR, plugin_obj)
else
local ok, ret = helpers.call_plugin(plugin_obj, "head")
local ok, ret = helpers.call_plugin(plugin_obj, "header")
if not ok then
logger:log(ngx.ERR, ret)
elseif not ret.ret then
logger:log(ngx.ERR, plugin.id .. ":head() call failed : " .. ret.msg)
logger:log(ngx.ERR, plugin.id .. ":header() call failed : " .. ret.msg)
else
logger:log(ngx.NOTICE, plugin.id .. ":head() call successful : " .. ret.msg)
logger:log(ngx.NOTICE, plugin.id .. ":header() call successful : " .. ret.msg)
end
end
else
logger:log(ngx.INFO, "skipped execution of " .. plugin.id .. " because method head() is not defined")
logger:log(ngx.INFO, "skipped execution of " .. plugin.id .. " because method header() is not defined")
end
end
end
logger:log(ngx.INFO, "called head() methods of plugins")
logger:log(ngx.INFO, "called header() methods of plugins")
return true