diff --git a/core/blacklist/blacklist.lua b/core/blacklist/blacklist.lua index a0f30a8a3..ce579183f 100644 --- a/core/blacklist/blacklist.lua +++ b/core/blacklist/blacklist.lua @@ -27,7 +27,12 @@ function _M:init() ["RDNS"] = {}, ["ASN"] = {}, ["USER_AGENT"] = {}, - ["URI"] = {} + ["URI"] = {}, + ["IGNORE_IP"] = {}, + ["IGNORE_RDNS"] = {}, + ["IGNORE_ASN"] = {}, + ["IGNORE_USER_AGENT"] = {}, + ["IGNORE_URI"] = {}, } local i = 0 for kind, _ in pairs(blacklists) do @@ -60,22 +65,34 @@ function _M:access() -- Check the cache local cached_ip, err = self:is_in_cache("ip" .. ngx.var.remote_addr) - if cached_ip and cached_ip ~= "ok" then + local cached_ignored_ip, err = self:is_in_cache("ignore_ip" .. ngx.var.remote_addr) + if cached_ignored_ip then + logger.log(ngx.NOTICE, "BLACKLIST", "IP is in cached ignore blacklist (info: " .. cached_ignored_ip .. ")") + elseif cached_ip and cached_ip ~= "ok" then return true, "IP is in blacklist cache (info = " .. cached_ip .. ")", true, utils.get_deny_status() end local cached_uri, err = self:is_in_cache("uri" .. ngx.var.uri) - if cached_uri and cached_uri ~= "ok" then + local cached_ignored_uri, err = self:is_in_cache("ignore_uri" .. ngx.var.uri) + if cached_ignored_uri then + logger.log(ngx.NOTICE, "BLACKLIST", "URI is in cached ignore blacklist (info: " .. cached_ignored_uri .. ")") + elseif cached_uri and cached_uri ~= "ok" then return true, "URI is in blacklist cache (info = " .. cached_uri .. ")", true, utils.get_deny_status() end local cached_ua = true + local cached_ignored_ua = false if ngx.var.http_user_agent then cached_ua, err = self:is_in_cache("ua" .. ngx.var.http_user_agent) - if cached_ua and cached_ua ~= "ok" then + cached_ignored_ua, err = self:is_in_cache("ignore_ua" .. ngx.var.http_user_agent) + if cached_ignored_ua then + logger.log(ngx.NOTICE, "BLACKLIST", "User-Agent is in cached ignore blacklist (info: " .. cached_ignored_ua .. ")") + elseif cached_ua and cached_ua ~= "ok" then return true, "User-Agent is in blacklist cache (info = " .. cached_ua .. ")", true, utils.get_deny_status() end end - if cached_ip and cached_uri and cached_ua then - return true, "full request is in blacklist cache (not blacklisted)", nil, nil + if cached_ignored_ip and cached_ignored_uri and cached_ignored_ua then + logger.log(ngx.NOTICE, "BLACKLIST", "full request is in cached ignore blacklist") + elseif cached_ip and cached_uri and cached_ua then + return true, "full request is in blacklist cache (not blacklisted)", false, nil end -- Get list @@ -93,24 +110,42 @@ function _M:access() -- Check if IP is in IP/net blacklist local ip_net, err = utils.get_variable("BLACKLIST_IP") + local ignored_ip_net, err = utils.get_variable("BLACKLIST_IGNORE_IP") if ip_net and ip_net ~= "" then for element in ip_net:gmatch("%S+") do table.insert(blacklists["IP"], element) end end + if ignored_ip_net and ignored_ip_net ~= "" then + for element in ignored_ip_net:gmatch("%S+") do + table.insert(blacklists["IGNORE_IP"], element) + end + end if not cached_ip then local ipm, err = ipmatcher.new(blacklists["IP"]) + local ipm_ignore, err_ignore = ipmatcher.new(blacklists["IGNORE_IP"]) if not ipm then ret = false ret_err = "can't instantiate ipmatcher " .. err + elseif not ipm_ignore then + ret = false + ret_err = "can't instantiate ipmatcher " .. err_ignore else if ipm:match(ngx.var.remote_addr) then - self:add_to_cache("ip" .. ngx.var.remote_addr, "ip/net") - return ret, "client IP " .. ngx.var.remote_addr .. " is in blacklist", true, utils.get_deny_status() + if ipm_ignore:match(ngx.var.remote_addr) then + self:add_to_cache("ignore_ip" .. ngx.var.remote_addr, "ip/net") + logger.log(ngx.NOTICE, "BLACKLIST", "client IP " .. ngx.var.remote_addr .. " is in blacklist but is ignored") + else + self:add_to_cache("ip" .. ngx.var.remote_addr, "ip/net") + return ret, "client IP " .. ngx.var.remote_addr .. " is in blacklist", true, utils.get_deny_status() + end end end end + -- Instantiate ignore variable + local ignore = false + -- Check if rDNS is in blacklist local rdns_global, err = utils.get_variable("BLACKLIST_RDNS_GLOBAL") local check = true @@ -129,15 +164,31 @@ function _M:access() ret_err = "error while trying to get reverse dns : " .. err else local rdns_list, err = utils.get_variable("BLACKLIST_RDNS") + local ignored_rdns_list, err = utils.get_variable("BLACKLIST_IGNORE_RDNS") if rdns_list and rdns_list ~= "" then for element in rdns_list:gmatch("%S+") do table.insert(blacklists["RDNS"], element) end end + if ignored_rdns_list and ignored_rdns_list ~= "" then + for element in ignored_rdns_list:gmatch("%S+") do + table.insert(blacklists["IGNORE_RDNS"], element) + end + end for i, suffix in ipairs(blacklists["RDNS"]) do if rdns:sub(-#suffix) == suffix then - self:add_to_cache("ip" .. ngx.var.remote_addr, "rDNS " .. suffix) - return ret, "client IP " .. ngx.var.remote_addr .. " is in blacklist (info = rDNS " .. suffix .. ")", true, utils.get_deny_status() + for j, ignore_suffix in ipairs(blacklists["IGNORE_RDNS"]) do + if rdns:sub(-#ignore_suffix) == ignore_suffix then + ignore = true + self:add_to_cache("ignore_rdns" .. ngx.var.remote_addr, "rDNS" .. suffix) + logger.log(ngx.NOTICE, "BLACKLIST", "client IP " .. ngx.var.remote_addr .. " is in blacklist (info = rDNS " .. suffix .. ") but is ignored") + break + end + end + if not ignore then + self:add_to_cache("ip" .. ngx.var.remote_addr, "rDNS" .. suffix) + return ret, "client IP " .. ngx.var.remote_addr .. " is in blacklist (info = rDNS " .. suffix .. ")", true, utils.get_deny_status() + end end end end @@ -152,15 +203,31 @@ function _M:access() ret_err = "error while trying to get asn number : " .. err else local asn_list, err = utils.get_variable("BLACKLIST_ASN") + local ignored_asn_list, err = utils.get_variable("BLACKLIST_IGNORE_ASN") if asn_list and asn_list ~= "" then for element in asn_list:gmatch("%S+") do table.insert(blacklists["ASN"], element) end end + if ignored_asn_list and ignored_asn_list ~= "" then + for element in ignored_asn_list:gmatch("%S+") do + table.insert(blacklists["IGNORE_ASN"], element) + end + end for i, asn_bl in ipairs(blacklists["ASN"]) do if tostring(asn) == asn_bl then - self:add_to_cache("ip" .. ngx.var.remote_addr, "ASN " .. tostring(asn)) - return ret, "client IP " .. ngx.var.remote_addr .. " is in blacklist (kind = ASN " .. tostring(asn) .. ")", true, utils.get_deny_status() + for j, ignore_asn_bl in ipairs(blacklists["IGNORE_ASN"]) do + if tostring(asn) == ignore_asn_bl then + ignore = true + self:add_to_cache("ignore_asn" .. ngx.var.remote_addr, "ASN" .. tostring(asn)) + logger.log(ngx.NOTICE, "BLACKLIST", "client IP " .. ngx.var.remote_addr .. " is in blacklist (info = ASN " .. tostring(asn) .. ") but is ignored") + break + end + end + if not ignore then + self:add_to_cache("ip" .. ngx.var.remote_addr, "ASN " .. tostring(asn)) + return ret, "client IP " .. ngx.var.remote_addr .. " is in blacklist (kind = ASN " .. tostring(asn) .. ")", true, utils.get_deny_status() + end end end end @@ -177,15 +244,31 @@ function _M:access() -- Check if User-Agent is in blacklist if not cached_ua and ngx.var.http_user_agent then local ua_list, err = utils.get_variable("BLACKLIST_USER_AGENT") + local ignored_ua_list, err = utils.get_variable("BLACKLIST_IGNORE_USER_AGENT") if ua_list and ua_list ~= "" then for element in ua_list:gmatch("%S+") do table.insert(blacklists["USER_AGENT"], element) end end + if ignored_ua_list and ignored_ua_list ~= "" then + for element in ignored_ua_list:gmatch("%S+") do + table.insert(blacklists["IGNORE_USER_AGENT"], element) + end + end for i, ua_bl in ipairs(blacklists["USER_AGENT"]) do if ngx.var.http_user_agent:match(ua_bl) then - self:add_to_cache("ua" .. ngx.var.http_user_agent, "UA " .. ua_bl) - return ret, "client User-Agent " .. ngx.var.http_user_agent .. " is in blacklist (matched " .. ua_bl .. ")", true, utils.get_deny_status() + for j, ignore_ua_bl in ipairs(blacklists["IGNORE_USER_AGENT"]) do + if ngx.var.http_user_agent:match(ignore_ua_bl) then + ignore = true + self:add_to_cache("ignore_ua" .. ngx.var.remote_addr, "UA" .. ua_bl) + logger.log(ngx.NOTICE, "BLACKLIST", "client User-Agent " .. ngx.var.http_user_agent .. " is in blacklist (matched " .. ua_bl .. ") but is ignored") + break + end + end + if not ignore then + self:add_to_cache("ua" .. ngx.var.http_user_agent, "UA " .. ua_bl) + return ret, "client User-Agent " .. ngx.var.http_user_agent .. " is in blacklist (matched " .. ua_bl .. ")", true, utils.get_deny_status() + end end end -- UA is not blacklisted @@ -199,15 +282,31 @@ function _M:access() -- Check if URI is in blacklist if not cached_uri then local uri_list, err = utils.get_variable("BLACKLIST_URI") + local ignored_uri_list, err = utils.get_variable("BLACKLIST_IGNORE_URI") if uri_list and uri_list ~= "" then for element in uri_list:gmatch("%S+") do table.insert(blacklists["URI"], element) end end + if ignored_uri_list and ignored_uri_list ~= "" then + for element in ignored_uri_list:gmatch("%S+") do + table.insert(blacklists["IGNORE_URI"], element) + end + end for i, uri_bl in ipairs(blacklists["URI"]) do if ngx.var.uri:match(uri_bl) then - self:add_to_cache("uri" .. ngx.var.uri, "URI " .. uri_bl) - return ret, "client URI " .. ngx.var.uri .. " is in blacklist (matched " .. uri_bl .. ")", true, utils.get_deny_status() + for j, ignore_uri_bl in ipairs(blacklists["IGNORE_URI"]) do + if ngx.var.uri:match(ignore_uri_bl) then + ignore = true + self:add_to_cache("ignore_uri" .. ngx.var.remote_addr, "URI" .. uri_bl) + logger.log(ngx.NOTICE, "BLACKLIST", "client URI " .. ngx.var.uri .. " is in blacklist (matched " .. uri_bl .. ") but is ignored") + break + end + end + if not ignore then + self:add_to_cache("uri" .. ngx.var.uri, "URI " .. uri_bl) + return ret, "client URI " .. ngx.var.uri .. " is in blacklist (matched " .. uri_bl .. ")", true, utils.get_deny_status() + end end end end diff --git a/core/blacklist/jobs/blacklist-download.py b/core/blacklist/jobs/blacklist-download.py index 439099516..ad1d00b8f 100755 --- a/core/blacklist/jobs/blacklist-download.py +++ b/core/blacklist/jobs/blacklist-download.py @@ -75,7 +75,12 @@ try : "RDNS": True, "ASN" : True, "USER_AGENT": True, - "URI": True + "URI": True, + "IGNORE_IP": True, + "IGNORE_RDNS": True, + "IGNORE_ASN" : True, + "IGNORE_USER_AGENT": True, + "IGNORE_URI": True } all_fresh = True for kind in kinds_fresh : @@ -94,7 +99,12 @@ try : "RDNS": [], "ASN" : [], "USER_AGENT": [], - "URI": [] + "URI": [], + "IGNORE_IP": [], + "IGNORE_RDNS": [], + "IGNORE_ASN" : [], + "IGNORE_USER_AGENT": [], + "IGNORE_URI": [] } for kind in urls : for url in os.getenv("BLACKLIST_" + kind + "_URLS", "").split(" ") : diff --git a/core/blacklist/plugin.json b/core/blacklist/plugin.json index fd1983af3..09f8ce55d 100644 --- a/core/blacklist/plugin.json +++ b/core/blacklist/plugin.json @@ -112,6 +112,96 @@ "label": "Blacklist URI URLs", "regex": "^.*$", "type": "text" + }, + "BLACKLIST_IGNORE_IP_URLS": { + "context": "global", + "default": "", + "help": "List of URLs, separated with spaces, containing IP/network to ignore in the blacklist.", + "id": "blacklist-ip-urls", + "label": "Blacklist IP/network URLs", + "regex": "^.*$", + "type": "text" + }, + "BLACKLIST_IGNORE_IP": { + "context": "multisite", + "default": "", + "help": "List of IP/network, separated with spaces, to ignore in the blacklist.", + "id": "blacklist-ip", + "label": "Blacklist IP/network", + "regex": "^.*$", + "type": "text" + }, + "BLACKLIST_IGNORE_RDNS": { + "context": "multisite", + "default": "", + "help": "List of reverse DNS suffixes, separated with spaces, to ignore in the blacklist.", + "id": "blacklist-rdns", + "label": "Blacklist reverse DNS", + "regex": "^.*$", + "type": "text" + }, + "BLACKLIST_IGNORE_RDNS_URLS": { + "context": "global", + "default": "", + "help": "List of URLs, separated with spaces, containing reverse DNS suffixes to ignore in the blacklist.", + "id": "blacklist-rdns-urls", + "label": "Blacklist reverse DNS URLs", + "regex": "^.*$", + "type": "text" + }, + "BLACKLIST_IGNORE_ASN": { + "context": "multisite", + "default": "", + "help": "List of ASN numbers, separated with spaces, to ignore in the blacklist.", + "id": "blacklist-asn", + "label": "Blacklist ASN", + "regex": "^.*$", + "type": "text" + }, + "BLACKLIST_IGNORE_ASN_URLS": { + "context": "global", + "default": "", + "help": "List of URLs, separated with spaces, containing ASN to ignore in the blacklist.", + "id": "blacklist-rdns-urls", + "label": "Blacklist ASN URLs", + "regex": "^.*$", + "type": "text" + }, + "BLACKLIST_IGNORE_USER_AGENT": { + "context": "multisite", + "default": "", + "help": "List of User-Agent, separated with spaces, to ignore in the blacklist.", + "id": "blacklist-user-agent", + "label": "Blacklist User-Agent", + "regex": "^.*$", + "type": "text" + }, + "BLACKLIST_IGNORE_USER_AGENT_URLS": { + "context": "global", + "default": "", + "help": "List of URLs, separated with spaces, containing User-Agent to ignore in the blacklist.", + "id": "blacklist-user-agent-urls", + "label": "Blacklist User-Agent URLs", + "regex": "^.*$", + "type": "text" + }, + "BLACKLIST_IGNORE_URI": { + "context": "multisite", + "default": "", + "help": "List of URI, separated with spaces, to ignore in the blacklist.", + "id": "blacklist-uri", + "label": "Blacklist URI", + "regex": "^.*$", + "type": "text" + }, + "BLACKLIST_IGNORE_URI_URLS": { + "context": "global", + "default": "", + "help": "List of URLs, separated with spaces, containing URI to ignore in the blacklist.", + "id": "blacklist-uri-urls", + "label": "Blacklist URI URLs", + "regex": "^.*$", + "type": "text" } }, "jobs": [