diff --git a/src/bw/lua/clusterstore.lua b/src/bw/lua/clusterstore.lua index fe43d62ab..234daad8a 100644 --- a/src/bw/lua/clusterstore.lua +++ b/src/bw/lua/clusterstore.lua @@ -18,7 +18,7 @@ function M:connect() ["REDIS_KEEPALIVE_POOL"] = "" } for k, v in pairs(variables) do - local value, err = utils.get_variable(k) + local value, err = utils.get_variable(k, false) if value == nil then return false, err end @@ -47,7 +47,7 @@ function M:close(redis_client) ["REDIS_KEEPALIVE_POOL"] = "" } for k, v in pairs(variables) do - local value, err = utils.get_variable(k) + local value, err = utils.get_variable(k, false) if value == nil then return false, err end diff --git a/src/bw/lua/redisutils.lua b/src/bw/lua/redisutils.lua new file mode 100644 index 000000000..db32cedbf --- /dev/null +++ b/src/bw/lua/redisutils.lua @@ -0,0 +1,73 @@ +local clusterstore = require "clusterstore" +local datastore = require "datastore" +local utils = require "utils" + +local redisutils = {} + +redisutils.ban = function(ip) + -- Connect + local redis_client, err = clusterstore:connect() + if not redis_client then + return nil, "can't connect to redis server : " .. err + end + -- Start transaction + local ok, err = redis_client:multi() + if not ok then + clusterstore:close(redis_client) + return nil, "MULTI failed : " .. err + end + -- Get ban + ok, err = redis_client:get("ban_" .. ip) + if not ok then + clusterstore:close(redis_client) + return nil, "GET failed : " .. err + end + -- Get ttl + ok, err = redis_client:ttl("ban_" .. ip) + if not ok then + clusterstore:close(redis_client) + return nil, "TTL failed : " .. err + end + -- Exec transaction + local exec, err = redis_client:exec() + if err then + clusterstore:close(redis_client) + return nil, "EXEC failed : " .. err + end + if type(exec) ~= "table" then + clusterstore:close(redis_client) + return nil, "EXEC result is not a table" + end + -- Extract ban reason + local reason = exec[1] + if type(reason) == "table" then + clusterstore:close(redis_client) + return nil, "GET failed : " .. reason[2] + end + if reason == ngx.null then + clusterstore:close(redis_client) + datastore:delete("bans_ip_" .. ip) + return false + end + -- Extract ttl + local ttl = exec[2] + if type(ttl) == "table" then + clusterstore:close(redis_client) + return nil, "TTL failed : " .. ttl[2] + end + if ttl <= 0 then + clusterstore:close(redis_client) + return nil, "TTL returned invalid value : " .. tostring(ttl) + end + ok, err = datastore:set("bans_ip_" .. ip, reason, ttl) + if not ok then + clusterstore:close(redis_client) + datastore:delete("bans_ip_" .. ip) + return nil, "can't save ban to local datastore : " .. err + end + -- Return reason + clusterstore:close(redis_client) + return true, reason +end + +return redisutils \ No newline at end of file diff --git a/src/bw/lua/utils.lua b/src/bw/lua/utils.lua index ae86ef459..f84feabce 100644 --- a/src/bw/lua/utils.lua +++ b/src/bw/lua/utils.lua @@ -1,9 +1,9 @@ -local datastore = require "datastore" -local ipmatcher = require "resty.ipmatcher" -local cjson = require "cjson" -local resolver = require "resty.dns.resolver" -local mmdb = require "mmdb" -local logger = require "logger" +local datastore = require "datastore" +local ipmatcher = require "resty.ipmatcher" +local cjson = require "cjson" +local resolver = require "resty.dns.resolver" +local mmdb = require "mmdb" +local logger = require "logger" local utils = {} diff --git a/src/common/confs/server-http/access-lua.conf b/src/common/confs/server-http/access-lua.conf index 5261fda26..000d9f58f 100644 --- a/src/common/confs/server-http/access-lua.conf +++ b/src/common/confs/server-http/access-lua.conf @@ -4,7 +4,7 @@ local logger = require "logger" local datastore = require "datastore" local plugins = require "plugins" local utils = require "utils" -local clusterstore = require "clusterstore" +local redisutils = require "redisutils" -- Don't process internal requests if ngx.req.is_internal() then @@ -15,39 +15,33 @@ end logger.log(ngx.INFO, "ACCESS", "Access phase started") -- Process bans as soon as possible -local banned, err = datastore:get("bans_ip_" .. ngx.var.remote_addr) -if banned then - logger.log(ngx.WARN, "ACCESS", "IP " .. ngx.var.remote_addr .. " is banned with reason : " .. banned) - ngx.exit(utils.get_deny_status()) -end +local banned = nil -- Redis case local use_redis = utils.get_variable("USE_REDIS") if use_redis == "yes" then - -- Connect - local redis_client, err = clusterstore:connect() - if not redis_client then - logger.log(ngx.ERR, "ACCESS", "can't connect to redis server : " .. err) + local redis_banned, reason = redisutils.ban(ngx.var.remote_addr) + if redis_banned == nil then + logger.log(ngx.ERR, "ACCESS", "Error while checking ban from redis, falling back to local : " .. reason) + elseif not redis_banned then + banned = false else - -- Get ban - local ban, err = redis_client:get("ban_" .. ngx.var.remote_addr) - if err then - logger.log(ngx.ERR, "ACCESS", "GET failed : " .. err) - elseif ban then - -- Get TTL - local ttl, err = redis_client:ttl("ban_" .. ngx.var.remote_addr) - if not ttl then - logger.log(ngx.ERR, "ACCESS", "TTL failed : " .. err) - else - local ok, err = datastore:set("bans_ip_" .. ngx.var.remote_addr, ban, ttl) - if not ok then - logger.log(ngx.ERR, "ACCESS", "can't save ban to the datastore : " .. err) - return - end - end - end - redis_client:close() + banned = reason end end +-- Local case +if banned == nil then + local reason, err = datastore:get("bans_ip_" .. ngx.var.remote_addr) + if reason then + banned = reason + else + banned = false + end +end +-- Deny request +if banned then + logger.log(ngx.WARN, "ACCESS", "IP " .. ngx.var.remote_addr .. " is banned with reason : " .. banned) + ngx.exit(utils.get_deny_status()) +end -- List all plugins local list, err = plugins:list() diff --git a/src/common/confs/server-stream/preread-stream-lua.conf b/src/common/confs/server-stream/preread-stream-lua.conf index 6eb01cf9f..5837c76d6 100644 --- a/src/common/confs/server-stream/preread-stream-lua.conf +++ b/src/common/confs/server-stream/preread-stream-lua.conf @@ -1,46 +1,41 @@ preread_by_lua_block { -local logger = require "logger" -local datastore = require "datastore" -local plugins = require "plugins" -local utils = require "utils" +local logger = require "logger" +local datastore = require "datastore" +local plugins = require "plugins" +local utils = require "utils" +local redisutils = require "redisutils" logger.log(ngx.INFO, "PREREAD", "Preread phase started") -- Process bans as soon as possible -local banned, err = datastore:get("bans_ip_" .. ngx.var.remote_addr) -if banned then - logger.log(ngx.WARN, "PREREAD", "IP " .. ngx.var.remote_addr .. " is banned with reason : " .. banned) - ngx.exit(utils.get_deny_status()) -end +local banned = nil -- Redis case local use_redis = utils.get_variable("USE_REDIS") if use_redis == "yes" then - -- Connect - local redis_client, err = clusterstore:connect() - if not redis_client then - logger.log(ngx.ERR, "PREREAD", "can't connect to redis server : " .. err) + local redis_banned, reason = redisutils.ban(ngx.var.remote_addr) + if redis_banned == nil then + logger.log(ngx.ERR, "ACCESS", "Error while checking ban from redis, falling back to local : " .. reason) + elseif not redis_banned then + banned = false else - -- Get ban - local ban, err = redis_client:get("ban_" .. ngx.var.remote_addr) - if err then - logger.log(ngx.ERR, "PREREAD", "GET failed : " .. err) - elseif ban then - -- Get TTL - local ttl, err = redis_client:ttl("ban_" .. ngx.var.remote_addr) - if not ttl then - logger.log(ngx.ERR, "PREREAD", "TTL failed : " .. err) - else - local ok, err = datastore:set("bans_ip_" .. ip, ban, ttl) - if not ok then - logger.log(ngx.ERR, "PREREAD", "can't save ban to the datastore : " .. err) - return - end - end - end - redis_client:close() + banned = reason end end +-- Local case +if banned == nil then + local reason, err = datastore:get("bans_ip_" .. ngx.var.remote_addr) + if reason then + banned = reason + else + banned = false + end +end +-- Deny request +if banned then + logger.log(ngx.WARN, "ACCESS", "IP " .. ngx.var.remote_addr .. " is banned with reason : " .. banned) + ngx.exit(utils.get_deny_status()) +end -- List all plugins local list, err = plugins:list() diff --git a/src/common/core/badbehavior/badbehavior.lua b/src/common/core/badbehavior/badbehavior.lua index 108f528fc..7663d0bf2 100644 --- a/src/common/core/badbehavior/badbehavior.lua +++ b/src/common/core/badbehavior/badbehavior.lua @@ -13,69 +13,32 @@ function _M.new() end function _M.increase(premature, use_redis, ip, count_time, ban_time, threshold) - -- Local case - local counter, err = datastore:get("plugin_badbehavior_count_" .. ip) - if not counter and err ~= "not found" then - return false, "can't get counts from the datastore : " .. err - end - if counter == nil then - counter = 0 - end - counter = counter + 1 + -- Declare counter + local counter = false -- Redis case if use_redis then - -- Connect to server - local redis_client, err = clusterstore:connect() - if not redis_client then - logger.log(ngx.ERR, "BAD-BEHAVIOR", "(increase) Can't connect to redis server : " .. err) - return + local redis_counter = _M.redis_increase(ip, count_time, ban_time, threshold) + if not redis_counter then + logger.log(ngx.ERR, "BAD-BEHAVIOR", "(increase) redis_increase failed, falling back to local") + else + counter = redis_counter end - -- Start transaction - local ok, err = redis_client:multi() - if not ok then - logger.log(ngx.ERR, "BAD-BEHAVIOR", "(increase) Can't start transaction : " .. err) - redis_client:close() - return + end + -- Local case + if not counter then + local local_counter, err = datastore:get("plugin_badbehavior_count_" .. ip) + if not local_counter and err ~= "not found" then + return false, "can't get counts from the datastore : " .. err end - -- Increment counter - counter, err = redis_client:incr("bad_behavior_" .. ip) - if not counter then - logger.log(ngx.ERR, "BAD-BEHAVIOR", "(increase) INCR failed : " .. err) - redis_client:close() - return + if local_counter == nil then + counter = 0 end - -- Expires counter - local expire, err = redis_client:expire("bad_behavior_" .. ip, count_time) - if not counter then - logger.log(ngx.ERR, "BAD-BEHAVIOR", "(increase) EXPIRE failed : " .. err) - redis_client:close() - return - end - -- Add IP to redis bans if needed - if counter > threshold then - local ban, err = redis_client:set("ban_" .. ip, "bad behavior", "EX", ban_time) - if not ban then - logger.log(ngx.ERR, "BAD-BEHAVIOR", "(increase) SET failed : " .. err) - redis_client:close() - return - end - end - -- Exec transaction - local exec, err = redis_client:exec() - if not exec then - logger.log(ngx.ERR, "BAD-BEHAVIOR", "(increase) EXEC failed : " .. err) - redis_client:close() - return - end - for i, v in ipairs(exec) do - if v[1] == false then - logger.log(ngx.ERR, "BAD-BEHAVIOR", "(increase) Transaction failed : " .. v[2]) - redis_client:close() - return - end - end - -- End connection - redis_client:close() + counter = counter + 1 + end + -- Call decrease later + local ok, err = ngx.timer.at(count_time, _M.decrease, use_redis, ip) + if not ok then + logger.log(ngx.ERR, "BAD-BEHAVIOR", "(increase) can't create decrease timer : " .. err) end -- Store local counter local ok, err = datastore:set("plugin_badbehavior_count_" .. ip, counter) @@ -92,26 +55,35 @@ function _M.increase(premature, use_redis, ip, count_time, ban_time, threshold) end logger.log(ngx.WARN, "BAD-BEHAVIOR", "IP " .. ip .. " is banned for " .. ban_time .. "s (" .. tostring(counter) .. "/" .. tostring(threshold) .. ")") end - -- Call decrease later - local ok, err = ngx.timer.at(count_time, _M.decrease, use_redis, ip) - if not ok then - logger.log(ngx.ERR, "BAD-BEHAVIOR", "(increase) can't create decrease timer : " .. err) - end end function _M.decrease(premature, use_redis, ip) - -- Decrease from local store - local count, err = datastore:get("plugin_badbehavior_count_" .. ip) - if err then - logger.log(ngx.ERR, "BAD-BEHAVIOR", "(decrease) Can't get counts from the datastore : " .. err) - return + -- Declare counter + local counter = false + -- Redis case + if use_redis then + local redis_counter = _M.redis_decrease(ip) + if not redis_counter then + logger.log(ngx.ERR, "BAD-BEHAVIOR", "(increase) redis_decrease failed, falling back to local") + else + counter = redis_counter + end end - if not count then - logger.log(ngx.ERR, "BAD-BEHAVIOR", "(decrease) Count is null") - return + -- Local case + if not counter then + local local_counter, err = datastore:get("plugin_badbehavior_count_" .. ip) + if err then + logger.log(ngx.ERR, "BAD-BEHAVIOR", "(decrease) Can't get counts from the datastore : " .. err) + return + end + if not local_counter then + logger.log(ngx.ERR, "BAD-BEHAVIOR", "(decrease) Count is null") + return + end + counter = local_counter - 1 end - local new_count = count - 1 - if new_count <= 0 then + -- Update local counter + if counter <= 0 then datastore:delete("plugin_badbehavior_count_" .. ip) return end @@ -120,23 +92,6 @@ function _M.decrease(premature, use_redis, ip) logger.log(ngx.ERR, "BAD-BEHAVIOR", "(decrease) Can't save counts to the datastore : " .. err) return end - -- Decrease from redis - if use_redis then - -- Connect to server - local redis_client, err = clusterstore:connect() - if not redis_client then - logger.log(ngx.ERR, "BAD-BEHAVIOR", "(decrease) Can't connect to redis server : " .. err) - return - end - -- Decrement counter - local counter, err = redis_client:decr("bad_behavior_" .. ip) - if not counter then - logger.log(ngx.ERR, "BAD-BEHAVIOR", "(decrease) DECR failed : " .. err) - redis_client:close() - return - end - redis_client:close() - end end function _M:log() @@ -159,6 +114,11 @@ function _M:log() if ngx.var.is_whitelisted == "yes" then return true, "client is whitelisted" end + -- Check if we are already banned + local banned, err = datastore:get("bans_ip_" .. ngx.var.remote_addr) + if banned then + return true, "already banned" + end -- Call increase function later and with cosocket enabled local use_redis = false if self.use_redis == "yes" then @@ -166,7 +126,7 @@ function _M:log() end local ok, err = ngx.timer.at(0, _M.increase, use_redis, ngx.var.remote_addr, tonumber(self.count_time), tonumber(self.ban_time), tonumber(self.threshold)) if not ok then - return false, "can't create decrease timer : " .. err + return false, "can't create increase timer : " .. err end return true, "success" end @@ -175,4 +135,103 @@ function _M:log_default() return _M:log() end +function _M.redis_increase(ip, count_time, ban_time, threshold) + -- Connect to server + local redis_client, err = clusterstore:connect() + if not redis_client then + logger.log(ngx.ERR, "BAD-BEHAVIOR", "(increase) Can't connect to redis server : " .. err) + return false + end + -- Start transaction + local ok, err = redis_client:multi() + if not ok then + logger.log(ngx.ERR, "BAD-BEHAVIOR", "(increase) Can't start transaction : " .. err) + clusterstore:close(redis_client) + return false + end + -- Increment counter + ok, err = redis_client:incr("bad_behavior_" .. ip) + if not ok then + logger.log(ngx.ERR, "BAD-BEHAVIOR", "(increase) INCR failed : " .. err) + clusterstore:close(redis_client) + return false + end + -- Expires counter + ok, err = redis_client:expire("bad_behavior_" .. ip, count_time) + if not ok then + logger.log(ngx.ERR, "BAD-BEHAVIOR", "(increase) EXPIRE failed : " .. err) + clusterstore:close(redis_client) + return false + end + -- Exec transaction + local exec, err = redis_client:exec() + if err then + logger.log(ngx.ERR, "BAD-BEHAVIOR", "(increase) EXEC failed : " .. err) + clusterstore:close(redis_client) + return false + end + if type(exec) ~= "table" then + logger.log(ngx.ERR, "BAD-BEHAVIOR", "(increase) EXEC result is not a table") + clusterstore:close(redis_client) + return false + end + -- Extract counter + local counter = exec[1] + if type(counter) == "table" then + logger.log(ngx.ERR, "BAD-BEHAVIOR", "(increase) INCR error : " .. counter[2]) + clusterstore:close(redis_client) + return false + end + -- Check expire result + local expire = exec[2] + if type(expire) == "table" then + logger.log(ngx.ERR, "BAD-BEHAVIOR", "(increase) EXPIRE error : " .. expire[2]) + clusterstore:close(redis_client) + return false + end + -- Add IP to redis bans if needed + if counter > threshold then + local ban, err = redis_client:set("ban_" .. ip, "bad behavior", "EX", ban_time) + if err then + logger.log(ngx.ERR, "BAD-BEHAVIOR", "(increase) SET failed : " .. err) + clusterstore:close(redis_client) + return false + end + end + -- End connection + clusterstore:close(redis_client) + return counter +end + +function _M.redis_decrease(ip) + -- Connect to server + local redis_client, err = clusterstore:connect() + if not redis_client then + logger.log(ngx.ERR, "BAD-BEHAVIOR", "(decrease) Can't connect to redis server : " .. err) + return false + end + -- Decrement counter + local counter, err = redis_client:decr("bad_behavior_" .. ip) + if err then + logger.log(ngx.ERR, "BAD-BEHAVIOR", "(decrease) DECR failed : " .. err) + clusterstore:close(redis_client) + return false + end + -- Delete counter + if counter < 0 then + counter = 0 + end + if counter == 0 then + local ok, err = redis_client:del("bad_behavior_" .. ip) + if err then + logger.log(ngx.ERR, "BAD-BEHAVIOR", "(decrease) DEL failed : " .. err) + clusterstore:close(redis_client) + return false + end + end + -- End connection + clusterstore:close(redis_client) + return counter +end + return _M