bw - various fixes for tls management and init work on shared ctx on subrequests

src/bw/lua/bunkerweb/ctx.lua

@@ -0,0 +1,108 @@
+-- A module for sharing ngx.ctx between subrequests.
+-- Original work by Alex Zhang (openresty/lua-nginx-module/issues/1057)
+-- updated by 3scale/apicast.
+--
+-- Copyright (c) 2016 3scale Inc.
+-- Licensed under the Apache License, Version 2.0.
+-- License text: See LICENSE
+--
+-- Modifications by Kong Inc.
+--   * updated module functions signatures
+--   * made module function idempotent
+--   * replaced thrown errors with warn logs
+--   * allow passing of context
+--   * updated to work with new 1.19.x apis
+
+local ffi = require "ffi"
+local base = require "resty.core.base"
+require "resty.core.ctx"
+
+
+local C = ffi.C
+local ngx = ngx
+local var = ngx.var
+local ngx_log = ngx.log
+local ngx_WARN = ngx.WARN
+local tonumber = tonumber
+local registry = debug.getregistry()
+local subsystem = ngx.config.subsystem
+local get_request = base.get_request
+
+-- BW edits
+local logger = require "bunkerweb.logger":new("CTX")
+local ngx_ERR = ngx.ERR
+
+local ngx_lua_ffi_get_ctx_ref
+if subsystem == "http" then
+  ngx_lua_ffi_get_ctx_ref = C.ngx_http_lua_ffi_get_ctx_ref
+elseif subsystem == "stream" then
+  ngx_lua_ffi_get_ctx_ref = C.ngx_stream_lua_ffi_get_ctx_ref
+end
+
+
+local in_ssl_phase = ffi.new("int[1]")
+local ssl_ctx_ref = ffi.new("int[1]")
+
+
+local FFI_NO_REQ_CTX = base.FFI_NO_REQ_CTX
+
+
+local _M = {}
+
+
+function _M.stash_ref(ctx)
+  local r = get_request()
+  if not r then
+    logger:log(ngx_ERR, "could not stash ngx.ctx ref: no request found")
+    return
+  end
+
+  do
+    local ctx_ref = var.ctx_ref
+    if not ctx_ref or ctx_ref ~= "" then
+      return
+    end
+
+    if not ctx then
+      local _ = ngx.ctx -- load context if not previously loaded
+    end
+  end
+  local ctx_ref = ngx_lua_ffi_get_ctx_ref(r, in_ssl_phase, ssl_ctx_ref)
+  if ctx_ref == FFI_NO_REQ_CTX then
+    logger:log(ngx_ERR, "could not stash ngx.ctx ref: no ctx found")
+    return
+  end
+
+  var.ctx_ref = ctx_ref
+end
+
+
+function _M.apply_ref()
+  local r = get_request()
+  if not r then
+    logger:log(ngx_ERR, "could not apply ngx.ctx: no request found")
+    return
+  end
+
+  local ctx_ref = var.ctx_ref
+  if not ctx_ref or ctx_ref == "" then
+    return
+  end
+
+  ctx_ref = tonumber(ctx_ref)
+  if not ctx_ref then
+    return
+  end
+
+  local orig_ctx = registry.ngx_lua_ctx_tables[ctx_ref]
+  if not orig_ctx then
+    logger:log(ngx_ERR, "could not apply ngx.ctx: no ctx found")
+    return
+  end
+
+  ngx.ctx = orig_ctx
+  var.ctx_ref = ""
+end
+
+
+return _M

src/bw/lua/bunkerweb/helpers.lua

@@ -1,5 +1,7 @@
 local cjson = require "cjson"
 local utils = require "bunkerweb.utils"
+local bwctx = require "bunkerweb.ctx"
+local base 	= require "resty.core.base"
 
 local helpers = {}
 
@@ -148,6 +150,10 @@ end
 helpers.fill_ctx = function()
 	-- Return errors as table
 	local errors = {}
+	-- Try to load saved ctx
+	if base.get_request() then
+		bwctx.apply_ref()
+	end
 	local ctx = ngx.ctx
 	-- Check if ctx is already filled
 	if not ctx.bw then
@@ -200,6 +206,12 @@ helpers.fill_ctx = function()
 	return true, "ctx filled", errors, ctx
 end
 
+helpers.save_ctx = function(ctx)
+	if base.get_request() then
+		bwctx.stash_ref(ctx)
+	end
+end
+
 function helpers.load_variables(all_variables, plugins)
 	-- Extract settings from plugins and global ones
 	local all_settings = {}

src/common/confs/server-http/access-lua.conf

@@ -36,6 +36,8 @@ access_by_lua_block {
 		if banned == nil then
 			logger:log(ngx.ERR, "can't check if IP " .. ctx.bw.remote_addr .. " is banned : " .. reason)
 		elseif banned then
+			ctx.bw.is_banned = true
+			helpers.save_ctx(ctx)
 			logger:log(ngx.WARN,
 				"IP " .. ctx.bw.remote_addr .. " is banned with reason " .. reason .. " (" .. tostring(ttl) .. "s remaining)")
 			return ngx.exit(utils.get_deny_status(ctx))
@@ -101,7 +103,7 @@ access_by_lua_block {
 	logger:log(ngx.INFO, "called access() methods of plugins")
 
 	-- Save ctx
-	ngx.ctx = ctx
+	helpers.save_ctx(ctx)
 
 	logger:log(ngx.INFO, "access phase ended")
 

src/common/confs/server-http/header-lua.conf

@@ -63,7 +63,7 @@ header_filter_by_lua_block {
 	logger:log(ngx.INFO, "called header() methods of plugins")
 
 	-- Save ctx
-	ngx.ctx = ctx
+	helpers.save_ctx(ctx)
 
 	return true
 }

src/common/confs/server-http/log-lua.conf

@@ -63,12 +63,9 @@ log_by_lua_block {
 	logger:log(ngx.INFO, "called log() methods of plugins")
 
 	-- Display reason at info level
-	if ctx.reason then
-		logger:log(ngx.INFO, "client was denied with reason : " .. ctx.reason)
+	if ctx.bw.reason then
+		logger:log(ngx.INFO, "client was denied with reason : " .. ctx.bw.reason)
 	end
 
-	-- Save ctx
-	ngx.ctx = ctx
-
 	logger:log(ngx.INFO, "log phase ended")
 }

src/common/confs/server-http/server.conf

@@ -18,8 +18,9 @@ server {
 	include /etc/bunkerweb/configs/server-http/{{ SERVER_NAME.split(" ")[0] }}/*.conf;
 {% endif %}
 
-	# reason variable
+	# variables
 	set $reason '';
+	set $ctx_ref '';
 
 	# include LUA files
 	include {{ NGINX_PREFIX }}set-lua.conf;

src/common/confs/server-http/set-lua.conf

@@ -78,7 +78,7 @@ set_by_lua_block $dummy_set {
 	logger:log(ngx.INFO, "called set() methods of plugins")
 
 	-- Save ctx
-	ngx.ctx = ctx
+	helpers.save_ctx(ctx)
 
 	return true
 }

src/common/confs/server-stream/log-stream-lua.conf

@@ -63,12 +63,9 @@ log_by_lua_block {
 	logger:log(ngx.INFO, "called log_stream() methods of plugins")
 
 	-- Display reason at info level
-	if ctx.reason then
-		logger:log(ngx.INFO, "client was denied with reason : " .. ctx.reason)
+	if ctx.bw.reason then
+		logger:log(ngx.INFO, "client was denied with reason : " .. ctx.bw.reason)
 	end
 
-	-- Save ctx
-	ngx.ctx = ctx
-
 	logger:log(ngx.INFO, "log phase ended")
 }

src/common/confs/server-stream/preread-stream-lua.conf

@@ -31,9 +31,11 @@ preread_by_lua_block {
 		if banned == nil then
 			logger:log(ngx.ERR, "can't check if IP " .. ctx.bw.remote_addr .. " is banned : " .. reason)
 		elseif banned then
+			ctx.bw.is_banned = true
+			helpers.save_ctx(ctx)
 			logger:log(ngx.WARN,
 				"IP " .. ctx.bw.remote_addr .. " is banned with reason " .. reason .. " (" .. tostring(ttl) .. "s remaining)")
-			return ngx.exit(utils.get_deny_status())
+			return ngx.exit(utils.get_deny_status(ctx))
 		else
 			logger:log(ngx.INFO, "IP " .. ctx.bw.remote_addr .. " is not banned")
 		end
@@ -91,7 +93,7 @@ preread_by_lua_block {
 	logger:log(ngx.INFO, "called preread() methods of plugins")
 
 	-- Save ctx
-	ngx.ctx = ctx
+	helpers.save_ctx(ctx)
 
 	logger:log(ngx.INFO, "preread phase ended")
 

src/common/confs/server-stream/server-stream.conf

@@ -14,10 +14,9 @@ server {
 	include /etc/bunkerweb/configs/server-stream/{{ SERVER_NAME.split(" ")[0] }}/*.conf;
 {% endif %}
 
-	# reason variable
+	# variables
 	set $reason '';
-
-	# server_name variable
+	set $ctx_ref '';
 	set $server_name '{{ SERVER_NAME.split(" ")[0] }}';
 
 	# include LUA files

src/common/core/badbehavior/badbehavior.lua

@@ -23,8 +23,7 @@ function badbehavior:log()
 		return self:ret(true, "not increasing counter")
 	end
 	-- Check if we are already banned
-	local banned, _ = self.datastore:get("bans_ip_" .. self.ctx.bw.remote_addr)
-	if banned then
+	if self.ctx.bw.is_banned then
 		return self:ret(true, "already banned")
 	end
 	-- Call increase function later and with cosocket enabled

src/common/core/customcert/customcert.lua

@@ -18,34 +18,19 @@ function customcert:init()
 			return self:ret(false, "can't get MULTISITE variable : " .. err)
 		end
 		if multisite == "yes" then
-			local vars, err = utils.get_multiple_variables({"USE_CUSTOM_SSL"})
+			local vars, err = utils.get_multiple_variables({"USE_CUSTOM_SSL", "SERVER_NAME"})
 			if not vars then
 				return self:ret(false, "can't get USE_CUSTOM_SSL variables : " .. err)
 			end
-			if vars["global"]["USE_CUSTOM_SSL"] == "yes" then
-				local check, data = self:read_files()
-				if not check then
-					self.logger:log(ngx.ERR, "error while reading files : " .. err)
-					ret_ok = false
-					ret_err = "error reading files"
-				else
-					local check, err = self:load_data(data)
-					if not check then
-						self.logger:log(ngx.ERR, "error while loading data : " .. err)
-						ret_ok = false
-						ret_err = "error loading data"
-					end
-				end
-			end
 			for server_name, multisite_vars in pairs(vars) do
-				if multisite_vars["USE_CUSTOM_SSL"] == "yes" then
+				if multisite_vars["USE_CUSTOM_SSL"] == "yes" and server_name ~= "global" then
 					local check, data = self:read_files(server_name)
 					if not check then
 						self.logger:log(ngx.ERR, "error while reading files : " .. data)
 						ret_ok = false
 						ret_err = "error reading files"
 					else
-						local check, err = self:load_data(data, server_name)
+						local check, err = self:load_data(data, multisite_vars["SERVER_NAME"])
 						if not check then
 							self.logger:log(ngx.ERR, "error while loading data : " .. err)
 							ret_ok = false
@@ -55,13 +40,17 @@ function customcert:init()
 				end
 			end
 		else
-			local check, data = self:read_files()
+			local server_name, err = utils.get_variable("SERVER_NAME", false)
+			if not server_name then
+				return self:ret(false, "can't get SERVER_NAME variable : " .. err)
+			end
+			local check, data = self:read_files(server_name:match("%S+"))
 			if not check then
 				self.logger:log(ngx.ERR, "error while reading files : " .. data)
 				ret_ok = false
 				ret_err = "error reading files"
 			else
-				local check, err = self:load_data(data)
+				local check, err = self:load_data(data, server_name)
 				if not check then
 					self.logger:log(ngx.ERR, "error while loading data : " .. err)
 					ret_ok = false
@@ -70,7 +59,7 @@ function customcert:init()
 			end
 		end
 	else
-		ret_err = "custom ssl is not used"
+		ret_err = "custom cert is not used"
     end
 	return self:ret(ret_ok, ret_err)
 end
@@ -81,26 +70,19 @@ function customcert:ssl_certificate()
 		return self:ret(false, "can't get server_name : " .. err)
 	end
     if self.variables["USE_CUSTOM_SSL"] == "yes" then
-		local global_data, err = self.datastore:get("plugin_customcert_global", true)
-		if not global_data and err ~= "not found" then
-			return self:ret(false, "error while getting plugin_customcert_global from datastore : " .. err)
-		end
-		local site_data, err = self.datastore:get("plugin_customcert_" .. server_name, true)
-		if not site_data and err ~= "not found" then
+		local data, err = self.datastore:get("plugin_customcert_" .. server_name, true)
+		if not data then
 			return self:ret(false, "error while getting plugin_customcert_" .. server_name .. " from datastore : " .. err)
 		end
-		if not global_data and not site_data then
-			return self:ret(false, "both global and site cert are not present in datastore")
-		end
-        return self:ret(true, "certificate/key data found", site_data or global_data)
+        return self:ret(true, "certificate/key data found", data)
     end
     return self:ret(true, "custom certificate is not used")
 end
 
 function customcert:read_files(server_name)
 	local files = {
-		"/var/cache/bunkerweb/customcert/" .. (server_name or "") .. "/cert.pem",
-		"/var/cache/bunkerweb/customcert/" .. (server_name or "") .. "/key.pem"
+		"/var/cache/bunkerweb/customcert/" .. server_name .. "/cert.pem",
+		"/var/cache/bunkerweb/customcert/" .. server_name .. "/key.pem"
 	}
 	local data = {}
 	for i, file in ipairs(files) do
@@ -125,10 +107,13 @@ function customcert:load_data(data, server_name)
 	if not priv_key then
 		return false, "error while parsing pem priv key : " .. err
 	end
-	local cache_key = "plugin_customcert_" .. (server_name or "global")
-	local ok, err = self.datastore:set(cache_key, {cert_chain, priv_key}, nil, true)
-	if not ok then
-		return false, "error while setting data into datastore : " .. err
+	-- Cache data
+	for key in server_name:gmatch("%S+") do
+		local cache_key = "plugin_customcert_" .. key
+		local ok, err = self.datastore:set(cache_key, {cert_chain, priv_key}, nil, true)
+		if not ok then
+			return false, "error while setting data into datastore : " .. err
+		end
 	end
 	return true
 end

src/common/core/customcert/jobs/custom-cert.py

@@ -27,7 +27,7 @@ logger = setup_logger("CUSTOM-CERT", getenv("LOG_LEVEL", "INFO"))
 db = None
 
 
-def check_cert(cert_path: str, key_path: str, first_server: Optional[str] = None) -> bool:
+def check_cert(cert_path: str, key_path: str, first_server: str) -> bool:
     try:
         if not cert_path or not key_path:
             logger.warning("Both variables CUSTOM_SSL_CERT and CUSTOM_SSL_KEY have to be set to use custom certificates")
@@ -49,7 +49,7 @@ def check_cert(cert_path: str, key_path: str, first_server: Optional[str] = None
             "cache",
             "bunkerweb",
             "customcert",
-            first_server or "",
+            first_server,
             "cert.pem",
         )
         cert_cache_path.parent.mkdir(parents=True, exist_ok=True)
@@ -69,7 +69,7 @@ def check_cert(cert_path: str, key_path: str, first_server: Optional[str] = None
             "cache",
             "bunkerweb",
             "customcert",
-            first_server or "",
+            first_server,
             "key.pem",
         )
         key_cache_path.parent.mkdir(parents=True, exist_ok=True)
@@ -94,17 +94,18 @@ status = 0
 try:
     Path(sep, "var", "cache", "bunkerweb", "customcert").mkdir(parents=True, exist_ok=True)
 
-    if getenv("USE_CUSTOM_SSL", "no") == "yes" and getenv("SERVER_NAME", "") != "":
+    if getenv("MULTISITE", "no") == "no" and getenv("USE_CUSTOM_SSL", "no") == "yes" and getenv("SERVER_NAME", "") != "":
         db = Database(logger, sqlalchemy_string=getenv("DATABASE_URI", None), pool=False)
 
         cert_path = getenv("CUSTOM_SSL_CERT", "")
         key_path = getenv("CUSTOM_SSL_KEY", "")
+        first_server = getenv("SERVER_NAME").split(" ")[0]
 
         cert_data = b64decode(getenv("CUSTOM_SSL_CERT_DATA", ""))
         key_data = b64decode(getenv("CUSTOM_SSL_KEY_DATA", ""))
         for file, data in [("cert.pem", cert_data), ("key.pem", key_data)]:
             if data != b"":
-                file_path = Path(sep, "var", "tmp", "bunkerweb", "customcert", file)
+                file_path = Path(sep, "var", "tmp", "bunkerweb", "customcert", first_server, file)
                 file_path.parent.mkdir(parents=True, exist_ok=True)
                 file_path.write_bytes(data)
                 if file == "cert.pem":
@@ -114,14 +115,14 @@ try:
 
         if cert_path and key_path:
             logger.info(f"Checking certificate {cert_path} ...")
-            need_reload = check_cert(cert_path, key_path)
+            need_reload = check_cert(cert_path, key_path, first_server)
             if need_reload:
                 logger.info(f"Detected change for certificate {cert_path}")
                 status = 1
             else:
                 logger.info(f"No change for certificate {cert_path}")
 
-    if getenv("MULTISITE", "no") == "yes":
+    elif getenv("MULTISITE", "no") == "yes":
         servers = getenv("SERVER_NAME") or []
 
         if isinstance(servers, str):
@@ -141,7 +142,7 @@ try:
             key_data = b64decode(getenv(f"{first_server}_CUSTOM_SSL_KEY_DATA", ""))
             for file, data in [("cert.pem", cert_data), ("key.pem", key_data)]:
                 if data != b"":
-                    file_path = Path(sep, "var", "tmp", "bunkerweb", "customcert", server_name, file)
+                    file_path = Path(sep, "var", "tmp", "bunkerweb", "customcert", first_server, file)
                     file_path.parent.mkdir(parents=True, exist_ok=True)
                     file_path.write_bytes(data)
                     if file == "cert.pem":

src/common/core/letsencrypt/letsencrypt.lua

@@ -19,19 +19,19 @@ function letsencrypt:init()
 			return self:ret(false, "can't get MULTISITE variable : " .. err)
 		end
 		if multisite == "yes" then
-			local vars, err = utils.get_multiple_variables({"AUTO_LETS_ENCRYPT"})
+			local vars, err = utils.get_multiple_variables({"AUTO_LETS_ENCRYPT", "SERVER_NAME"})
 			if not vars then
 				return self:ret(false, "can't get AUTO_LETS_ENCRYPT variables : " .. err)
 			end
 			for server_name, multisite_vars in pairs(vars) do
-				if multisite_vars["AUTO_LETS_ENCRYPT"] == "yes" then
+				if multisite_vars["AUTO_LETS_ENCRYPT"] == "yes" and server_name ~= "global" then
 					local check, data = self:read_files(server_name)
 					if not check then
 						self.logger:log(ngx.ERR, "error while reading files : " .. data)
 						ret_ok = false
 						ret_err = "error reading files"
 					else
-						local check, err = self:load_data(data, server_name)
+						local check, err = self:load_data(data, multisite_vars["SERVER_NAME"])
 						if not check then
 							self.logger:log(ngx.ERR, "error while loading data : " .. err)
 							ret_ok = false
@@ -51,7 +51,7 @@ function letsencrypt:init()
 				ret_ok = false
 				ret_err = "error reading files"
 			else
-				local check, err = self:load_data(data)
+				local check, err = self:load_data(data, server_name)
 				if not check then
 					self.logger:log(ngx.ERR, "error while loading data : " .. err)
 					ret_ok = false
@@ -104,14 +104,17 @@ function letsencrypt:load_data(data, server_name)
 		return false, "error while parsing pem cert : " .. err
 	end
 	-- Load key
-	local priv_key, err = ssl.pars_pem_priv_key(data[2])
+	local priv_key, err = ssl.parse_pem_priv_key(data[2])
 	if not priv_key then
 		return false, "error while parsing pem priv key : " .. err
 	end
-	local cache_key = "plugin_letsencrypt_" .. (server_name or "global")
-	local ok, err = self.datastore:set(cache_key, {cert_chain, priv_key}, nil, true)
-	if not ok then
-		return false, "error while setting data into datastore : " .. err
+	-- Cache data
+	for key in server_name:gmatch("%S+") do
+		local cache_key = "plugin_letsencrypt_" .. key
+		local ok, err = self.datastore:set(cache_key, {cert_chain, priv_key}, nil, true)
+		if not ok then
+			return false, "error while setting data into datastore : " .. err
+		end
 	end
 	return true
 end

src/common/core/order.json

@@ -32,7 +32,7 @@
     "antibot"
   ],
   "headers": ["headers", "cors", "reverseproxy", "clientcache", "antibot"],
-  "log": ["badbheavior", "bunkernet"],
+  "log": ["badbehavior", "bunkernet"],
   "preread": [
     "whitelist",
     "blacklist",
@@ -42,5 +42,5 @@
     "reversescan"
   ],
   "log_stream": ["badbehavior", "bunkernet"],
-  "log_default": ["badbheavior", "bunkernet"]
+  "log_default": ["badbehavior", "bunkernet"]
 }

src/common/core/selfsigned/selfsigned.lua

@@ -18,19 +18,19 @@ function selfsigned:init()
 			return self:ret(false, "can't get MULTISITE variable : " .. err)
 		end
 		if multisite == "yes" then
-			local vars, err = utils.get_multiple_variables({"GENERATE_SELF_SIGNED_SSL"})
+			local vars, err = utils.get_multiple_variables({"GENERATE_SELF_SIGNED_SSL", "SERVER_NAME"})
 			if not vars then
 				return self:ret(false, "can't get GENERATE_SELF_SIGNED_SSL variables : " .. err)
 			end
 			for server_name, multisite_vars in pairs(vars) do
-				if multisite_vars["GENERATE_SELF_SIGNED_SSL"] == "yes" then
+				if multisite_vars["GENERATE_SELF_SIGNED_SSL"] == "yes" and server_name ~= "global" then
 					local check, data = self:read_files(server_name)
 					if not check then
 						self.logger:log(ngx.ERR, "error while reading files : " .. data)
 						ret_ok = false
 						ret_err = "error reading files"
 					else
-						local check, err = self:load_data(data, server_name)
+						local check, err = self:load_data(data, multisite_vars["SERVER_NAME"])
 						if not check then
 							self.logger:log(ngx.ERR, "error while loading data : " .. err)
 							ret_ok = false
@@ -50,7 +50,7 @@ function selfsigned:init()
 				ret_ok = false
 				ret_err = "error reading files"
 			else
-				local check, err = self:load_data(data)
+				local check, err = self:load_data(data, server_name)
 				if not check then
 					self.logger:log(ngx.ERR, "error while loading data : " .. err)
 					ret_ok = false
@@ -107,10 +107,13 @@ function selfsigned:load_data(data, server_name)
 	if not priv_key then
 		return false, "error while parsing pem priv key : " .. err
 	end
-	local cache_key = "plugin_selfsigned_" .. (server_name or "global")
-	local ok, err = self.datastore:set(cache_key, {cert_chain, priv_key}, nil, true)
-	if not ok then
-		return false, "error while setting data into datastore : " .. err
+	-- Cache data
+	for key in server_name:gmatch("%S+") do
+		local cache_key = "plugin_selfsigned_" .. key
+		local ok, err = self.datastore:set(cache_key, {cert_chain, priv_key}, nil, true)
+		if not ok then
+			return false, "error while setting data into datastore : " .. err
+		end
 	end
 	return true
 end