Merge branch 'dev' into staging

This commit is contained in:
florian 2023-08-04 13:28:37 +02:00
commit 31ca183b1e
No known key found for this signature in database
GPG key ID: 3D80806F12602A7C
73 changed files with 336 additions and 181 deletions

View file

@ -24,6 +24,7 @@ jobs:
uses: ./.github/workflows/container-build.yml
with:
RELEASE: dev
CACHE: true
ARCH: linux/amd64
IMAGE: ${{ matrix.image }}
DOCKERFILE: ${{ matrix.dockerfile }}
@ -93,3 +94,28 @@ jobs:
secrets:
PRIVATE_REGISTRY: ${{ secrets.PRIVATE_REGISTRY }}
PRIVATE_REGISTRY_TOKEN: ${{ secrets.PRIVATE_REGISTRY_TOKEN }}
# Push with dev tag
push-dev:
needs: [tests-core]
runs-on: ubuntu-latest
steps:
- name: Login to Docker Hub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_TOKEN }}
- name: Login to private repository
uses: docker/login-action@v2
with:
registry: ${{ secrets.PRIVATE_REGISTRY }}
username: registry
password: ${{ secrets.PRIVATE_REGISTRY_TOKEN }}
- name: Push BW image
run: docker pull ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-tests:dev && docker tag ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-tests:dev bunkerity/bunkerweb:dev && docker push bunkerity/bunkerweb:dev
- name: Push scheduler image
run: docker pull ${{ secrets.PRIVATE_REGISTRY }}/infra/scheduler-tests:dev && docker tag ${{ secrets.PRIVATE_REGISTRY }}/infra/scheduler-tests:dev bunkerity/bunkerweb-scheduler:dev && docker push bunkerity/bunkerweb-scheduler:dev
- name: Push UI image
run: docker pull ${{ secrets.PRIVATE_REGISTRY }}/infra/ui-tests:dev && docker tag ${{ secrets.PRIVATE_REGISTRY }}/infra/ui-tests:dev bunkerity/bunkerweb-ui:dev && docker push bunkerity/bunkerweb-ui:dev
- name: Push autoconf image
run: docker pull ${{ secrets.PRIVATE_REGISTRY }}/infra/autoconf-tests:dev && docker tag ${{ secrets.PRIVATE_REGISTRY }}/infra/autoconf-tests:dev bunkerity/bunkerweb-autoconf:dev && docker push bunkerity/bunkerweb-autoconf:dev

View file

@ -10,10 +10,15 @@
- [BUGFIX] Variable being ignored instead of saved in the database when value is empty
- [BUGFIX] ALLOWED_METHODS regex working with LOCK/UNLOCK methods
- [BUGFIX] Custom certificate bug after the refactoring
- [BUGFIX] Fix wrong variables in header phase (fix CORS feature too)
- [BUGFIX] Fix UI not working in Ubuntu (python zope module)
- [BUGFIX] Wrong variables in header phase (fix CORS feature too)
- [BUGFIX] UI not working in Ubuntu (python zope module)
- [BUGFIX] Patch ModSecurity to run it after LUA code (should fix whitelist problems)
- [BUGFIX] Custom configurations from env were not being deleted properly
- [BUGFIX] Missing concepts image not displayed in the documentation
- [BUGFIX] Scheduler not picking up new instances IPs in autoconf modes
- [BUGFIX] Autoconf deadlock in k8s
- [BUGFIX] Missing HTTP and HTTPS ports for temp nginx
- [BUGFIX] Infinite loop when sessions is not valid
- [PERFORMANCE] Reduce CPU and RAM usage of scheduler
- [PERFORMANCE] Cache ngx.ctx instead of loading it each time
- [PERFORMANCE] Use per-worker LRU cache for common RO LUA values
@ -22,10 +27,15 @@
- [FEATURE] Add KEEP_UPSTREAM_HEADERS to preserve headers when using reverse proxy
- [FEATURE] Add the possibility to download the different lists and plugins from a local file (like the blacklist)
- [FEATURE] External plugins can now be downloaded from a tar.gz and tar.xz file as well as zip
- [FEATURE] Add X-Forwarded-Prefix header when using reverse proxy
- [FEATURE] Add REDIRECT_TO_STATUS_CODE to choose status code 301 or 302 when redirecting
- [DOCUMENTATION] Add timezone information
- [DOCUMENTATION] Add timezone informat
- [MISC] Add LOG_LEVEL=warning for docker socket proxy in docs, examples and boilerplates
- [MISC] Temp remove VMWare provider for Vagrant integration
- [MISC] Remove X-Script-Name header and ABSOLUTE_URI variable when using UI
- [MISC] Move logs to /var/log/bunkerweb folder
- [MISC] Reduce "Got an error reading communication packets" warnings in mariadb/mysql
## v1.5.0 - 2023/05/23

View file

@ -1,7 +1,7 @@
# Concepts
<figure markdown>
![Overwiew](assets/img/concepts.svg){ align=center }
![Overwiew](assets/img/concepts.svg){ align=center, width="600" }
</figure>
## Integrations

View file

@ -1,5 +1,5 @@
mkdocs==1.4.3
mkdocs-material==9.1.18
mkdocs-material==9.1.19
pytablewriter==1.0.0
mike==1.1.2
jinja2<3.1.0

View file

@ -159,11 +159,12 @@ STREAM support :white_check_mark:
If you want to use your own certificates, here is the list of related settings :
| Setting | Default | Description |
| :-----------------: | :-----: | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `USE_CUSTOM_SSL` | `no` | When set to `yes`, HTTPS will be enabled with custom certificates. |
| `CUSTOM_SSL_CERT` | | Full path to the certificate. If you have one or more intermediate certificate(s) in your chain of trust, you will need to provide the bundle (more info [here](https://nginx.org/en/docs/http/configuring_https_servers.html#chains)). |
| `CUSTOM_SSL_KEY` | | Full path to the private key. |
| Setting |Default| Context |Multiple| Description |
|-----------------|-------|---------|--------|--------------------------------------------------------------------------------|
|`USE_CUSTOM_SSL` |`no` |multisite|no |Use custom HTTPS certificate. |
|`CUSTOM_SSL_CERT`| |multisite|no |Full path of the certificate or bundle file (must be readable by the scheduler).|
|`CUSTOM_SSL_KEY` | |multisite|no |Full path of the key file (must be readable by the scheduler). |
When `USE_CUSTOM_SSL` is set to `yes`, BunkerWeb will check every day if the custom certificate specified in `CUSTOM_SSL_CERT` is modified and will reload NGINX if that's the case.

View file

@ -205,11 +205,11 @@ STREAM support :white_check_mark:
Choose custom certificate for HTTPS.
| Setting |Default| Context |Multiple| Description |
|-----------------|-------|---------|--------|--------------------------------------------|
|`USE_CUSTOM_SSL` |`no` |multisite|no |Use custom HTTPS certificate. |
|`CUSTOM_SSL_CERT`| |multisite|no |Full path of the certificate or bundle file.|
|`CUSTOM_SSL_KEY` | |multisite|no |Full path of the key file. |
| Setting |Default| Context |Multiple| Description |
|-----------------|-------|---------|--------|--------------------------------------------------------------------------------|
|`USE_CUSTOM_SSL` |`no` |multisite|no |Use custom HTTPS certificate. |
|`CUSTOM_SSL_CERT`| |multisite|no |Full path of the certificate or bundle file (must be readable by the scheduler).|
|`CUSTOM_SSL_KEY` | |multisite|no |Full path of the key file (must be readable by the scheduler). |
### DB
@ -344,25 +344,25 @@ STREAM support :warning:
Miscellaneous settings.
| Setting | Default | Context |Multiple| Description |
|-----------------------------|-----------------------|---------|--------|----------------------------------------------------------------------------------------------------------------------|
|`DISABLE_DEFAULT_SERVER` |`no` |global |no |Close connection if the request vhost is unknown. |
|`REDIRECT_HTTP_TO_HTTPS` |`no` |multisite|no |Redirect all HTTP request to HTTPS. |
|`AUTO_REDIRECT_HTTP_TO_HTTPS`|`yes` |multisite|no |Try to detect if HTTPS is used and activate HTTP to HTTPS redirection if that's the case. |
|`ALLOWED_METHODS` |`GET|POST|HEAD` |multisite|no |Allowed HTTP and WebDAV methods, separated with pipes to be sent by clients. |
|`MAX_CLIENT_SIZE` |`10m` |multisite|no |Maximum body size (0 for infinite). |
|`SERVE_FILES` |`yes` |multisite|no |Serve files from the local folder. |
|`ROOT_FOLDER` | |multisite|no |Root folder containing files to serve (/var/www/html/{server_name} if unset). |
|`SSL_PROTOCOLS` |`TLSv1.2 TLSv1.3` |multisite|no |The supported version of TLS. We recommend the default value TLSv1.2 TLSv1.3 for compatibility reasons. |
|`HTTP2` |`yes` |multisite|no |Support HTTP2 protocol when HTTPS is enabled. |
|`LISTEN_HTTP` |`yes` |multisite|no |Respond to (insecure) HTTP requests. |
|`USE_OPEN_FILE_CACHE` |`no` |multisite|no |Enable open file cache feature |
|`OPEN_FILE_CACHE` |`max=1000 inactive=20s`|multisite|no |Open file cache directive |
|`OPEN_FILE_CACHE_ERRORS` |`yes` |multisite|no |Enable open file cache for errors |
|`OPEN_FILE_CACHE_MIN_USES` |`2` |multisite|no |Enable open file cache minimum uses |
|`OPEN_FILE_CACHE_VALID` |`30s` |multisite|no |Open file cache valid time |
|`EXTERNAL_PLUGIN_URLS` | |global |no |List of external plugins URLs (direct download to .zip file) to download and install (URLs are separated with space). |
|`DENY_HTTP_STATUS` |`403` |global |no |HTTP status code to send when the request is denied (403 or 444). When using 444, BunkerWeb will close the connection.|
| Setting | Default | Context |Multiple| Description |
|-----------------------------|-----------------------|---------|--------|-----------------------------------------------------------------------------------------------------------------------------|
|`DISABLE_DEFAULT_SERVER` |`no` |global |no |Close connection if the request vhost is unknown. |
|`REDIRECT_HTTP_TO_HTTPS` |`no` |multisite|no |Redirect all HTTP request to HTTPS. |
|`AUTO_REDIRECT_HTTP_TO_HTTPS`|`yes` |multisite|no |Try to detect if HTTPS is used and activate HTTP to HTTPS redirection if that's the case. |
|`ALLOWED_METHODS` |`GET|POST|HEAD` |multisite|no |Allowed HTTP and WebDAV methods, separated with pipes to be sent by clients. |
|`MAX_CLIENT_SIZE` |`10m` |multisite|no |Maximum body size (0 for infinite). |
|`SERVE_FILES` |`yes` |multisite|no |Serve files from the local folder. |
|`ROOT_FOLDER` | |multisite|no |Root folder containing files to serve (/var/www/html/{server_name} if unset). |
|`SSL_PROTOCOLS` |`TLSv1.2 TLSv1.3` |multisite|no |The supported version of TLS. We recommend the default value TLSv1.2 TLSv1.3 for compatibility reasons. |
|`HTTP2` |`yes` |multisite|no |Support HTTP2 protocol when HTTPS is enabled. |
|`LISTEN_HTTP` |`yes` |multisite|no |Respond to (insecure) HTTP requests. |
|`USE_OPEN_FILE_CACHE` |`no` |multisite|no |Enable open file cache feature |
|`OPEN_FILE_CACHE` |`max=1000 inactive=20s`|multisite|no |Open file cache directive |
|`OPEN_FILE_CACHE_ERRORS` |`yes` |multisite|no |Enable open file cache for errors |
|`OPEN_FILE_CACHE_MIN_USES` |`2` |multisite|no |Enable open file cache minimum uses |
|`OPEN_FILE_CACHE_VALID` |`30s` |multisite|no |Open file cache valid time |
|`EXTERNAL_PLUGIN_URLS` | |global |no |List of external plugins URLs (direct download to .zip or .tar file) to download and install (URLs are separated with space).|
|`DENY_HTTP_STATUS` |`403` |global |no |HTTP status code to send when the request is denied (403 or 444). When using 444, BunkerWeb will close the connection. |
### ModSecurity
@ -416,6 +416,7 @@ Manage HTTP redirects.
|-------------------------|-------|---------|--------|-------------------------------------------------|
|`REDIRECT_TO` | |multisite|no |Redirect a whole site to another one. |
|`REDIRECT_TO_REQUEST_URI`|`no` |multisite|no |Append the requested URI to the redirect address.|
|`REDIRECT_TO_STATUS_CODE`|`301` |multisite|no |Status code to send to client when redirecting. |
### Redis

View file

@ -280,4 +280,8 @@ If you have bots that need to access your website, the recommended way to avoid
- Healthcheck / status bot
- Callback like IPN or webhook
- Social media crawler
- Social media crawler
## Timezone
When using container-based integrations, the timezone of the container may not match the one of the host machine. To resolve that, you can set the `TZ` environment variable to the timezone of your choice on your containers (e.g. `TZ=Europe/Paris`). You will find the list of timezone identifers [here](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones#List).

View file

@ -1,7 +1,6 @@
#!/usr/bin/python3
from os import getenv
from threading import Lock
from time import sleep
from typing import Optional
@ -11,9 +10,8 @@ from logger import setup_logger # type: ignore
class Config(ConfigCaller):
def __init__(self, lock: Optional[Lock] = None):
def __init__(self):
super().__init__()
self.__lock = lock
self.__logger = setup_logger("Config", getenv("LOG_LEVEL", "INFO"))
self.__instances = []
self.__services = []
@ -80,9 +78,6 @@ class Config(ConfigCaller):
)
sleep(5)
if self.__lock:
self.__lock.acquire()
# update instances in database
err = self._db.update_instances(self.__instances)
if err:
@ -104,7 +99,4 @@ class Config(ConfigCaller):
f"Can't save autoconf custom configs in database: {err}, custom configs may not work as expected",
)
if self.__lock:
self.__lock.release()
return success

View file

@ -2,7 +2,6 @@
from abc import ABC, abstractmethod
from os import getenv
from threading import Lock
from time import sleep
from typing import Literal, Optional, Union
@ -15,9 +14,8 @@ class Controller(Config):
def __init__(
self,
ctrl_type: Union[Literal["docker"], Literal["swarm"], Literal["kubernetes"]],
lock: Optional[Lock] = None,
):
super().__init__(lock)
super().__init__()
self._type = ctrl_type
self._instances = []
self._services = []

View file

@ -14,7 +14,7 @@ RUN apk add --no-cache --virtual .build-deps g++ gcc musl-dev jpeg-dev zlib-dev
# Install python requirements
RUN export MAKEFLAGS="-j$(nproc)" && \
pip install --no-cache-dir --upgrade pip && \
pip install --no-cache-dir --upgrade wheel && \
pip install --no-cache-dir --upgrade pip-tools wheel setuptools && \
mkdir -p /usr/share/bunkerweb/deps/python && \
pip install --no-cache-dir --require-hashes --target /usr/share/bunkerweb/deps/python -r /usr/share/bunkerweb/deps/requirements.txt

View file

@ -13,7 +13,7 @@ from Controller import Controller
class IngressController(Controller):
def __init__(self):
self.__internal_lock = Lock()
super().__init__("kubernetes", self.__internal_lock)
super().__init__("kubernetes")
config.load_incluster_config()
self.__corev1 = client.CoreV1Api()
self.__networkingv1 = client.NetworkingV1Api()
@ -31,7 +31,7 @@ class IngressController(Controller):
def _to_instances(self, controller_instance) -> List[dict]:
instance = {}
instance["name"] = controller_instance.metadata.name
instance["hostname"] = controller_instance.status.pod_ip
instance["hostname"] = controller_instance.status.pod_ip or controller_instance.metadata.name
health = False
if controller_instance.status.conditions:
for condition in controller_instance.status.conditions:

View file

@ -20,10 +20,10 @@ COPY src/common/gen/requirements.txt /usr/share/bunkerweb/deps/requirements.txt
# Install python requirements
RUN apk add --no-cache --virtual .build-deps py3-pip && \
pip install --no-cache-dir --upgrade pip && \
pip install wheel && \
mkdir -p /usr/share/bunkerweb/deps/python && \
export MAKEFLAGS="-j$(nproc)" && \
pip install --no-cache-dir --upgrade pip && \
pip install --no-cache-dir --upgrade pip-tools wheel setuptools && \
pip install --no-cache-dir --require-hashes --target /usr/share/bunkerweb/deps/python -r /usr/share/bunkerweb/deps/requirements.txt && \
apk del .build-deps && \
rm -rf /var/cache/apk/*
@ -73,7 +73,7 @@ RUN apk add --no-cache pcre bash python3 && \
ln -s /proc/1/fd/1 /var/log/bunkerweb/access.log
# Fix CVEs
RUN apk add "libx11>=1.8.4-r1" "tiff>=4.4.0-r4" "libcrypto3>=3.0.9-r2" "libssl3>=3.0.9-r2"
RUN apk add "libx11>=1.8.4-r1" "tiff>=4.4.0-r4" "libcrypto3>=3.0.9-r2" "libssl3>=3.0.9-r2" "nghttp2-libs>=1.51.0-r1"
VOLUME /data /etc/nginx

View file

@ -39,7 +39,7 @@ if [ -f "/etc/nginx/variables.env" ] ; then
log "ENTRYPOINT" "⚠️ " "Looks like BunkerWeb has already been loaded, will not generate temp config"
else
# generate "temp" config
echo -e "IS_LOADING=yes\nSERVER_NAME=\nAPI_HTTP_PORT=${API_HTTP_PORT:-5000}\nAPI_SERVER_NAME=${API_SERVER_NAME:-bwapi}\nAPI_WHITELIST_IP=${API_WHITELIST_IP:-127.0.0.0/8}\nUSE_REAL_IP=${USE_REAL_IP:-no}\nUSE_REAL_IP=${USE_REAL_IP:-no}\nUSE_PROXY_PROTOCOL=${USE_PROXY_PROTOCOL:-no}\nREAL_IP_FROM=${REAL_IP_FROM:-192.168.0.0/16 172.16.0.0/12 10.0.0.0/8}\nREAL_IP_HEADER=${REAL_IP_HEADER:-X-Forwarded-For}" > /tmp/variables.env
echo -e "IS_LOADING=yes\nSERVER_NAME=\nAPI_HTTP_PORT=${API_HTTP_PORT:-5000}\nAPI_SERVER_NAME=${API_SERVER_NAME:-bwapi}\nAPI_WHITELIST_IP=${API_WHITELIST_IP:-127.0.0.0/8}\nUSE_REAL_IP=${USE_REAL_IP:-no}\nUSE_PROXY_PROTOCOL=${USE_PROXY_PROTOCOL:-no}\nREAL_IP_FROM=${REAL_IP_FROM:-192.168.0.0/16 172.16.0.0/12 10.0.0.0/8}\nREAL_IP_HEADER=${REAL_IP_HEADER:-X-Forwarded-For}\nHTTP_PORT=${HTTP_PORT}\nHTTPS_PORT=${HTTPS_PORT}" > /tmp/variables.env
python3 /usr/share/bunkerweb/gen/main.py --variables /tmp/variables.env
fi

View file

@ -171,6 +171,7 @@ helpers.fill_ctx = function()
data.http_content_length = ngx.var.http_content_length
data.http_origin = ngx.var.http_origin
data.http_version = ngx.req.http_version()
data.scheme = ngx.var.scheme
end
-- IP data : global
local ip_is_global, err = utils.ip_is_global(data.remote_addr)

View file

@ -486,9 +486,9 @@ utils.check_session = function(ctx)
local key = check[1]
local value = check[2]
if _session:get(key) ~= value then
_session:clear_request_cookie()
local ok, err = _session:destroy()
if not ok then
_session:close()
return false, "session:destroy() error : " .. err
end
logger:log(ngx.WARN, "session check " .. key .. " failed, destroying session")
@ -506,7 +506,6 @@ utils.check_session = function(ctx)
end
end
ctx.bw.sessions_is_checked = true
_session:close()
return true, exists
end

Binary file not shown.

Binary file not shown.

View file

@ -48,7 +48,7 @@ function antibot:header()
end
local header = "Content-Security-Policy"
if utils.get_variable("CONTENT_SECURITY_POLICY_REPORT_ONLY", true) == "yes" then
if self.variables["CONTENT_SECURITY_POLICY_REPORT_ONLY"] == "yes" then
header = header .. "-Report-Only"
end

View file

@ -80,6 +80,7 @@ try:
db = Database(
logger,
sqlalchemy_string=getenv("DATABASE_URI", None),
pool=False
)
# Create directories if they don't exist

View file

@ -60,6 +60,7 @@ try:
db = Database(
logger,
sqlalchemy_string=getenv("DATABASE_URI", None),
pool=False
)
bunkernet_id = get_file_in_db("instance.id", db)
if bunkernet_id:

View file

@ -61,6 +61,7 @@ try:
db = Database(
logger,
sqlalchemy_string=getenv("DATABASE_URI", None),
pool=False
)
bunkernet_id = get_file_in_db("instance.id", db)
if bunkernet_id:

View file

@ -1,4 +1,5 @@
{% if USE_CLIENT_CACHE == "yes" +%}
add_header Cache-Control $cache_control;
{% if CLIENT_CACHE_ETAG == "yes" and SERVE_FILES == "yes" and USE_REVERSE_PROXY == "no" +%}
etag on;
{% else +%}

View file

@ -111,6 +111,7 @@ try:
db = Database(
logger,
sqlalchemy_string=getenv("DATABASE_URI", None),
pool=False
)
cert_path = getenv("CUSTOM_SSL_CERT", "")
@ -142,6 +143,7 @@ try:
db = Database(
logger,
sqlalchemy_string=getenv("DATABASE_URI", None),
pool=False
)
cert_path = getenv(f"{first_server}_CUSTOM_SSL_CERT", "")

View file

@ -17,7 +17,7 @@
"CUSTOM_SSL_CERT": {
"context": "multisite",
"default": "",
"help": "Full path of the certificate or bundle file.",
"help": "Full path of the certificate or bundle file (must be readable by the scheduler).",
"id": "custom-https-cert",
"label": "Certificate path",
"regex": "^(/[\\w. -]+)*/?$",
@ -26,7 +26,7 @@
"CUSTOM_SSL_KEY": {
"context": "multisite",
"default": "",
"help": "Full path of the key file.",
"help": "Full path of the key file (must be readable by the scheduler).",
"id": "custom-https-key",
"label": "Key path",
"regex": "^(/[\\w. -]+)*/?$",

View file

@ -80,6 +80,7 @@ try:
db = Database(
logger,
sqlalchemy_string=getenv("DATABASE_URI", None),
pool=False
)
# Create directories if they don't exist

View file

@ -4,9 +4,9 @@ local utils = require "bunkerweb.utils"
local headers = class("headers", plugin)
function headers:initialize()
function headers:initialize(ctx)
-- Call parent initialize
plugin.initialize(self, "headers")
plugin.initialize(self, "headers", ctx)
self.all_headers = {
["STRICT_TRANSPORT_SECURITY"] = "Strict-Transport-Security",
["CONTENT_SECURITY_POLICY"] = "Content-Security-Policy",
@ -17,15 +17,63 @@ function headers:initialize()
["X_CONTENT_TYPE_OPTIONS"] = "X-Content-Type-Options",
["X_XSS_PROTECTION"] = "X-XSS-Protection"
}
-- Load data from datastore if needed
if ngx.get_phase() ~= "init" then
-- Get custom headers from datastore
local custom_headers, err = self.datastore:get("plugin_headers_custom_headers", true)
if not custom_headers then
self.logger:log(ngx.ERR, err)
return
end
self.custom_headers = {}
-- Extract global headers
if custom_headers.global then
for k, v in pairs(custom_headers.global) do
self.custom_headers[k] = v
end
end
-- Extract and overwrite if needed server headers
if custom_headers[self.ctx.bw.server_name] then
for k, v in pairs(custom_headers[self.ctx.bw.server_name]) do
self.custom_headers[k] = v
end
end
end
end
function headers:init()
-- Get variables
local variables, err = utils.get_multiple_variables({ "CUSTOM_HEADER" })
if variables == nil then
return self:ret(false, err)
end
-- Store custom headers name and value
local data = {}
local i = 0
for srv, vars in pairs(variables) do
for var, value in pairs(vars) do
if data[srv] == nil then
data[srv] = {}
end
local m = utils.regex_match(value, "([\\w-]+): ([^,]+)")
if m then
data[srv][m[1]] = m[2]
end
i = i + 1
end
end
local ok, err = self.datastore:set("plugin_headers_custom_headers", data, nil, true)
if not ok then
return self:ret(false, err)
end
return self:ret(true, "successfully loaded " .. tostring(i) .. " custom headers")
end
function headers:header()
-- Override upstream headers if needed
local ssl = utils.get_variable("AUTO_LETS_ENCRYPT", true) == "yes" or
utils.get_variable("USE_CUSTOM_SSL", true) == "yes" or
utils.get_variable("GENERATE_SELF_SIGNED_SSL", true) == "yes"
local ssl = self.ctx.bw.scheme == "https"
for variable, header in pairs(self.all_headers) do
if ngx.header[header] == nil or self.variables[variable] and self.variables["KEEP_UPSTREAM_HEADERS"] ~= "*" and utils.regex_match(self.variables["KEEP_UPSTREAM_HEADERS"], "(^| )" .. header .. "($| )") == nil then
if ngx.header[header] == nil or (self.variables[variable] ~= "" and self.variables["KEEP_UPSTREAM_HEADERS"] ~= "*" and utils.regex_match(self.variables["KEEP_UPSTREAM_HEADERS"], "(^| )" .. header .. "($| )") == nil) then
if (header ~= "Strict-Transport-Security" or ssl) then
if header == "Content-Security-Policy" and self.variables["CONTENT_SECURITY_POLICY_REPORT_ONLY"] == "yes" then
ngx.header["Content-Security-Policy-Report-Only"] = self.variables[variable]
@ -35,43 +83,17 @@ function headers:header()
end
end
end
-- Get variables
local variables, err = utils.get_multiple_variables({ "CUSTOM_HEADER" })
if variables == nil then
return self:ret(false, err)
end
-- Add custom headers
for srv, vars in pairs(variables) do
if srv == self.ctx.bw.server_name or srv == "global" then
for var, value in pairs(vars) do
if utils.regex_match(var, "CUSTOM_HEADER") and value then
local m = utils.regex_match(value, "([\\w-]+): ([^,]+)")
if m then
ngx.header[m[1]] = m[2]
end
end
end
end
for header, value in pairs(self.custom_headers) do
ngx.header[header] = value
end
-- Remove headers
if self.variables["REMOVE_HEADERS"] ~= "" then
local iterator, err = ngx.re.gmatch(self.variables["REMOVE_HEADERS"], "([\\w-]+)")
if not iterator then
return self:ret(false, "Error while matching remove headers: " .. err)
end
while true do
local m, err = iterator()
if err then
return self:ret(false, "Error while matching remove headers: " .. err)
end
if not m then
-- No more remove headers
break
end
ngx.header[m[1]] = nil
for header in self.variables["REMOVE_HEADERS"]:gmatch("%S+") do
ngx.header[header] = nil
end
end
return self:ret(true, "Edited headers for request")
return self:ret(true, "edited headers for request")
end
return headers

View file

@ -177,6 +177,7 @@ try:
db = Database(
logger,
sqlalchemy_string=getenv("DATABASE_URI"),
pool=False
)
lock = Lock()

View file

@ -71,6 +71,7 @@ try:
db = Database(
logger,
sqlalchemy_string=getenv("DATABASE_URI", None),
pool=False
)
if dl_mmdb:

View file

@ -71,6 +71,7 @@ try:
db = Database(
logger,
sqlalchemy_string=getenv("DATABASE_URI", None),
pool=False
)
if dl_mmdb:

View file

@ -52,6 +52,7 @@ try:
db = Database(
logger,
sqlalchemy_string=getenv("DATABASE_URI", None),
pool=False
)
lock = Lock()

View file

@ -51,6 +51,7 @@ try:
db = Database(
logger,
sqlalchemy_string=getenv("DATABASE_URI", None),
pool=False
)
lock = Lock()
with lock:

View file

@ -67,6 +67,7 @@ try:
db = Database(
logger,
sqlalchemy_string=getenv("DATABASE_URI", None),
pool=False
)
lock = Lock()

View file

@ -99,6 +99,7 @@ try:
db = Database(
logger,
sqlalchemy_string=getenv("DATABASE_URI", None),
pool=False
)
tgz = get_file_in_db("folder.tgz", db, job_name="certbot-renew")

View file

@ -91,6 +91,7 @@ try:
db = Database(
logger,
sqlalchemy_string=getenv("DATABASE_URI", None),
pool=False
)
tgz = get_file_in_db("folder.tgz", db)

View file

@ -4,9 +4,9 @@ local cjson = require "cjson"
local letsencrypt = class("letsencrypt", plugin)
function letsencrypt:initialize()
function letsencrypt:initialize(ctx)
-- Call parent initialize
plugin.initialize(self, "letsencrypt")
plugin.initialize(self, "letsencrypt", ctx)
end
function letsencrypt:access()

View file

@ -5,9 +5,9 @@ local cjson = require "cjson"
local limit = class("limit", plugin)
function limit:initialize()
function limit:initialize(ctx)
-- Call parent initialize
plugin.initialize(self, "limit")
plugin.initialize(self, "limit", ctx)
-- Load rules if needed
if ngx.get_phase() ~= "init" and self:is_needed() then
-- Get all rules from datastore

View file

@ -102,6 +102,7 @@ try:
db = Database(
logger,
sqlalchemy_string=getenv("DATABASE_URI", None),
pool=False
)
cached, err = set_file_in_db(

View file

@ -4,9 +4,9 @@ local utils = require "bunkerweb.utils"
local misc = class("misc", plugin)
function misc:initialize()
function misc:initialize(ctx)
-- Call parent initialize
plugin.initialize(self, "misc")
plugin.initialize(self, "misc", ctx)
end
function misc:access()

View file

@ -76,6 +76,7 @@ try:
db = Database(
logger,
sqlalchemy_string=getenv("DATABASE_URI", None),
pool=False
)
# Don't go further if the cache is fresh

View file

@ -1,7 +1,7 @@
{% if REDIRECT_TO != "" +%}
{% if REDIRECT_TO_REQUEST_URI == "yes" +%}
return 301 {{ REDIRECT_TO }}$request_uri;
return {{ REDIRECT_TO_STATUS_CODE }} {{ REDIRECT_TO }}$request_uri;
{% else +%}
return 301 {{ REDIRECT_TO }};
return {{ REDIRECT_TO_STATUS_CODE }} {{ REDIRECT_TO }};
{% endif %}
{% endif %}

View file

@ -22,6 +22,16 @@
"label": "Append request URI",
"regex": "^(yes|no)$",
"type": "check"
},
"REDIRECT_TO_STATUS_CODE": {
"context": "multisite",
"default": "301",
"help": "Status code to send to client when redirecting.",
"id": "redirect-to-status-code",
"label": "Append request URI",
"regex": "^(301|302)$",
"type": "select",
"select": ["301", "302"]
}
}
}

View file

@ -5,7 +5,7 @@ local redis = class("redis", plugin)
function redis:initialize()
-- Call parent initialize
plugin.initialize(self, "redis")
plugin.initialize(self, "redis", ctx)
end
function redis:init_worker()

View file

@ -48,12 +48,9 @@ location {{ url }} {% raw %}{{% endraw +%}
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-Protocol $scheme;
proxy_set_header X-Forwarded-Host $http_host;
{% if USE_UI == "yes" +%}
{% if url.endswith("/") +%}
{% set url = url[:-1] +%}
{% endif +%}
proxy_set_header X-Forwarded-Prefix {{ url }};
{% endif +%}
{% if url.startswith("/") +%}
proxy_set_header X-Forwarded-Prefix "{{ url }}";
{% endif %}
{% if buffering == "yes" +%}
proxy_buffering on;
{% else +%}

View file

@ -6,9 +6,9 @@ local cjson = require "cjson"
local reversescan = class("reversescan", plugin)
function reversescan:initialize()
function reversescan:initialize(ctx)
-- Call parent initialize
plugin.initialize(self, "reversescan")
plugin.initialize(self, "reversescan", ctx)
end
function reversescan:access()

View file

@ -134,6 +134,7 @@ try:
db = Database(
logger,
sqlalchemy_string=getenv("DATABASE_URI", None),
pool=False
)
ret, ret_status = generate_cert(
@ -155,6 +156,7 @@ try:
db = Database(
logger,
sqlalchemy_string=getenv("DATABASE_URI", None),
pool=False
)
first_server = getenv("SERVER_NAME", "").split(" ")[0]

View file

@ -5,9 +5,9 @@ local session = require "resty.session"
local sessions = class("sessions", plugin)
function sessions:initialize()
function sessions:initialize(ctx)
-- Call parent initialize
plugin.initialize(self, "sessions")
plugin.initialize(self, "sessions", ctx)
-- Check if random cookie name and secrets are already generated
local is_random = {
"SESSIONS_SECRET",

View file

@ -80,6 +80,7 @@ try:
db = Database(
logger,
sqlalchemy_string=getenv("DATABASE_URI", None),
pool=False
)
# Create directories if they don't exist

View file

@ -6,9 +6,9 @@ local env = require "resty.env"
local whitelist = class("whitelist", plugin)
function whitelist:initialize()
function whitelist:initialize(ctx)
-- Call parent initialize
plugin.initialize(self, "whitelist")
plugin.initialize(self, "whitelist", ctx)
-- Decode lists
if ngx.get_phase() ~= "init" and self:is_needed() then
local lists, err = self.datastore:get("plugin_whitelist_lists", true)

View file

@ -50,6 +50,7 @@ from sqlalchemy.exc import (
SQLAlchemyError,
)
from sqlalchemy.orm import scoped_session, sessionmaker
from sqlalchemy.pool import NullPool
install_as_MySQLdb()
@ -61,6 +62,7 @@ class Database:
sqlalchemy_string: Optional[str] = None,
*,
ui: bool = False,
pool: bool = True
) -> None:
"""Initialize the database"""
self.__logger = logger
@ -92,6 +94,8 @@ class Database:
self.__sql_engine = create_engine(
sqlalchemy_string,
future=True,
poolclass=None if pool else NullPool,
pool_pre_ping=True
)
except ArgumentError:
self.__logger.error(f"Invalid database URI: {sqlalchemy_string}")
@ -135,6 +139,8 @@ class Database:
self.__sql_engine = create_engine(
sqlalchemy_string,
future=True,
poolclass=None if pool else NullPool,
pool_pre_ping=True
)
if "Unknown table" in str(e):
not_connected = False
@ -296,7 +302,7 @@ class Database:
return ""
def check_changes(self) -> Union[Dict[str, bool], bool, str]:
"""Check if either the config, the custom configs or plugins have changed inside the database"""
"""Check if either the config, the custom configs, plugins or instances have changed inside the database"""
with self.__db_session() as session:
try:
metadata = (
@ -305,6 +311,7 @@ class Database:
Metadata.custom_configs_changed,
Metadata.external_plugins_changed,
Metadata.config_changed,
Metadata.instances_changed
)
.filter_by(id=1)
.first()
@ -316,13 +323,14 @@ class Database:
external_plugins_changed=metadata is not None
and metadata.external_plugins_changed,
config_changed=metadata is not None and metadata.config_changed,
instances_changed=metadata is not None and metadata.instances_changed
)
except BaseException:
return format_exc()
def checked_changes(self, changes: Optional[List[str]] = None) -> str:
"""Set that the config, the custom configs and the plugins didn't change"""
changes = changes or ["config", "custom_configs", "external_plugins"]
"""Set that the config, the custom configs, the plugins and instances didn't change"""
changes = changes or ["config", "custom_configs", "external_plugins", "instances"]
with self.__db_session() as session:
try:
metadata = session.query(Metadata).get(1)
@ -336,6 +344,8 @@ class Database:
metadata.custom_configs_changed = False
if "external_plugins" in changes:
metadata.external_plugins_changed = False
if "instances" in changes:
metadata.instances_changed = False
session.commit()
except BaseException:
return format_exc()
@ -1739,6 +1749,11 @@ class Database:
)
)
with suppress(ProgrammingError, OperationalError):
metadata = session.query(Metadata).get(1)
if metadata is not None:
metadata.instances_changed = True
try:
session.add_all(to_put)
session.commit()

View file

@ -274,5 +274,6 @@ class Metadata(Base):
custom_configs_changed = Column(Boolean, default=False, nullable=True)
external_plugins_changed = Column(Boolean, default=False, nullable=True)
config_changed = Column(Boolean, default=False, nullable=True)
instances_changed = Column(Boolean, default=False, nullable=True)
integration = Column(INTEGRATIONS_ENUM, default="Unknown", nullable=False)
version = Column(String(32), default="1.5.0", nullable=False)

View file

@ -1,8 +1,7 @@
docker==6.1.3
jinja2==3.1.2
kubernetes==17.17.0
kubernetes==27.2.0
python-dotenv==1.0.0
pyyaml==5.3.1
redis==4.6.0
requests==2.31.0
urllib3==2.0.3
urllib3==2.0.4

View file

@ -111,9 +111,9 @@ jinja2==3.1.2 \
--hash=sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852 \
--hash=sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61
# via -r requirements.in
kubernetes==17.17.0 \
--hash=sha256:225a95a0aadbd5b645ab389d941a7980db8cdad2a776fde64d1b43fc3299bde9 \
--hash=sha256:c69b318696ba797dcf63eb928a8d4370c52319f4140023c502d7dfdf2080eb79
kubernetes==27.2.0 \
--hash=sha256:0f9376329c85cf07615ed6886bf9bf21eb1cbfc05e14ec7b0f74ed8153cd2815 \
--hash=sha256:d479931c6f37561dbfdf28fc5f46384b1cb8b28f9db344ed4a232ce91990825a
# via -r requirements.in
markupsafe==2.1.3 \
--hash=sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e \
@ -170,7 +170,9 @@ markupsafe==2.1.3 \
oauthlib==3.2.2 \
--hash=sha256:8139f29aac13e25d502680e9e19963e83f16838d48a0d71c287fe40e7067fbca \
--hash=sha256:9859c40929662bec5d64f34d01c99e093149682a3f38915dc0655d5a633dd918
# via requests-oauthlib
# via
# kubernetes
# requests-oauthlib
packaging==23.1 \
--hash=sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61 \
--hash=sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f
@ -193,23 +195,48 @@ python-dotenv==1.0.0 \
--hash=sha256:a8df96034aae6d2d50a4ebe8216326c61c3eb64836776504fcca410e5937a3ba \
--hash=sha256:f5971a9226b701070a4bf2c38c89e5a3f0d64de8debda981d1db98583009122a
# via -r requirements.in
pyyaml==5.3.1 \
--hash=sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97 \
--hash=sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76 \
--hash=sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2 \
--hash=sha256:6034f55dab5fea9e53f436aa68fa3ace2634918e8b5994d82f3621c04ff5ed2e \
--hash=sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648 \
--hash=sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf \
--hash=sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f \
--hash=sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2 \
--hash=sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee \
--hash=sha256:ad9c67312c84def58f3c04504727ca879cb0013b2517c85a9a253f0cb6380c0a \
--hash=sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d \
--hash=sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c \
--hash=sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a
# via
# -r requirements.in
# kubernetes
pyyaml==6.0.1 \
--hash=sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc \
--hash=sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741 \
--hash=sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206 \
--hash=sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27 \
--hash=sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595 \
--hash=sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62 \
--hash=sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98 \
--hash=sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696 \
--hash=sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d \
--hash=sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867 \
--hash=sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47 \
--hash=sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486 \
--hash=sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6 \
--hash=sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3 \
--hash=sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007 \
--hash=sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938 \
--hash=sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c \
--hash=sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735 \
--hash=sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d \
--hash=sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba \
--hash=sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8 \
--hash=sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5 \
--hash=sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd \
--hash=sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3 \
--hash=sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0 \
--hash=sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515 \
--hash=sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c \
--hash=sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c \
--hash=sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924 \
--hash=sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34 \
--hash=sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43 \
--hash=sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859 \
--hash=sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673 \
--hash=sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a \
--hash=sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab \
--hash=sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa \
--hash=sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c \
--hash=sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585 \
--hash=sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d \
--hash=sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f
# via kubernetes
redis==4.6.0 \
--hash=sha256:585dc516b9eb042a619ef0a39c3d7d55fe81bdb4df09a52c9cdde0d07bf1aa7d \
--hash=sha256:e2b03db868160ee4591de3cb90d40ebb50a90dd302138775937f6a42b7ed183c
@ -237,9 +264,9 @@ six==1.16.0 \
# google-auth
# kubernetes
# python-dateutil
urllib3==2.0.3 \
--hash=sha256:48e7fafa40319d358848e1bc6809b208340fafe2096f1725d05d67443d0483d1 \
--hash=sha256:bee28b5e56addb8226c96f7f13ac28cb4c301dd5ea8a6ca179c0b9835e032825
urllib3==2.0.4 \
--hash=sha256:8d22f86aae8ef5e410d4f539fde9ce6b2113a001bb4d189e0aed70642d602b11 \
--hash=sha256:de7df1803967d2c2a98e4b11bb7d6bd9210474c46e8a0401514e3a42a75ebde4
# via
# -r requirements.in
# docker
@ -251,9 +278,3 @@ websocket-client==1.6.1 \
# via
# docker
# kubernetes
# The following packages are considered to be unsafe in a requirements file:
setuptools==68.0.0 \
--hash=sha256:11e52c67415a381d10d6b462ced9cfb97066179f0e871399e006c4ab101fc85f \
--hash=sha256:baf1fdb41c6da4cd2eae722e135500da913332ab3f2f5c7d33af9b492acb5235
# via kubernetes

View file

@ -64,6 +64,7 @@ def get_instance_configs_and_apis(instance: Any, db, _type="Docker"):
db = Database(
logger,
sqlalchemy_string=splitted[1],
pool=False
)
elif splitted[0] == "API_HTTP_PORT":
api_http_port = splitted[1]
@ -164,7 +165,7 @@ if __name__ == "__main__":
external_plugins = args.plugins
if not Path(sep, "usr", "sbin", "nginx").exists() and args.method == "ui":
db = Database(logger)
db = Database(logger, pool=False)
external_plugins = []
for plugin in db.get_plugins():
external_plugins.append(plugin)
@ -224,7 +225,7 @@ if __name__ == "__main__":
f"Found custom conf env var {'for service ' + custom_conf[0] if custom_conf[0] else 'without service'} with type {custom_conf[1]} and name {custom_conf[2]}"
)
db = Database(logger, config_files.get("DATABASE_URI", None))
db = Database(logger, config_files.get("DATABASE_URI", None), pool=False)
else:
docker_client = DockerClient(
base_url=getenv("DOCKER_HOST", "unix:///var/run/docker.sock")
@ -269,6 +270,7 @@ if __name__ == "__main__":
db = Database(
logger,
sqlalchemy_string=splitted[1],
pool=False
)
elif splitted[0] == "API_HTTP_PORT":
api_http_port = splitted[1]
@ -283,7 +285,7 @@ if __name__ == "__main__":
)
if not db:
db = Database(logger)
db = Database(logger, pool=False)
# Compute the config
if not config_files:

View file

@ -28,7 +28,7 @@ RUN dnf install yum-utils redhat-lsb-core -y && \
# Compile and install dependencies
RUN dnf install -y python39-pip brotli brotli-devel wget gperftools-devel perl libxslt-devel libxml2 libxslt bash gd gd-devel gcc-c++ kernel-devel curl znc-modtcl libmpc-devel gmp-devel gawk mpfr-devel libtool pcre-devel automake autoconf readline-devel gcc make openssl-devel git zlib-devel libxml2-devel pkgconf libcurl-devel geoip-devel lmdb-libs && \
pip3.9 install --no-cache-dir --upgrade pip && \
pip3.9 install wheel && \
pip3.9 install --no-cache-dir --upgrade pip-tools wheel setuptools && \
chmod +x /tmp/bunkerweb/deps/install.sh && \
bash /tmp/bunkerweb/deps/install.sh && \
mkdir /usr/share/bunkerweb/deps/python && \

View file

@ -30,7 +30,7 @@ RUN apt-get install gnupg2 ca-certificates wget -y && \
# Compile and install dependencies
RUN apt install --no-install-recommends python3-pip bash libssl-dev git libpcre++-dev zlib1g-dev libxml2-dev libyajl-dev pkgconf libcurl4-openssl-dev libgeoip-dev liblmdb-dev apt-utils bash build-essential autoconf libtool automake g++ gcc libxml2-dev make musl-dev gnupg patch libreadline-dev libpcre3-dev libgd-dev -y && \
pip3 install --no-cache-dir --upgrade pip && \
pip3 install wheel && \
pip3 install --no-cache-dir --upgrade pip-tools wheel setuptools && \
#mkdir -p /usr/share/bunkerweb/deps && \
chmod +x /tmp/bunkerweb/deps/install.sh && \
bash /tmp/bunkerweb/deps/install.sh && \

View file

@ -26,7 +26,7 @@ RUN mkdir -p /usr/share/bunkerweb/deps && \
# Compile and install dependencies
RUN dnf install -y python3-pip brotli brotli-devel gperftools-devel perl libxslt-devel libxml2 libxslt bash gd gd-devel gcc-c++ kernel-devel curl znc-modtcl libmpc-devel gmp-devel gawk mpfr-devel libtool pcre-devel automake autoconf readline-devel gcc make openssl-devel git zlib-devel libxml2-devel pkgconf libcurl-devel geoip-devel lmdb-devel && \
pip install --no-cache-dir --upgrade pip && \
pip install wheel && \
pip3 install --no-cache-dir --upgrade pip-tools wheel setuptools && \
#mkdir -p /usr/share/bunkerweb/deps && \
chmod +x /tmp/bunkerweb/deps/install.sh && \
bash /tmp/bunkerweb/deps/install.sh && \

View file

@ -41,7 +41,7 @@ RUN mkdir -p /usr/share/bunkerweb/deps && \
# Compile and install dependencies
RUN dnf install -y readline-devel python39-pip brotli brotli-devel gperftools-devel perl libxslt-devel libxml2 libxslt bash gd gd-devel gcc-c++ curl znc-modtcl gawk libtool pcre-devel automake autoconf gcc make openssl-devel git zlib-devel libxml2-devel pkgconf libcurl-devel geoip-devel --skip-broken && \
pip3.9 install --no-cache-dir --upgrade pip && \
pip3.9 install wheel && \
pip3.9 install --no-cache-dir --upgrade pip-tools wheel setuptools && \
chmod +x /tmp/bunkerweb/deps/install.sh && \
bash /tmp/bunkerweb/deps/install.sh && \
mkdir /usr/share/bunkerweb/deps/python && \

View file

@ -31,7 +31,7 @@ RUN apt update && \
# Compile and install dependencies
RUN apt install --no-install-recommends bash libssl-dev git libpcre++-dev zlib1g-dev libxml2-dev libyajl-dev pkgconf libcurl4-openssl-dev libgeoip-dev liblmdb-dev apt-utils bash build-essential autoconf libtool automake g++ gcc libxml2-dev make musl-dev gnupg patch libreadline-dev libpcre3-dev libgd-dev python3-pip -y && \
pip install --no-cache-dir --upgrade pip && \
pip install wheel && \
pip3 install --no-cache-dir --upgrade pip-tools wheel setuptools && \
chmod +x /tmp/bunkerweb/deps/install.sh && \
bash /tmp/bunkerweb/deps/install.sh && \
mkdir /usr/share/bunkerweb/deps/python && \

View file

@ -15,7 +15,7 @@ RUN apk add --no-cache --virtual .build-deps g++ gcc musl-dev jpeg-dev zlib-dev
# Install python requirements
RUN export MAKEFLAGS="-j$(nproc)" && \
pip install --no-cache-dir --upgrade pip && \
pip install --no-cache-dir --upgrade wheel && \
pip install --no-cache-dir --upgrade pip-tools wheel setuptools && \
mkdir -p /usr/share/bunkerweb/deps/python && \
pip install --no-cache-dir --require-hashes --target /usr/share/bunkerweb/deps/python -r /usr/share/bunkerweb/deps/requirements.txt

View file

@ -30,7 +30,7 @@ for deps_path in [
from Database import Database # type: ignore
from logger import setup_logger # type: ignore
from ApiCaller import ApiCaller # type: ignore
from API import API # type: ignore
class JobScheduler(ApiCaller):
def __init__(
@ -68,6 +68,23 @@ class JobScheduler(ApiCaller):
def auto_setup(self):
super().auto_setup(bw_integration=self.__integration)
def update_instances(self):
super(JobScheduler, type(self)).apis.fset(self, self.__get_apis())
def __get_apis(self):
apis = []
try:
with self.__thread_lock:
instances = self.__db.get_instances()
for instance in instances:
api = API(f"http://{instance['hostname']}:{instance['port']}", host=instance["server_name"])
apis.append(api)
except:
self.__logger.warning(
f"Exception while getting jobs instances : {format_exc()}",
)
return apis
def update_jobs(self):
self.__jobs = self.__get_jobs()

View file

@ -587,6 +587,7 @@ if __name__ == "__main__":
CONFIG_NEED_GENERATION = False
CONFIGS_NEED_GENERATION = False
PLUGINS_NEED_GENERATION = False
INSTANCES_NEED_GENERATION = False
# infinite schedule for the jobs
logger.info("Executing job scheduler ...")
@ -659,6 +660,12 @@ if __name__ == "__main__":
CONFIG_NEED_GENERATION = True
NEED_RELOAD = True
# check if the instances have changed since last time
if changes["instances_changed"]:
logger.info("Instances changed, generating ...")
INSTANCES_NEED_GENERATION = True
NEED_RELOAD = True
FIRST_RUN = False
if NEED_RELOAD:
@ -683,6 +690,10 @@ if __name__ == "__main__":
if CONFIG_NEED_GENERATION:
CHANGES.append("config")
env = db.get_config()
if INSTANCES_NEED_GENERATION:
CHANGES.append("instances")
SCHEDULER.update_instances()
except:
logger.error(
f"Exception while executing scheduler : {format_exc()}",

View file

@ -256,9 +256,9 @@ six==1.16.0 \
--hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \
--hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254
# via configobj
urllib3==2.0.3 \
--hash=sha256:48e7fafa40319d358848e1bc6809b208340fafe2096f1725d05d67443d0483d1 \
--hash=sha256:bee28b5e56addb8226c96f7f13ac28cb4c301dd5ea8a6ca179c0b9835e032825
urllib3==2.0.4 \
--hash=sha256:8d22f86aae8ef5e410d4f539fde9ce6b2113a001bb4d189e0aed70642d602b11 \
--hash=sha256:de7df1803967d2c2a98e4b11bb7d6bd9210474c46e8a0401514e3a42a75ebde4
# via requests
# The following packages are considered to be unsafe in a requirements file:

View file

@ -15,7 +15,7 @@ RUN apk add --no-cache --virtual .build-deps g++ gcc musl-dev jpeg-dev zlib-dev
# Install python requirements
RUN export MAKEFLAGS="-j$(nproc)" && \
pip install --no-cache-dir --upgrade pip && \
pip install --no-cache-dir --upgrade wheel && \
pip install --no-cache-dir --upgrade pip-tools wheel setuptools && \
mkdir -p /usr/share/bunkerweb/deps/python && \
pip install --no-cache-dir --require-hashes --target /usr/share/bunkerweb/deps/python -r /usr/share/bunkerweb/deps/requirements.txt

View file

@ -3,6 +3,6 @@ beautifulsoup4==4.12.2
Flask==2.3.2
Flask_Login==0.6.2
Flask_WTF==1.1.1
gunicorn[gevent]==20.1.0
gunicorn[gevent]==21.2.0
python_dateutil==2.8.2
regex==2023.6.3

View file

@ -35,9 +35,9 @@ blinker==1.6.2 \
--hash=sha256:4afd3de66ef3a9f8067559fb7a1cbe555c17dcbe15971b05d1b625c3e7abe213 \
--hash=sha256:c3d739772abb7bc2860abf5f2ec284223d9ad5c76da018234f6f50d6f31ab1f0
# via flask
click==8.1.5 \
--hash=sha256:4be4b1af8d665c6d942909916d31a213a106800c47d0eeba73d34da3cbc11367 \
--hash=sha256:e576aa487d679441d7d30abb87e1b43d24fc53bffb8758443b1a9e1cee504548
click==8.1.6 \
--hash=sha256:48ee849951919527a045bfe3bf7baa8a959c423134e1a5b98c05c20ba75a1cbd \
--hash=sha256:fa244bb30b3b5ee2cae3da8f55c9e5e0c0e86093306301fb418eb9dc40fbded5
# via flask
flask==2.3.2 \
--hash=sha256:77fd4e1249d8c9923de34907236b747ced06e5467ecac1a7bb7115ae0e9670b0 \
@ -149,9 +149,9 @@ greenlet==2.0.2 \
--hash=sha256:f82d4d717d8ef19188687aa32b8363e96062911e63ba22a0cff7802a8e58e5f1 \
--hash=sha256:fc3a569657468b6f3fb60587e48356fe512c1754ca05a564f11366ac9e306526
# via gevent
gunicorn[gevent]==20.1.0 \
--hash=sha256:9dcc4547dbb1cb284accfb15ab5667a0e5d1881cc443e0677b4882a4067a807e \
--hash=sha256:e0a968b5ba15f8a328fdfd7ab1fcb5af4470c28aaf7e55df02a99bc13138e6e8
gunicorn[gevent]==21.2.0 \
--hash=sha256:3213aa5e8c24949e792bcacfc176fef362e7aac80b76c56f6b5122bf350722f0 \
--hash=sha256:88ec8bff1d634f98e61b9f65bc4bf3cd918a90806c6f5c48bc5603849ec81033
# via -r requirements.in
importlib-metadata==6.8.0 \
--hash=sha256:3ebb78df84a805d7698245025b975d9d67053cd94c79245ba4b3eb694abe68bb \
@ -222,6 +222,10 @@ markupsafe==2.1.3 \
# jinja2
# werkzeug
# wtforms
packaging==23.1 \
--hash=sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61 \
--hash=sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f
# via gunicorn
python-dateutil==2.8.2 \
--hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \
--hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9
@ -380,6 +384,5 @@ setuptools==68.0.0 \
--hash=sha256:11e52c67415a381d10d6b462ced9cfb97066179f0e871399e006c4ab101fc85f \
--hash=sha256:baf1fdb41c6da4cd2eae722e135500da913332ab3f2f5c7d33af9b492acb5235
# via
# gunicorn
# zope-event
# zope-interface

View file

@ -1,2 +1,2 @@
fastapi==0.100.0
uvicorn[standard]==0.23.0
uvicorn[standard]==0.23.1

View file

@ -1,2 +1,2 @@
fastapi==0.100.0
uvicorn[standard]==0.23.0
uvicorn[standard]==0.23.1

View file

@ -1,2 +1,2 @@
fastapi==0.100.0
uvicorn[standard]==0.23.0
uvicorn[standard]==0.23.1

View file

@ -1,5 +1,5 @@
requests==2.31.0
redis==4.6.0
fastapi==0.100.0
uvicorn[standard]==0.23.0
uvicorn[standard]==0.23.1
selenium==4.10.0

View file

@ -1,3 +1,3 @@
requests==2.31.0
fastapi==0.100.0
uvicorn[standard]==0.23.0
uvicorn[standard]==0.23.1

View file

@ -1,2 +1,2 @@
fastapi==0.100.0
uvicorn[standard]==0.23.0
uvicorn[standard]==0.23.1

View file

@ -50,6 +50,11 @@ if [ $? -ne 0 ] ; then
exit 1
fi
docker-compose -f docker-compose.test.yml up --abort-on-container-exit --exit-code-from ui-tests
ret=$?
if [ $ret -ne 0 ] ; then
docker-compose logs
echo "❌ Up failed"
fi
# Exit
exit $?