Elgato_dark/bunkerweb
1
0
Fork 0
mirror of https://github.com/bunkerity/bunkerweb synced 2026-05-24 09:28:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'dev' of github.com:bunkerity/bunkerweb into dev

Browse source
This commit is contained in:
fl0ppy-d1sk 2024-05-27 17:04:11 +02:00
commit baba82757f
No known key found for this signature in database
GPG key ID: 93EE47CC3D061500
30 changed files with 387 additions and 171 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
CHANGELOG.md
View file

@ -2,11 +2,23 @@
## v1.5.8 - ????/??/??
- [BUGFIX] Fix potential errors when upgrading from a previous version
- [BUGFIX] Fix rare bug on the web UI when editing the SERVER_NAME setting of a service
- [BUGFIX] Fix potential race conditions between the autoconf and the scheduler waiting for each other indefinitely
- [FEATURE] Add nightly build of the OWASP coreruleset that are automatically downloaded and updated
- [FEATURE] Enhance security on error pages, default server page and loading page by adding a custom `Content-Security-Policy` header with nonces and removing the `Server` header
- [FEATURE] Add new DATABASE_URI_READONLY setting to allow setting up a fallback read-only database URI in case the main database URI is not available
- [FEATURE] Add automatic fallback to either read-only on the primary database or to the read-only database URI when the main database URI is not available and automatically switch back to the main database URI when it becomes available again
- [FEATURE] Add experimental support of HTTP/3 (QUIC)
- [FEATURE] Optimize the way the scheduler handles jobs and the way the jobs are executed
- [FEATURE] Optimize the way the cache files are being refreshed from the database
- [UI] Force HTTPS on setup wizard
- [UI] Fallback to self-signed certificate when UI is installed with setup wizard and let's encrypt is not used
- [UI] Add OVERRIDE_ADMIN_CREDS environment variable to allow overriding the default admin credentials even if an admin user already exists
- [DEPS] Updated NGINX version to v1.26.0
- [DEPS] Updated stream-lua-nginx-module version to the latest commit to incorporate the latest changes and fixes for NGINX v1.26.0
- [DEPS] Updated coreruleset-v4 version to v4.3.0
- [DEPS] Updated lua-resty-openssl version to v1.4.0
## v1.5.7 - 2024/05/14

5
misc/dev/docker-compose.autoconf.misc.yml
View file

@ -10,8 +10,9 @@ services:
context: ../..
dockerfile: ./src/bw/Dockerfile
ports:
- 80:8080
- 443:8443
- 80:8080/tcp
- 443:8443/tcp
- 443:8443/udp
labels:
- "bunkerweb.INSTANCE=yes"
environment:

5
misc/dev/docker-compose.autoconf.ui.misc.yml
View file

@ -10,8 +10,9 @@ services:
context: ../..
dockerfile: ./src/bw/Dockerfile
ports:
- 80:8080
- 443:8443
- 80:8080/tcp
- 443:8443/tcp
- 443:8443/udp
labels:
- "bunkerweb.INSTANCE=yes"
environment:

5
misc/dev/docker-compose.autoconf.ui.yml
View file

@ -10,8 +10,9 @@ services:
context: ../..
dockerfile: ./src/bw/Dockerfile
ports:
- 80:8080
- 443:8443
- 80:8080/tcp
- 443:8443/tcp
- 443:8443/udp
labels:
- "bunkerweb.INSTANCE=yes"
environment:

5
misc/dev/docker-compose.autoconf.wizard.yml
View file

@ -10,8 +10,9 @@ services:
context: ../..
dockerfile: ./src/bw/Dockerfile
ports:
- 80:8080
- 443:8443
- 80:8080/tcp
- 443:8443/tcp
- 443:8443/udp
labels:
- "bunkerweb.INSTANCE=yes"
environment:

5
misc/dev/docker-compose.autoconf.yml
View file

@ -10,8 +10,9 @@ services:
context: ../..
dockerfile: ./src/bw/Dockerfile
ports:
- 80:8080
- 443:8443
- 80:8080/tcp
- 443:8443/tcp
- 443:8443/udp
labels:
- "bunkerweb.INSTANCE=yes"
environment:

5
misc/dev/docker-compose.misc.yml
View file

@ -4,8 +4,9 @@ services:
context: ../..
dockerfile: ./src/bw/Dockerfile
ports:
- 80:8080
- 443:8443
- 80:8080/tcp
- 443:8443/tcp
- 443:8443/udp
labels:
- "bunkerweb.INSTANCE=yes"
environment:

5
misc/dev/docker-compose.ui.misc.yml
View file

@ -9,8 +9,9 @@ services:
context: ../..
dockerfile: ./src/bw/Dockerfile
ports:
- 80:8080
- 443:8443
- 80:8080/tcp
- 443:8443/tcp
- 443:8443/udp
labels:
- "bunkerweb.INSTANCE=yes"
environment:

5
misc/dev/docker-compose.ui.yml
View file

@ -9,8 +9,9 @@ services:
context: ../..
dockerfile: ./src/bw/Dockerfile
ports:
- 80:8080
- 443:8443
- 80:8080/tcp
- 443:8443/tcp
- 443:8443/udp
labels:
- "bunkerweb.INSTANCE=yes"
environment:

5
misc/dev/docker-compose.wizard.yml
View file

@ -9,8 +9,9 @@ services:
context: ../..
dockerfile: ./src/bw/Dockerfile
ports:
- 80:8080
- 443:8443
- 80:8080/tcp
- 443:8443/tcp
- 443:8443/udp
labels:
- "bunkerweb.INSTANCE=yes"
environment:

5
misc/dev/docker-compose.yml
View file

@ -4,8 +4,9 @@ services:
context: ../..
dockerfile: ./src/bw/Dockerfile
ports:
- 80:8080
- 443:8443
- 80:8080/tcp
- 443:8443/tcp
- 443:8443/udp
labels:
- "bunkerweb.INSTANCE=yes"
environment:

36
src/autoconf/Config.py
View file

@ -120,9 +120,7 @@ class Config(ConfigCaller):
)
while not self._db.is_initialized():
self.__logger.warning(
"Database is not initialized, retrying in 5 seconds ...",
)
self.__logger.warning("Database is not initialized, retrying in 5 seconds ...")
sleep(5)
# wait until changes are applied
@ -133,9 +131,7 @@ class Config(ConfigCaller):
elif not any(curr_changes.values()):
break
else:
self.__logger.warning(
"Scheduler is already applying a configuration, retrying in 5 seconds ...",
)
self.__logger.warning("Scheduler is already applying a configuration, retrying in 5 seconds ...")
sleep(5)
# update instances in database
@ -144,27 +140,23 @@ class Config(ConfigCaller):
if err:
self.__logger.error(f"Failed to update instances: {err}")
# save config to database
if "config" in changes:
err = self._db.save_config(self.__config, "autoconf", changed=False)
if err:
success = False
self.__logger.error(
f"Can't save config in database: {err}, config may not work as expected",
)
# save custom configs to database
if "custom_configs" in changes:
err = self._db.save_custom_configs(custom_configs, "autoconf", changed=False)
if err:
success = False
self.__logger.error(
f"Can't save autoconf custom configs in database: {err}, custom configs may not work as expected",
)
self.__logger.error(f"Can't save autoconf custom configs in database: {err}, custom configs may not work as expected")
# update changes in db
ret = self._db.checked_changes(changes, value=True)
if ret:
self.__logger.error(f"An error occurred when setting the changes to checked in the database : {ret}")
# save config to database
if "config" in changes:
err = self._db.save_config(self.__config, "autoconf")
if err:
success = False
self.__logger.error(f"Can't save config in database: {err}, config may not work as expected")
else:
# update changes in db
ret = self._db.checked_changes(changes, value=True)
if ret:
self.__logger.error(f"An error occurred when setting the changes to checked in the database : {ret}")
return success

189
src/common/db/Database.py
View file

@ -487,7 +487,6 @@ class Database:
Metadata.custom_configs_changed,
Metadata.external_plugins_changed,
Metadata.pro_plugins_changed,
Metadata.config_changed,
Metadata.instances_changed,
)
.filter_by(id=1)
@ -498,21 +497,24 @@ class Database:
custom_configs_changed=metadata is not None and metadata.custom_configs_changed,
external_plugins_changed=metadata is not None and metadata.external_plugins_changed,
pro_plugins_changed=metadata is not None and metadata.pro_plugins_changed,
config_changed=metadata is not None and metadata.config_changed,
instances_changed=metadata is not None and metadata.instances_changed,
plugins_config_changed=self.check_plugin_changes(),
)
except BaseException as e:
return str(e)
def check_plugin_changes(self) -> Union[List[str], str]:
"""Check if the plugins have changed inside the database"""
with self.__db_session() as session:
try:
plugins = session.query(Plugins).with_entities(Plugins.id).filter_by(config_changed=True).all()
return [plugin.id for plugin in plugins]
except BaseException as e:
return str(e)
def checked_changes(self, changes: Optional[List[str]] = None, value: Optional[bool] = False) -> str:
"""Set changed bit for config, custom configs, instances and plugins"""
changes = changes or [
"config",
"custom_configs",
"external_plugins",
"pro_plugins",
"instances",
]
changes = changes or ["config", "custom_configs", "external_plugins", "pro_plugins", "instances"]
with self.__db_session() as session:
if self.readonly:
return "The database is read-only, the changes will not be saved"
@ -526,7 +528,6 @@ class Database:
if "config" in changes:
if not metadata.first_config_saved:
metadata.first_config_saved = True
metadata.config_changed = value
if "custom_configs" in changes:
metadata.custom_configs_changed = value
if "external_plugins" in changes:
@ -541,6 +542,25 @@ class Database:
return ""
def checked_plugins_changes(self, plugins: Optional[List[str]] = None, value: Optional[bool] = False) -> str:
"""Set changed bit for plugins"""
with self.__db_session() as session:
if self.readonly:
return "The database is read-only, the changes will not be saved"
plugins = plugins or []
try:
query = session.query(Plugins)
if plugins:
query = query.filter(Plugins.id.in_(plugins))
query.update({Plugins.config_changed: value})
session.commit()
except BaseException as e:
return str(e)
return ""
def init_tables(self, default_plugins: List[dict], bunkerweb_version: str) -> Tuple[bool, str]:
"""Initialize the database tables and return the result"""
@ -1117,9 +1137,29 @@ class Database:
if self.readonly:
return "The database is read-only, the changes will not be saved"
# Delete all the old config
session.query(Global_values).filter(Global_values.method == method).delete()
session.query(Services_settings).filter(Services_settings.method == method).delete()
changed_plugins = set()
changed_services = False
for db_global_config in session.query(Global_values).filter_by(method=method).all():
key = db_global_config.setting_id
if db_global_config.suffix:
key = f"{key}_{db_global_config.suffix}"
if key not in config and (db_global_config.suffix or f"{key}_0" not in config):
session.delete(db_global_config)
changed_plugins.add(session.query(Settings).with_entities(Settings.plugin_id).filter_by(id=db_global_config.setting_id).first().plugin_id)
if key == "SERVER_NAME":
changed_services = True
for db_service_config in session.query(Services_settings).filter_by(method=method).all():
key = f"{db_service_config.service_id}_{db_service_config.setting_id}"
if db_service_config.suffix:
key = f"{key}_{db_service_config.suffix}"
if key not in config and (db_service_config.suffix or f"{key}_0" not in config):
session.delete(db_service_config)
changed_plugins.add(session.query(Settings).with_entities(Settings.plugin_id).filter_by(id=db_service_config.setting_id).first().plugin_id)
if config:
config.pop("DATABASE_URI", None)
@ -1140,6 +1180,7 @@ class Database:
session.query(Services_settings).filter(Services_settings.service_id.in_(missing_ids)).delete()
session.query(Custom_configs).filter(Custom_configs.service_id.in_(missing_ids)).delete()
session.query(Jobs_cache).filter(Jobs_cache.service_id.in_(missing_ids)).delete()
changed_services = True
drafts = {service for service in services if config.pop(f"{service}_IS_DRAFT", "no") == "yes"}
db_drafts = {service.id for service in db_services if service.is_draft}
@ -1152,6 +1193,7 @@ class Database:
if missing_drafts:
# Remove drafts that are no longer in the list
session.query(Services).filter(Services.id.in_(missing_drafts)).update({Services.is_draft: False})
changed_services = True
for draft in drafts:
if draft not in db_drafts:
@ -1160,6 +1202,7 @@ class Database:
db_ids[draft] = {"method": method, "is_draft": True}
elif method == db_ids[draft]["method"]:
session.query(Services).filter(Services.id == draft).update({Services.is_draft: True})
changed_services = True
if config.get("MULTISITE", "no") == "yes":
global_values = []
@ -1170,7 +1213,7 @@ class Database:
suffix = int(key.split("_")[-1])
key = key[: -len(str(suffix)) - 1]
setting = session.query(Settings).with_entities(Settings.default).filter_by(id=key).first()
setting = session.query(Settings).with_entities(Settings.default, Settings.plugin_id).filter_by(id=key).first()
if not setting and services:
try:
@ -1181,10 +1224,12 @@ class Database:
if server_name not in db_ids:
to_put.append(Services(id=server_name, method=method, is_draft=server_name in drafts))
db_ids[server_name] = {"method": method, "is_draft": server_name in drafts}
if server_name not in drafts:
changed_services = True
key = key.replace(f"{server_name}_", "")
original_key = original_key.replace(f"{server_name}_", "")
setting = session.query(Settings).with_entities(Settings.default).filter_by(id=key).first()
setting = session.query(Settings).with_entities(Settings.default, Settings.plugin_id).filter_by(id=key).first()
if not setting:
continue
@ -1192,11 +1237,7 @@ class Database:
service_setting = (
session.query(Services_settings)
.with_entities(Services_settings.value, Services_settings.method)
.filter_by(
service_id=server_name,
setting_id=key,
suffix=suffix,
)
.filter_by(service_id=server_name, setting_id=key, suffix=suffix)
.first()
)
@ -1206,45 +1247,29 @@ class Database:
):
continue
to_put.append(
Services_settings(
service_id=server_name,
setting_id=key,
value=value,
suffix=suffix,
method=method,
)
)
changed_plugins.add(setting.plugin_id)
to_put.append(Services_settings(service_id=server_name, setting_id=key, value=value, suffix=suffix, method=method))
elif method in (service_setting.method, "autoconf") and service_setting.value != value:
if key != "SERVER_NAME" and (
(original_key not in config and value == setting.default) or (original_key in config and value == config[original_key])
):
session.query(Services_settings).filter(
Services_settings.service_id == server_name,
Services_settings.setting_id == key,
Services_settings.suffix == suffix,
).delete()
continue
session.query(Services_settings).filter(
changed_plugins.add(setting.plugin_id)
query = session.query(Services_settings).filter(
Services_settings.service_id == server_name,
Services_settings.setting_id == key,
Services_settings.suffix == suffix,
).update(
{
Services_settings.value: value,
Services_settings.method: method,
}
)
if key != "SERVER_NAME" and (
(original_key not in config and value == setting.default) or (original_key in config and value == config[original_key])
):
query.delete()
continue
query.update({Services_settings.value: value, Services_settings.method: method})
elif setting and original_key not in global_values:
global_values.append(original_key)
global_value = (
session.query(Global_values)
.with_entities(Global_values.value, Global_values.method)
.filter_by(
setting_id=key,
suffix=suffix,
)
.filter_by(setting_id=key, suffix=suffix)
.first()
)
@ -1252,31 +1277,16 @@ class Database:
if value == setting.default:
continue
to_put.append(
Global_values(
setting_id=key,
value=value,
suffix=suffix,
method=method,
)
)
changed_plugins.add(setting.plugin_id)
to_put.append(Global_values(setting_id=key, value=value, suffix=suffix, method=method))
elif method in (global_value.method, "autoconf") and global_value.value != value:
if value == setting.default:
session.query(Global_values).filter(
Global_values.setting_id == key,
Global_values.suffix == suffix,
).delete()
continue
changed_plugins.add(setting.plugin_id)
query = session.query(Global_values).filter(Global_values.setting_id == key, Global_values.suffix == suffix)
session.query(Global_values).filter(
Global_values.setting_id == key,
Global_values.suffix == suffix,
).update(
{
Global_values.value: value,
Global_values.method: method,
}
)
if value == setting.default:
query.delete()
continue
query.update({Global_values.value: value, Global_values.method: method})
else:
if (
config.get("SERVER_NAME", "www.example.com")
@ -1286,6 +1296,7 @@ class Database:
.first()
):
to_put.append(Services(id=config.get("SERVER_NAME", "www.example.com").split(" ")[0], method=method))
changed_services = True
for key, value in config.items():
suffix = 0
@ -1293,7 +1304,7 @@ class Database:
suffix = int(key.split("_")[-1])
key = key[: -len(str(suffix)) - 1]
setting = session.query(Settings).with_entities(Settings.default).filter_by(id=key).first()
setting = session.query(Settings).with_entities(Settings.default, Settings.plugin_id).filter_by(id=key).first()
if not setting:
continue
@ -1309,26 +1320,16 @@ class Database:
if value == setting.default:
continue
to_put.append(
Global_values(
setting_id=key,
value=value,
suffix=suffix,
method=method,
)
)
changed_plugins.add(setting.plugin_id)
to_put.append(Global_values(setting_id=key, value=value, suffix=suffix, method=method))
elif global_value.method == method and value != global_value.value:
if value == setting.default:
session.query(Global_values).filter(
Global_values.setting_id == key,
Global_values.suffix == suffix,
).delete()
continue
changed_plugins.add(setting.plugin_id)
query = session.query(Global_values).filter(Global_values.setting_id == key, Global_values.suffix == suffix)
session.query(Global_values).filter(
Global_values.setting_id == key,
Global_values.suffix == suffix,
).update({Global_values.value: value})
if value == setting.default:
query.delete()
continue
query.update({Global_values.value: value})
if changed:
with suppress(ProgrammingError, OperationalError):
@ -1336,7 +1337,11 @@ class Database:
if metadata is not None:
if not metadata.first_config_saved:
metadata.first_config_saved = True
metadata.config_changed = True
if changed_services:
session.query(Plugins).update({Plugins.config_changed: True})
elif changed_plugins:
session.query(Plugins).filter(Plugins.id.in_(changed_plugins)).update({Plugins.config_changed: True})
try:
session.add_all(to_put)

2
src/common/db/model.py
View file

@ -58,6 +58,7 @@ class Plugins(Base):
method = Column(METHODS_ENUM, default="manual", nullable=False)
data = Column(LargeBinary(length=(2**32) - 1), nullable=True)
checksum = Column(String(128), nullable=True)
config_changed = Column(Boolean, default=False, nullable=True)
settings = relationship("Settings", back_populates="plugin", cascade="all, delete-orphan")
jobs = relationship("Jobs", back_populates="plugin", cascade="all, delete-orphan")
@ -243,7 +244,6 @@ class Metadata(Base):
custom_configs_changed = Column(Boolean, default=False, nullable=True)
external_plugins_changed = Column(Boolean, default=False, nullable=True)
pro_plugins_changed = Column(Boolean, default=False, nullable=True)
config_changed = Column(Boolean, default=False, nullable=True)
instances_changed = Column(Boolean, default=False, nullable=True)
integration = Column(INTEGRATIONS_ENUM, default="Unknown", nullable=False)
version = Column(String(32), default="1.5.8", nullable=False)

4
src/deps/deps.json
View file

@ -151,9 +151,9 @@
},
{
"id": "lua-resty-openssl",
"name": "lua-resty-openssl v1.3.1",
"name": "lua-resty-openssl v1.4.0",
"url": "https://github.com/fffonion/lua-resty-openssl.git",
"commit": "643956d990b3ef4d9b52d0b77ba4d69d7e912dcc",
"commit": "e56da6c5f285ff0e40fce703b870e83ee8eac321",
"post_install": "rm -r src/deps/src/lua-resty-openssl/t"
},
{

18
src/deps/src/lua-resty-openssl/.github/workflows/tests.yml vendored
View file

@ -30,14 +30,14 @@ jobs:
# latest and one version older for valgrind and perf test
- nginx: "1.19.9"
openssl: "3.1.5"
openssl_fips: "3.0.8"
openssl_fips: "3.0.9"
extras: "valgrind"
lua_nginx_module: "v0.10.20"
lua_resty_core: "v0.1.22"
nginx_cc_opts: "-Wno-error"
- nginx: "1.21.4"
openssl: "3.1.5"
openssl_fips: "3.0.8"
openssl_fips: "3.0.9"
extras: "valgrind"
lua_nginx_module: "v0.10.25"
lua_resty_core: "v0.1.27"
@ -49,25 +49,33 @@ jobs:
lua_resty_core: "v0.1.28"
- nginx: "1.25.3"
openssl: "3.0.13"
openssl_fips: "3.0.8"
openssl_fips: "3.0.9"
extras: "valgrind perf lua-kong-nginx-module"
lua_nginx_module: "v0.10.26"
lua_resty_core: "v0.1.28"
nginx_cc_opts: "-Wno-error"
- nginx: "1.25.3"
openssl: "3.1.5"
openssl_fips: "3.0.8"
openssl_fips: "3.0.9"
extras: "valgrind perf lua-kong-nginx-module"
lua_nginx_module: "v0.10.26"
lua_resty_core: "v0.1.28"
nginx_cc_opts: "-Wno-error"
- nginx: "1.25.3"
openssl: "3.2.1"
openssl_fips: "3.0.8"
openssl_fips: "3.0.9"
extras: "valgrind perf lua-kong-nginx-module"
lua_nginx_module: "v0.10.26"
lua_resty_core: "v0.1.28"
nginx_cc_opts: "-Wno-error"
- nginx: "1.25.3"
openssl: "3.3.0"
openssl_fips: "3.0.9"
extras: "valgrind perf lua-kong-nginx-module"
lua_nginx_module: "v0.10.26"
lua_resty_core: "v0.1.28"
nginx_cc_opts: "-Wno-error"
env:
JOBS: 3

17
src/deps/src/lua-resty-openssl/CHANGELOG.md vendored
View file

@ -2,12 +2,22 @@
## [Unreleased]
<a name="1.4.0"></a>
## [1.4.0] - 2024-05-27
### bug fixes
- **ec:** add missing cdef for EC_POINT_free [2093e88](https://github.com/fffonion/lua-resty-openssl/commit/2093e8814ccfbe830ba594a71f05870cac208e9c)
### features
- **pkey:** allow pkey.new to compose from parameters [91a30f6](https://github.com/fffonion/lua-resty-openssl/commit/91a30f6988e3fc696363ce1445b49a3f6ee8f35e)
- **pkey:** add pkey:verify_raw [0016308](https://github.com/fffonion/lua-resty-openssl/commit/0016308c9e3a2ccfdfe674ede64e462060f7b13b)
<a name="1.3.1"></a>
## [1.3.1] - 2024-04-22
### bug fixes
- **aux/jwk:** remove ecx.d if exporting as public key [9d34ff8](https://github.com/fffonion/lua-resty-openssl/commit/9d34ff8fd79debbcf155f74af0b161083b6a8385)
- **aux/nginx:** fix the typo of get_socket_ssl in the stream module [0aa315e](https://github.com/fffonion/lua-resty-openssl/commit/0aa315efe3d98d38d8d77dedf687958b62d8b184)
- **aux/nginx:** remove extra sanity test that prevent usage of lua-kong-nginx-module [7bd2d0a](https://github.com/fffonion/lua-resty-openssl/commit/7bd2d0aabe82219071fe9fd1b30e49ff88bd5472)
- **aux/nginx:** fix the typo of get_socket_ssl in the stream module [ad18b3c](https://github.com/fffonion/lua-resty-openssl/commit/ad18b3c18c7ec3db175aabfbf6928141c3b53b17)
- **aux/nginx:** remove extra sanity test that prevent usage of lua-kong-nginx-module [2323526](https://github.com/fffonion/lua-resty-openssl/commit/2323526766b131bde94052449c41a331e93288bd)
<a name="1.3.0"></a>
@ -591,7 +601,8 @@
- **x509:** export pubkey [ede4f81](https://github.com/fffonion/lua-resty-openssl/commit/ede4f817cb0fe092ad6f9ab5d6ecdcde864a9fd8)
[Unreleased]: https://github.com/fffonion/lua-resty-openssl/compare/1.3.1...HEAD
[Unreleased]: https://github.com/fffonion/lua-resty-openssl/compare/1.4.0...HEAD
[1.4.0]: https://github.com/fffonion/lua-resty-openssl/compare/1.3.1...1.4.0
[1.3.1]: https://github.com/fffonion/lua-resty-openssl/compare/1.3.0...1.3.1
[1.3.0]: https://github.com/fffonion/lua-resty-openssl/compare/1.2.1...1.3.0
[1.2.1]: https://github.com/fffonion/lua-resty-openssl/compare/1.2.0...1.2.1

48
src/deps/src/lua-resty-openssl/README.md vendored
View file

@ -66,6 +66,7 @@ Table of Contents
+ [pkey:encrypt](#pkeyencrypt)
+ [pkey:decrypt](#pkeydecrypt)
+ [pkey:sign_raw](#pkeysign_raw)
+ [pkey:verify_raw](#pkeyverify_raw)
+ [pkey:verify_recover](#pkeyverify_recover)
+ [pkey:derive](#pkeyderive)
+ [pkey:tostring](#pkeytostring)
@ -871,13 +872,14 @@ for EC keys. See [pkey:sign](#pkeysign) and [pkey.verify](#pkeyverify) for detai
- When running outside of OpenResty, needs to install a JSON library (`cjson` or `dkjson`)
and `basexx`.
[Back to TOC](#table-of-contents)
#### Key generation
**syntax**: *pk, err = pkey.new(config?)*
Generate a new public key or private key.
To generate RSA key, `config` table can have `bits` and `exp` field to control key generation.
When `config` is emitted, this function generates a 2048 bit RSA key with `exponent` of 65537,
which is equivalent to:
@ -936,6 +938,27 @@ pkey.new({
```
[Back to TOC](#table-of-contents)
#### Key composition
**syntax**: *pk, err = pkey.new(config?)*
Compose a public or private key using existing parameters. To see
list of parameters for each key, refer to [pkey:set_parameters](#pkeyset_parameters).
Only `type` and `params` should exist in `config` table, all other keys will be ignored.
```lua
local private_bn = require "resty.openssl.bn".new("7F48282CCA4C1A65D589C06DBE9C42AE50FBFFDF3A18CBB48498E1DE47F11BE1A3486CD8FA950D68F111970F922279D8", 16)
local p_384, err = assert(require("resty.openssl.pkey").new({
type = "EC",
params = {
private = private_bn,
group = "secp384r1",
}
}))
```
[Back to TOC](#table-of-contents)
@ -1195,6 +1218,8 @@ pk:sign(message, nil, nil, {
-- in pkeyutl CLI the above is equivalent to: `openssl pkeyutl -sign -pkeyopt rsa_padding_mode:oaep -pkeyopt rsa_oaep_md:sha256
```
To sign a message without doing message digest, please check [pkey:sign_raw](#pkeysign_raw).
[Back to TOC](#table-of-contents)
### pkey:verify
@ -1262,6 +1287,8 @@ ngx.say(ngx.encode_base64(signature))
```
To verify a message without doing message digest, please check [pkey:verify_raw](#pkeyverify_raw) and [pkey:verify_recover](#pkeyverify_recover).
[Back to TOC](#table-of-contents)
### pkey:encrypt
@ -1325,6 +1352,25 @@ for an example.
[Back to TOC](#table-of-contents)
### pkey:verify_raw
**syntax**: *ok, err = pk:verify_raw(signature, data, md_alg, padding?, opts?)*
Verify the cipher text `signature` with the message `data` with pkey instance, which must loaded a public key. Set the message digest to `md_alg` but doesn't do message digest
automatically, in other words, this function assumes `data` has already been hashed with `md_alg`.
When `md_alg` is undefined, for RSA and EC keys, this function does SHA256 by default. For Ed25519 or Ed448 keys, no default value is set.
The optinal fourth argument `padding` has same meaning as in [pkey:sign](#pkeysign).
If omitted, `padding` is default to `pkey.PADDINGS.RSA_PKCS1_PADDING`.
The fifth optional argument `opts` has same meaning as in [pkey:sign](#pkeysign).
See [examples/raw-sign-and-recover.lua](https://github.com/fffonion/lua-resty-openssl/blob/master/examples/raw-sign-and-recover.lua)
for an example.
[Back to TOC](#table-of-contents)
### pkey:verify_recover
**syntax**: *txt, err = pk:verify_recover(signature, padding?, opts?)*

24
src/deps/src/lua-resty-openssl/examples/raw-sign-and-recover.lua vendored
View file

@ -7,11 +7,29 @@ local original = "original text"
-- same as nodejs: crypto.privateEncrypt
-- php: openssl_private_encrypt
local digested = assert(priv:sign_raw(original))
local signed = assert(priv:sign_raw(original))
print("Digested message: " .. ngx.encode_base64(digested))
print("Signed message: " .. ngx.encode_base64(signed))
-- same as nodejs: crypto.publicDecrypt
-- php: openssl_public_decrypt
local recovered = assert(pub:verify_recover(digested))
local recovered = assert(pub:verify_recover(signed))
print("Recovered message: " .. recovered)
local priv = assert(pkey.new({
type = "EC",
}))
local pub = assert(pkey.new(priv:to_PEM("public")))
local md_alg = "sha512"
local hashed = require "resty.openssl.digest".new(md_alg):final(original)
local signed = assert(priv:sign_raw(hashed))
print("Signed message: " .. ngx.encode_base64(signed))
-- same as nodejs: crypto.publicDecrypt
-- php: openssl_public_decrypt
local verified = assert(pub:verify_raw(signed, hashed, md_alg))
print("Verification result: ", verified)

2
src/deps/src/lua-resty-openssl/lib/resty/openssl.lua vendored
View file

@ -24,7 +24,7 @@ try_require_modules()
local _M = {
_VERSION = '1.3.1',
_VERSION = '1.4.0',
}
function _M.load_modules()

1
src/deps/src/lua-resty-openssl/lib/resty/openssl/auxiliary/jwk.lua vendored
View file

@ -177,6 +177,7 @@ function _M.load_jwk(txt)
if key ~= nil then
return key
end
key_free = function() end
else
return nil, "not yet supported jwk type \"" .. (tbl["kty"] or "nil") .. "\""
end

6
src/deps/src/lua-resty-openssl/lib/resty/openssl/auxiliary/nginx.lua vendored
View file

@ -125,7 +125,8 @@ else
} ngx_connection_s;
]]
else
error("resty.openssl.auxiliary.nginx doesn't support Nginx version " .. ngx_version, 2)
error("resty.openssl.auxiliary.nginx development mode doesn't support Nginx version " .. ngx_version ..
", please compile nginx with lua-resty-openssl-aux-module or lua-kong-nginx-module.", 2)
end
ffi.cdef [[
@ -167,7 +168,8 @@ else
local NO_C_MODULE_WARNING_MSG_SHOWN = false
local NO_C_MODULE_WARNING_MSG = "note resty.openssl.auxiliary.nginx is using plain FFI " ..
"and it's only intended to be used in development, " ..
"consider using lua-resty-openssl.aux-module in production."
"consider using lua-resty-openssl-aux-module or " ..
"lua-kong-nginx-module in production."
local function get_ngx_ssl_from_req()
if not NO_C_MODULE_WARNING_MSG_SHOWN then

2
src/deps/src/lua-resty-openssl/lib/resty/openssl/ec.lua vendored
View file

@ -158,6 +158,8 @@ function _M.set_parameters(ec_key_st, opts)
end
end
return true
end
return _M

1
src/deps/src/lua-resty-openssl/lib/resty/openssl/include/ec.lua vendored
View file

@ -34,6 +34,7 @@ ffi.cdef [[
BIGNUM *x, BIGNUM *y, BN_CTX *ctx);
EC_POINT *EC_POINT_bn2point(const EC_GROUP *group, const BIGNUM *bn,
EC_POINT *p, BN_CTX *ctx);
void EC_POINT_free(EC_POINT *point);
point_conversion_form_t EC_KEY_get_conv_form(const EC_KEY *key);

4
src/deps/src/lua-resty-openssl/lib/resty/openssl/include/evp/pkey.lua vendored
View file

@ -41,6 +41,10 @@ ffi.cdef [[
int EVP_PKEY_sign(EVP_PKEY_CTX *ctx,
unsigned char *sig, size_t *siglen,
const unsigned char *tbs, size_t tbslen);
int EVP_PKEY_verify_init(EVP_PKEY_CTX *ctx);
int EVP_PKEY_verify(EVP_PKEY_CTX *ctx,
const unsigned char *sig, size_t siglen,
const unsigned char *tbs, size_t tbslen);
int EVP_PKEY_verify_recover_init(EVP_PKEY_CTX *ctx);
int EVP_PKEY_verify_recover(EVP_PKEY_CTX *ctx,
unsigned char *rout, size_t *routlen,

110
src/deps/src/lua-resty-openssl/lib/resty/openssl/pkey.lua vendored
View file

@ -402,6 +402,67 @@ local function generate_key(config)
return ctx_ptr[0]
end
local function compose_key(config)
local typ = config.type or 'RSA'
local key_type
if typ == "RSA" then
key_type = evp_macro.EVP_PKEY_RSA
elseif typ == "EC" then
key_type = evp_macro.EVP_PKEY_EC
elseif evp_macro.ecx_curves[typ] then
key_type = evp_macro.ecx_curves[typ]
else
return nil, "unsupported type " .. typ
end
if key_type == 0 then
return nil, "the linked OpenSSL library doesn't support " .. typ .. " key"
end
local key, err, key_free, _
if key_type == evp_macro.EVP_PKEY_EC then
key = C.EC_KEY_new()
if key == nil then
return nil, "EC_KEY_new failed"
end
key_free = C.EC_KEY_free
_, err = ec_lib.set_parameters(key, config.params)
elseif key_type == evp_macro.EVP_PKEY_RSA then
key = C.RSA_new()
if key == nil then
return nil, "RSA_new failed"
end
key_free = C.RSA_free
_, err = rsa_lib.set_parameters(key, config.params)
elseif key_type == evp_macro.EVP_PKEY_ED25519 or
key_type == evp_macro.EVP_PKEY_X25519 or
key_type == evp_macro.EVP_PKEY_ED448 or
key_type == evp_macro.EVP_PKEY_X448 then
key_free = function() end
key, err = ecx_lib.set_parameters(key_type, nil, config.params)
end
if err then
return nil, "failed to construct " .. typ.. " key from parameters: " .. err
end
local ctx = C.EVP_PKEY_new()
if ctx == nil then
key_free(key)
return nil, "EVP_PKEY_new() failed"
end
local code = C.EVP_PKEY_assign(ctx, key_type, key)
if code ~= 1 then
key_free(key)
C.EVP_PKEY_free(ctx)
return nil, "EVP_PKEY_assign() failed"
end
return ctx
end
local load_key_try_funcs = {} do
-- TODO: pkcs1 load functions are not required in openssl 3.0
local _load_key_try_funcs = {
@ -506,9 +567,13 @@ function _M.new(s, opts)
local ctx, err
s = s or {}
if type(s) == 'table' then
ctx, err = generate_key(s)
if s.params then
ctx, err = compose_key(s)
else
ctx, err = generate_key(s)
end
if err then
err = "pkey.new:generate_key: " .. err
err = "pkey.new:new_key: " .. err
end
elseif type(s) == 'string' then
ctx, err = load_pem_der(s, opts or empty_table, load_key_try_funcs)
@ -656,8 +721,9 @@ end
local ASYMMETRIC_OP_ENCRYPT = 0x1
local ASYMMETRIC_OP_DECRYPT = 0x2
local ASYMMETRIC_OP_SIGN_RAW = 0x4
local ASYMMETRIC_OP_VERIFY_RECOVER = 0x8
local ASYMMETRIC_OP_SIGN_RAW = 0x3
local ASYMMETRIC_OP_VERIFY_RAW = 0x4
local ASYMMETRIC_OP_VERIFY_RECOVER = 0x5
local function asymmetric_routine(self, s, op, padding, opts)
if type(s) ~= "string" then
@ -701,6 +767,10 @@ local function asymmetric_routine(self, s, op, padding, opts)
fint = C.EVP_PKEY_sign_init
f = C.EVP_PKEY_sign
op_name = "sign"
elseif op == ASYMMETRIC_OP_VERIFY_RAW then
fint = C.EVP_PKEY_verify_init
f = C.EVP_PKEY_verify
op_name = "verify"
elseif op == ASYMMETRIC_OP_VERIFY_RECOVER then
fint = C.EVP_PKEY_verify_recover_init
f = C.EVP_PKEY_verify_recover
@ -725,13 +795,25 @@ local function asymmetric_routine(self, s, op, padding, opts)
return nil, "pkey:asymmetric_routine: " .. err
end
local length = ptr_of_size_t(self.buf_size)
if f(pkey_ctx, self.buf, length, s, #s) <= 0 then
return nil, format_error("pkey:asymmetric_routine EVP_PKEY_" .. op_name)
local buf, buf_len
if opts and opts.buf_in then
buf = opts.buf_in
buf_len = #buf
else
buf = self.buf
buf_len = ptr_of_size_t(self.buf_size)
end
return ffi_str(self.buf, length[0]), nil
code = f(pkey_ctx, buf, buf_len, s, #s)
if code <= 0 then
return nil, format_error("pkey:asymmetric_routine EVP_PKEY_" .. op_name, code)
end
if not opts or not opts.buf_in then
return ffi_str(self.buf, buf_len[0]), nil
end
return true
end
_M.PADDINGS = rsa_macro.paddings
@ -753,6 +835,16 @@ function _M:sign_raw(s, padding, opts)
return asymmetric_routine(self, s, ASYMMETRIC_OP_SIGN_RAW, padding, opts)
end
function _M:verify_raw(signature, hashed_message, md_alg, padding, opts)
opts = opts or {}
opts.buf_in = signature
if md_alg then
table.insert(opts, "digest:" .. md_alg)
end
return asymmetric_routine(self, hashed_message, ASYMMETRIC_OP_VERIFY_RAW, padding, opts)
end
function _M:verify_recover(s, padding, opts)
return asymmetric_routine(self, s, ASYMMETRIC_OP_VERIFY_RECOVER, padding, opts)
end

4
src/deps/src/lua-resty-openssl/lua-resty-openssl-1.3.1-1.rockspec → src/deps/src/lua-resty-openssl/lua-resty-openssl-1.4.0-1.rockspec vendored
View file

@ -1,8 +1,8 @@
package = "lua-resty-openssl"
version = "1.3.1-1"
version = "1.4.0-1"
source = {
url = "git+https://github.com/fffonion/lua-resty-openssl.git",
tag = "1.3.1"
tag = "1.4.0"
}
description = {
detailed = "FFI-based OpenSSL binding for LuaJIT.",

2
src/deps/src/lua-resty-openssl/scripts/requirements.txt vendored
View file

@ -1 +1 @@
jinja2==3.1.3
jinja2==3.1.4

13
src/scheduler/JobScheduler.py
View file

@ -10,7 +10,7 @@ from os import cpu_count, environ, getenv, sep
from os.path import basename, dirname, join
from pathlib import Path
from re import match
from typing import Any, Dict, Optional
from typing import Any, Dict, List, Optional
from schedule import (
Job,
clear as schedule_clear,
@ -274,7 +274,7 @@ class JobScheduler(ApiCaller):
return success
def run_once(self) -> bool:
def run_once(self, plugins: Optional[List[str]] = None) -> bool:
err = self.try_database_readonly()
if err:
return True
@ -283,7 +283,12 @@ class JobScheduler(ApiCaller):
self.__job_success = True
self.__job_reload = False
plugins = plugins or []
for plugin, jobs in self.__jobs.items():
if plugins and plugin not in plugins:
continue
# Add job to the list of jobs to run in the order they are defined
jobs_jobs = [partial(self.__job_wrapper, job["path"], plugin, job["name"], job["file"]) for job in jobs]
@ -337,14 +342,14 @@ class JobScheduler(ApiCaller):
def clear(self):
schedule_clear()
def reload(self, env: Dict[str, Any], apis: Optional[list] = None) -> bool:
def reload(self, env: Dict[str, Any], apis: Optional[list] = None, *, changed_plugins: Optional[List[str]] = None) -> bool:
ret = True
try:
self.__env = env
super().__init__(apis or self.apis)
self.clear()
self.__jobs = self.__get_jobs()
ret = self.run_once()
ret = self.run_once(changed_plugins)
self.setup()
except:
self.__logger.error(f"Exception while reloading scheduler {format_exc()}")

13
src/scheduler/main.py
View file

@ -658,12 +658,14 @@ if __name__ == "__main__":
if INTEGRATION == "Docker" and not override_instances:
Thread(target=listen_for_instances_reload, name="listen_for_instances_reload").start()
changed_plugins = []
while True:
threads.clear()
if RUN_JOBS_ONCE:
# Only run jobs once
if not SCHEDULER.reload(env | environ):
if not SCHEDULER.reload(env | environ, changed_plugins=changed_plugins):
logger.error("At least one job in run_once() failed")
else:
logger.info("All jobs in run_once() were successful")
@ -748,6 +750,9 @@ if __name__ == "__main__":
ret = SCHEDULER.db.checked_changes(CHANGES)
if ret:
logger.error(f"An error occurred when setting the changes to checked in the database : {ret}")
ret = SCHEDULER.db.checked_plugins_changes(changed_plugins)
if ret:
logger.error(f"An error occurred when setting the plugins changes to checked in the database : {ret}")
except BaseException as e:
logger.error(f"Error while setting changes to checked in the database: {e}")
@ -758,6 +763,7 @@ if __name__ == "__main__":
PLUGINS_NEED_GENERATION = False
PRO_PLUGINS_NEED_GENERATION = False
INSTANCES_NEED_GENERATION = False
changed_plugins.clear()
if scheduler_first_start:
try:
@ -818,11 +824,12 @@ if __name__ == "__main__":
NEED_RELOAD = True
# check if the config have changed since last time
if changes["config_changed"]:
logger.info("Config changed, generating ...")
if changes["plugins_config_changed"]:
logger.info("Plugins config changed, generating ...")
CONFIG_NEED_GENERATION = True
RUN_JOBS_ONCE = True
NEED_RELOAD = True
changed_plugins = changes["plugins_config_changed"]
# check if the instances have changed since last time
if changes["instances_changed"]: