Merge branch 'staging' into dev

This commit is contained in:
bunkerity 2023-09-04 14:07:14 +02:00
commit 07725356b6
No known key found for this signature in database
GPG key ID: 93EE47CC3D061500
27 changed files with 163 additions and 146 deletions

View file

@ -146,5 +146,5 @@ jobs:
file: tests/linux/Dockerfile-${{ inputs.LINUX }}
platforms: ${{ inputs.PLATFORMS }}
push: true
tags: ${{ steps.meta.outputs.tags }}
tags: ghcr.io/bunkerity/${{ inputs.LINUX }}-tests:${{ inputs.RELEASE }}
labels: ${{ steps.meta.outputs.labels }}

View file

@ -57,10 +57,10 @@ jobs:
Documentation : https://docs.bunkerweb.io/${{ inputs.VERSION }}/
Docker tags :
- `bunkerity/bunkerweb:${{ inputs.VERSION }}`
- `bunkerity/bunkerweb-scheduler:${{ inputs.VERSION }}`
- `bunkerity/bunkerweb-autoconf:${{ inputs.VERSION }}`
- `bunkerity/bunkerweb-ui:${{ inputs.VERSION }}`
- BunkerWeb : `bunkerity/bunkerweb:${{ inputs.VERSION }}` or `ghcr.io/bunkerity/bunkerweb:${{ inputs.VERSION }}`
- Scheduler : `bunkerity/bunkerweb-scheduler:${{ inputs.VERSION }}` or `ghcr.io/bunkerity/bunkerweb-scheduler:${{ inputs.VERSION }}`
- Autoconf : `bunkerity/bunkerweb-autoconf:${{ inputs.VERSION }}` or `ghcr.io/bunkerity/bunkerweb-autoconf:${{ inputs.VERSION }}`
- UI : `bunkerity/bunkerweb-ui:${{ inputs.VERSION }}` or `ghcr.io/bunkerity/bunkerweb-ui:${{ inputs.VERSION }}`
Linux packages : https://packagecloud.io/app/bunkerity/bunkerweb/search?q=${{ inputs.VERSION }}&filter=all&dist=
@ -83,12 +83,14 @@ jobs:
Documentation : https://docs.bunkerweb.io/${{ inputs.VERSION }}/
Docker tags :
- `bunkerity/bunkerweb:${{ inputs.VERSION }}`
- `bunkerity/bunkerweb-scheduler:${{ inputs.VERSION }}`
- `bunkerity/bunkerweb-autoconf:${{ inputs.VERSION }}`
- `bunkerity/bunkerweb-ui:${{ inputs.VERSION }}`
- BunkerWeb : `bunkerity/bunkerweb:${{ inputs.VERSION }}` or `ghcr.io/bunkerity/bunkerweb:${{ inputs.VERSION }}`
- Scheduler : `bunkerity/bunkerweb-scheduler:${{ inputs.VERSION }}` or `ghcr.io/bunkerity/bunkerweb-scheduler:${{ inputs.VERSION }}`
- Autoconf : `bunkerity/bunkerweb-autoconf:${{ inputs.VERSION }}` or `ghcr.io/bunkerity/bunkerweb-autoconf:${{ inputs.VERSION }}`
- UI : `bunkerity/bunkerweb-ui:${{ inputs.VERSION }}` or `ghcr.io/bunkerity/bunkerweb-ui:${{ inputs.VERSION }}`
Linux packages : https://packagecloud.io/app/bunkerity/bunkerweb/search?q=${{ inputs.VERSION }}&filter=all&dist=
Please note that when using Linux Debian or Ubuntu integration, you will need to add the `force-bad-version` directive to your `/etc/dpkg/dpkg.cfg` file before installing the testing version of BunkerWeb.
draft: false
prerelease: ${{ inputs.PRERELEASE }}
name: Testing

View file

@ -9,6 +9,10 @@ on:
secrets:
CICD_SECRETS:
required: true
SECRET_KEY:
required: true
K8S_IP:
required: true
jobs:
create:
@ -41,12 +45,11 @@ jobs:
- run: ./tests/create.sh ${{ inputs.TYPE }}
env:
CICD_SECRETS: ${{ secrets.CICD_SECRETS }}
REG_USER: ${{ github.actor }}
REG_TOKEN: ${{ secrets.GITHUB_TOKEN }}
K8S_IP: ${{ secrets.K8S_IP }}
- run: |
tar -cvf terraform.tar /tmp/${{ inputs.TYPE }}
tar -cf terraform.tar /tmp/${{ inputs.TYPE }}
echo "$SECRET_KEY" > /tmp/.secret_key
openssl enc -in terraform.tar -aes-256-cbc -pbkdf2 -pass file:/tmp/.secret_key -out terraform.tar.enc
openssl enc -in terraform.tar -aes-256-cbc -pbkdf2 -iter 100000 -md sha256 -pass file:/tmp/.secret_key -out terraform.tar.enc
rm -f /tmp/.secret_key
if: always()
env:

View file

@ -9,27 +9,29 @@ on:
secrets:
CICD_SECRETS:
required: true
SECRET_KEY:
required: true
jobs:
delete:
if: ${{ always() }}
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
# Prepare
- name: Checkout source code
uses: actions/checkout@v3
- name: Install terraform
uses: hashicorp/setup-terraform@v2
- uses: actions/download-artifact@v3
with:
name: tf-${{ inputs.TYPE }}
path: /tmp
- run: |
echo "$SECRET_KEY" > /tmp/.secret_key
openssl dec -in /tmp/terraform.tar.enc -aes-256-cbc -pbkdf2 -pass file:/tmp/.secret_key -out /tmp/terraform.tar
openssl enc -d -in /tmp/terraform.tar.enc -aes-256-cbc -pbkdf2 -iter 100000 -md sha256 -pass file:/tmp/.secret_key -out /tmp/terraform.tar
rm -f /tmp/.secret_key
tar xvf /tmp/terraform.tar -C / && mkdir ~/.ssh && touch ~/.ssh/id_rsa.pub
tar xf /tmp/terraform.tar -C / && mkdir ~/.ssh && touch ~/.ssh/id_rsa.pub
env:
SECRET_KEY: ${{ secrets.SECRET_KEY }}
- uses: azure/setup-kubectl@v3

View file

@ -50,13 +50,21 @@ jobs:
if: inputs.TYPE == 'k8s'
- run: |
echo "$SECRET_KEY" > /tmp/.secret_key
openssl dec -in /tmp/terraform.tar.enc -aes-256-cbc -pbkdf2 -pass file:/tmp/.secret_key -out /tmp/terraform.tar
openssl enc -d -in /tmp/terraform.tar.enc -aes-256-cbc -pbkdf2 -iter 100000 -md sha256 -pass file:/tmp/.secret_key -out /tmp/terraform.tar
rm -f /tmp/.secret_key
tar xvf /tmp/terraform.tar -C /
tar xf /tmp/terraform.tar -C /
mkdir /tmp/reg
cp tests/terraform/k8s-reg.tf /tmp/reg
cp tests/terraform/providers.tf /tmp/reg
cd /tmp/reg
export TF_VAR_k8s_reg_user=${REG_USER}
export TF_VAR_k8s_reg_token=${REG_TOKEN}
terraform init
terraform apply -auto-approve
env:
SECRET_KEY: ${{ secrets.SECRET_KEY }}
if: inputs.TYPE == 'k8s'
- run: tar xvf /tmp/terraform.tar -C /
REG_USER: ${{ github.actor }}
REG_TOKEN: ${{ secrets.GITHUB_TOKEN }}
if: inputs.TYPE == 'k8s'
- uses: azure/setup-kubectl@v3
if: inputs.TYPE == 'k8s'
@ -64,16 +72,16 @@ jobs:
if: inputs.TYPE == 'k8s'
- name: Pull BW linux ubuntu test image
if: inputs.TYPE == 'linux'
run: docker pull ghcr.io/bunkerity/ubuntu-tests:testing && docker tag ghcr.io/bunkerity ubuntu-tests:testing local/ubuntu:latest
run: docker pull ghcr.io/bunkerity/ubuntu-tests:testing && docker tag ghcr.io/bunkerity/ubuntu-tests:testing local/ubuntu:latest
- name: Pull BW linux debian test image
if: inputs.TYPE == 'linux'
run: docker pull ghcr.io/bunkerity/debian-tests:testing && docker tag ghcr.io/bunkerity debian-tests:testing local/debian:latest
run: docker pull ghcr.io/bunkerity/debian-tests:testing && docker tag ghcr.io/bunkerity/debian-tests:testing local/debian:latest
- name: Pull BW linux fedora test image
if: inputs.TYPE == 'linux'
run: docker pull ghcr.io/bunkerity/fedora-tests:testing && docker tag ghcr.io/bunkerity fedora-tests:testing local/fedora:latest
run: docker pull ghcr.io/bunkerity/fedora-tests:testing && docker tag ghcr.io/bunkerity/fedora-tests:testing local/fedora:latest
- name: Pull BW linux rhel test image
if: inputs.TYPE == 'linux'
run: docker pull ghcr.io/bunkerity/rhel-tests:testing && docker tag ghcr.io/bunkerity rhel-tests:testing local/rhel:latest
run: docker pull ghcr.io/bunkerity/rhel-tests:testing && docker tag ghcr.io/bunkerity/rhel-tests:testing local/rhel:latest
# Do tests
- name: Run tests
if: inputs.TYPE == 'docker'

View file

@ -100,6 +100,8 @@ jobs:
TYPE: ${{ matrix.type }}
secrets:
CICD_SECRETS: ${{ secrets.CICD_SECRETS }}
SECRET_KEY: ${{ secrets.SECRET_KEY }}
K8S_IP: ${{ secrets.K8S_IP }}
# Perform tests
staging-tests:
@ -163,6 +165,7 @@ jobs:
TYPE: ${{ matrix.type }}
secrets:
CICD_SECRETS: ${{ secrets.CICD_SECRETS }}
SECRET_KEY: ${{ secrets.SECRET_KEY }}
# Push Docker images
push-images:

View file

@ -975,6 +975,9 @@ To simplify the installation process, Linux package repositories for BunkerWeb a
sudo apt install -y nginx=1.24.0-1~$(lsb_release -cs)
```
!!! warning "Testing version"
If you use the `testing` version, you will need to add the `force-bad-version` directive to your `/etc/dpkg/dpkg.cfg` file before installing BunkerWeb.
And finally install BunkerWeb 1.5.1 :
```shell
@ -1009,6 +1012,9 @@ To simplify the installation process, Linux package repositories for BunkerWeb a
sudo apt install -y nginx=1.24.0-1~jammy
```
!!! warning "Testing version"
If you use the `testing` version, you will need to add the `force-bad-version` directive to your `/etc/dpkg/dpkg.cfg` file before installing BunkerWeb.
And finally install BunkerWeb 1.5.1 :
```shell

View file

@ -1,10 +0,0 @@
#!/bin/bash
if [ $(id -u) -ne 0 ] ; then
echo "❌ Run me as root"
exit 1
fi
mkdir /var/www/html/{app1.example.com,app2.example.com}
echo "hello" > /var/www/html/app1.example.com/index.html
echo "hello" > /var/www/html/app2.example.com/index.html

View file

@ -1,6 +1,6 @@
{
"name": "reverse-proxy-multisite",
"kinds": ["docker", "autoconf", "swarm", "kubernetes", "linux"],
"kinds": ["docker", "autoconf", "swarm", "kubernetes"],
"timeout": 60,
"delay": 90,
"tests": [

View file

@ -1,14 +0,0 @@
HTTP_PORT=80
HTTPS_PORT=443
DNS_RESOLVERS=8.8.8.8 8.8.4.4
# Replace with your domain
SERVER_NAME=app1.example.com app2.example.com
MULTISITE=yes
DISABLE_DEFAULT_SERVER=yes
AUTO_LETS_ENCRYPT=yes
USE_CLIENT_CACHE=yes
USE_GZIP=yes
app1.example.com_LOCAL_PHP=/run/php/php-fpm.sock
app1.example.com_LOCAL_PHP_PATH=/var/www/html/app1.example.com
app2.example.com_LOCAL_PHP=/run/php/php-fpm.sock
app2.example.com_LOCAL_PHP_PATH=/var/www/html/app2.example.com

View file

@ -1,12 +0,0 @@
#!/bin/bash
if [ $(id -u) -ne 0 ] ; then
echo "❌ Run me as root"
exit 1
fi
mkdir /var/www/html/{app1,app2}
echo "hello" > /var/www/html/app1/index.html
echo "hello" > /var/www/html/app2/index.html
cp -r bw-data/configs/* /etc/bunkerweb/configs
chown -R nginx:nginx /etc/bunkerweb/configs

View file

@ -1,6 +1,6 @@
{
"name": "reverse-proxy-singlesite",
"kinds": ["docker", "autoconf", "swarm", "linux"],
"kinds": ["docker", "autoconf", "swarm"],
"delay": 120,
"timeout": 60,
"no_copy_container": true,

View file

@ -1,11 +0,0 @@
HTTP_PORT=80
HTTPS_PORT=443
DNS_RESOLVERS=8.8.8.8 8.8.4.4
# Replace with your domain
SERVER_NAME=www.example.com
DISABLE_DEFAULT_SERVER=yes
AUTO_LETS_ENCRYPT=yes
USE_CLIENT_CACHE=yes
USE_GZIP=yes
LOCAL_PHP=/run/php/php-fpm.sock
LOCAL_PHP_PATH=/var/www/html

View file

@ -63,8 +63,7 @@ class Config(ConfigCaller):
"instances": False,
"services": False,
"configs": False,
"config": False
"config": False,
}
changes = []
if instances != self.__instances or first:
@ -79,9 +78,10 @@ class Config(ConfigCaller):
updates["configs"] = True
changes.append("custom_configs")
if updates["instances"] or updates["services"]:
old_env = self.__get_full_env()
self.__config = self.__get_full_env()
if self.__config != old_env or first:
old_env = deepcopy(self.__config)
new_env = self.__get_full_env()
if old_env != new_env or first:
self.__config = new_env
updates["config"] = True
changes.append("config")
@ -112,6 +112,20 @@ class Config(ConfigCaller):
)
sleep(5)
# wait until changes are applied
while True:
curr_changes = self._db.check_changes()
if isinstance(curr_changes, str):
self.__logger.error(
f"An error occurred when checking for changes in the database : {curr_changes}"
)
elif not any(curr_changes.values()):
break
else:
self.__logger.warning(
"Scheduler is already applying a configuration, retrying in 5 seconds ...",
)
sleep(5)
# update instances in database
if updates["instances"]:
err = self._db.update_instances(self.__instances, changed=False)
@ -127,7 +141,9 @@ class Config(ConfigCaller):
)
# save custom configs to database
if updates["configs"]:
err = self._db.save_custom_configs(custom_configs, "autoconf", changed=False)
err = self._db.save_custom_configs(
custom_configs, "autoconf", changed=False
)
if err:
success = False
self.__logger.error(

View file

@ -106,7 +106,12 @@ class DockerController(Controller):
return configs
def apply_config(self) -> bool:
return self.apply(self._instances, self._services, configs=self._configs, first=not self._loaded)
return self.apply(
self._instances,
self._services,
configs=self._configs,
first=not self._loaded,
)
def process_events(self):
self._set_autoconf_load_db()

View file

@ -298,7 +298,12 @@ class IngressController(Controller):
sleep(10)
def apply_config(self) -> bool:
return self.apply(self._instances, self._services, configs=self._configs, first=not self._loaded)
return self.apply(
self._instances,
self._services,
configs=self._configs,
first=not self._loaded,
)
def process_events(self):
self._set_autoconf_load_db()

View file

@ -127,7 +127,12 @@ class SwarmController(Controller):
return configs
def apply_config(self) -> bool:
return self.apply(self._instances, self._services, configs=self._configs, first=not self._loaded)
return self.apply(
self._instances,
self._services,
configs=self._configs,
first=not self._loaded,
)
def __event(self, event_type):
while True:

View file

@ -329,7 +329,9 @@ class Database:
except BaseException:
return format_exc()
def checked_changes(self, changes: Optional[List[str]] = None, value: Optional[bool] = False) -> str:
def checked_changes(
self, changes: Optional[List[str]] = None, value: Optional[bool] = False
) -> str:
"""Set changed bit for config, custom configs, instances and plugins"""
changes = changes or [
"config",
@ -470,7 +472,9 @@ class Database:
return True, ""
def save_config(self, config: Dict[str, Any], method: str, changed: Optional[bool] = True) -> str:
def save_config(
self, config: Dict[str, Any], method: str, changed: Optional[bool] = True
) -> str:
"""Save the config in the database"""
to_put = []
with self.__db_session() as session:
@ -733,7 +737,10 @@ class Database:
return ""
def save_custom_configs(
self, custom_configs: List[Dict[str, Tuple[str, List[str]]]], method: str, changed: Optional[bool] = True
self,
custom_configs: List[Dict[str, Tuple[str, List[str]]]],
method: str,
changed: Optional[bool] = True,
) -> str:
"""Save the custom configs in the database"""
message = ""
@ -1750,7 +1757,9 @@ class Database:
return ""
def update_instances(self, instances: List[Dict[str, Any]], changed: Optional[bool] = True) -> str:
def update_instances(
self, instances: List[Dict[str, Any]], changed: Optional[bool] = True
) -> str:
"""Update instances."""
to_put = []
with self.__db_session() as session:

View file

@ -511,14 +511,15 @@ if __name__ == "__main__":
logger.info(f"Successfully sent {CACHE_PATH} folder")
while True:
threads.clear()
ret = db.checked_changes(CHANGES)
if not FIRST_RUN:
threads.clear()
ret = db.checked_changes(CHANGES)
if ret:
logger.error(
f"An error occurred when setting the changes to checked in the database : {ret}"
)
stop(1)
if ret:
logger.error(
f"An error occurred when setting the changes to checked in the database : {ret}"
)
stop(1)
# Update the environment variables of the scheduler
SCHEDULER.env = env.copy() | environ.copy()

View file

@ -62,17 +62,17 @@ class KubernetesTest(Test):
Test.replace_in_file(
deploy,
r"bunkerity/bunkerweb:.*$",
f"{getenv('PRIVATE_REGISTRY')}/infra/bunkerweb-tests:{getenv('IMAGE_TAG')}",
f"ghcr.io/bunkerity/bunkerweb-tests:{getenv('IMAGE_TAG')}",
)
Test.replace_in_file(
deploy,
r"bunkerity/bunkerweb-autoconf:.*$",
f"{getenv('PRIVATE_REGISTRY')}/infra/autoconf-tests:{getenv('IMAGE_TAG')}",
f"ghcr.io/bunkerity/autoconf-tests:{getenv('IMAGE_TAG')}",
)
Test.replace_in_file(
deploy,
r"bunkerity/bunkerweb-scheduler:.*$",
f"{getenv('PRIVATE_REGISTRY')}/infra/scheduler-tests:{getenv('IMAGE_TAG')}",
f"ghcr.io/bunkerity/scheduler-tests:{getenv('IMAGE_TAG')}",
)
proc = run(
"kubectl apply -f bunkerweb.yml", cwd="/tmp/kubernetes", shell=True

View file

@ -2,8 +2,7 @@
# drop and export secrets
echo "${CICD_SECRETS}" > /opt/.env
echo "export TF_VAR_k8s_reg_user=${REG_USER}" >> /opt/.env
echo "export TF_VAR_k8s_reg_token=${REG_TOKEN}" >> /opt/.env
echo "export TF_VAR_k8s_ip=${K8S_IP}" >> /opt/.env
chmod +x /opt/.env
. /opt/.env

View file

@ -26,8 +26,8 @@ resource "scaleway_instance_server" "instance" {
}
# Create Ansible inventory file
resource "local_file" "ansible_inventory" {
sensitive_content = templatefile("templates/autoconf_inventory.tftpl", {
resource "local_sensitive_file" "ansible_inventory" {
content = templatefile("templates/autoconf_inventory.tftpl", {
public_ip = var.autoconf_ip
})
filename = "/tmp/autoconf_inventory"

View file

@ -26,8 +26,8 @@ resource "scaleway_instance_server" "instance" {
}
# Create Ansible inventory file
resource "local_file" "ansible_inventory" {
sensitive_content = templatefile("templates/docker_inventory.tftpl", {
resource "local_sensitive_file" "ansible_inventory" {
content = templatefile("templates/docker_inventory.tftpl", {
public_ip = var.docker_ip
})
filename = "/tmp/docker_inventory"

View file

@ -0,0 +1,32 @@
variable "k8s_reg_user" {
type = string
nullable = false
sensitive = true
}
variable "k8s_reg_token" {
type = string
nullable = false
sensitive = true
}
# Setup registry
provider "kubernetes" {
config_path = "/tmp/k8s/kubeconfig"
}
resource "kubernetes_secret" "reg" {
metadata {
name = "secret-registry"
}
type = "kubernetes.io/dockerconfigjson"
data = {
".dockerconfigjson" = jsonencode({
auths = {
"ghcr.io" = {
"username" = var.k8s_reg_user
"password" = var.k8s_reg_token
"auth" = base64encode("${var.k8s_reg_user}:${var.k8s_reg_token}")
}
}
})
}
}

View file

@ -4,16 +4,6 @@ variable "k8s_ip" {
nullable = false
sensitive = true
}
variable "k8s_reg_user" {
type = string
nullable = false
sensitive = true
}
variable "k8s_reg_token" {
type = string
nullable = false
sensitive = true
}
# Create k8s cluster
resource "scaleway_k8s_cluster" "cluster" {
@ -33,46 +23,24 @@ resource "scaleway_k8s_pool" "pool" {
}
# Get kubeconfig file
resource "local_file" "kubeconfig" {
resource "local_sensitive_file" "kubeconfig" {
depends_on = [scaleway_k8s_pool.pool]
sensitive_content = scaleway_k8s_cluster.cluster.kubeconfig[0].config_file
content = scaleway_k8s_cluster.cluster.kubeconfig[0].config_file
filename = "/tmp/k8s/kubeconfig"
}
provider "kubectl" {
config_path = "${local_file.kubeconfig.filename}"
config_path = "${local_sensitive_file.kubeconfig.filename}"
}
# Setup LB
resource "local_file" "lb_yml" {
depends_on = [local_file.kubeconfig]
sensitive_content = templatefile("templates/lb.yml.tftpl", {
resource "local_sensitive_file" "lb_yml" {
depends_on = [local_sensitive_file.kubeconfig]
content = templatefile("templates/lb.yml.tftpl", {
lb_ip = var.k8s_ip
})
filename = "/tmp/k8s/lb.yml"
}
resource "kubectl_manifest" "lb" {
depends_on = [local_file.lb_yml]
yaml_body = local_file.lb_yml.content
}
# Setup registry
provider "kubernetes" {
config_path = "${local_file.kubeconfig.filename}"
}
resource "kubernetes_secret" "reg" {
metadata = {
name = "secret-registry"
}
type = "kubernetes.io/dockerconfigjson"
data = {
".dockerconfigjson" = jsonencode({
auths = {
"ghcr.io" = {
"username" = var.k8s_reg_user
"password" = var.k8s_reg_token
"auth" = base64encode("${var.k8s_reg_user}:${var.k8s_reg_token}")
}
}
})
}
depends_on = [local_sensitive_file.lb_yml]
yaml_body = local_sensitive_file.lb_yml.content
}

View file

@ -26,8 +26,8 @@ resource "scaleway_instance_server" "instance" {
}
# Create Ansible inventory file
resource "local_file" "ansible_inventory" {
sensitive_content = templatefile("templates/linux_inventory.tftpl", {
resource "local_sensitive_file" "ansible_inventory" {
content = templatefile("templates/linux_inventory.tftpl", {
public_ip = var.linux_ip
})
filename = "/tmp/linux_inventory"

View file

@ -35,8 +35,8 @@ resource "scaleway_instance_server" "instances" {
}
# Create Ansible inventory file
resource "local_file" "ansible_inventory" {
sensitive_content = templatefile("templates/swarm_inventory.tftpl", {
resource "local_sensitive_file" "ansible_inventory" {
content = templatefile("templates/swarm_inventory.tftpl", {
public_ips = var.swarm_ips
})
filename = "/tmp/swarm_inventory"