From 6af3b985a05b7b23f9e5427f55bee98f2ae43795 Mon Sep 17 00:00:00 2001 From: bunkerity Date: Tue, 4 Apr 2023 15:53:03 +0200 Subject: [PATCH] fix deadlock in autoconf/swarm and fix missing favicon in default and loading pages --- .github/workflows/staging.yml | 78 ++++++++++++------------- src/autoconf/SwarmController.py | 64 ++++++++++++-------- src/bw/loading/index.html | 2 +- src/common/core/misc/files/default.html | 6 +- 4 files changed, 82 insertions(+), 68 deletions(-) diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index 0bc75bdd6..eab4cc471 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -138,20 +138,20 @@ jobs: # TYPE: autoconf # secrets: # CICD_SECRETS: ${{ secrets.CICD_SECRETS }} - # create-infra-swarm: - # needs: [code-security, build-bw, build-scheduler, build-autoconf] - # uses: ./.github/workflows/staging-create-infra.yml - # with: - # TYPE: swarm - # secrets: - # CICD_SECRETS: ${{ secrets.CICD_SECRETS }} - create-infra-k8s: + create-infra-swarm: needs: [code-security, build-bw, build-scheduler, build-autoconf] uses: ./.github/workflows/staging-create-infra.yml with: - TYPE: k8s + TYPE: swarm secrets: CICD_SECRETS: ${{ secrets.CICD_SECRETS }} + # create-infra-k8s: + # needs: [code-security, build-bw, build-scheduler, build-autoconf] + # uses: ./.github/workflows/staging-create-infra.yml + # with: + # TYPE: k8s + # secrets: + # CICD_SECRETS: ${{ secrets.CICD_SECRETS }} # create-infra-linux: # needs: # [ @@ -184,20 +184,20 @@ jobs: # TYPE: autoconf # RUNS_ON: "['self-hosted', 'bw-autoconf']" # secrets: inherit - # tests-swarm: - # needs: [create-infra-swarm] - # uses: ./.github/workflows/staging-tests.yml - # with: - # TYPE: swarm - # RUNS_ON: "['self-hosted', 'bw-swarm']" - # secrets: inherit - tests-k8s: - needs: [create-infra-k8s] + tests-swarm: + needs: [create-infra-swarm] uses: ./.github/workflows/staging-tests.yml with: - TYPE: k8s - RUNS_ON: "['ubuntu-latest']" + TYPE: swarm + RUNS_ON: "['self-hosted', 'bw-swarm']" secrets: inherit + # tests-k8s: + # needs: [create-infra-k8s] + # uses: ./.github/workflows/staging-tests.yml + # with: + # TYPE: k8s + # RUNS_ON: "['ubuntu-latest']" + # secrets: inherit # tests-linux: # needs: [create-infra-linux] # uses: ./.github/workflows/staging-tests.yml @@ -205,14 +205,14 @@ jobs: # TYPE: linux # RUNS_ON: "['self-hosted', 'bw-linux']" # secrets: inherit - tests-ui: - needs: [code-security, build-ui] - uses: ./.github/workflows/staging-tests-ui.yml - with: - MODE: staging - secrets: - PRIVATE_REGISTRY: ${{ secrets.PRIVATE_REGISTRY }} - PRIVATE_REGISTRY_TOKEN: ${{ secrets.PRIVATE_REGISTRY_TOKEN }} + # tests-ui: + # needs: [code-security, build-ui] + # uses: ./.github/workflows/staging-tests-ui.yml + # with: + # MODE: staging + # secrets: + # PRIVATE_REGISTRY: ${{ secrets.PRIVATE_REGISTRY }} + # PRIVATE_REGISTRY_TOKEN: ${{ secrets.PRIVATE_REGISTRY_TOKEN }} # Delete infrastructures # delete-infra-docker: @@ -231,22 +231,22 @@ jobs: # TYPE: autoconf # secrets: # CICD_SECRETS: ${{ secrets.CICD_SECRETS }} - # delete-infra-swarm: - # if: ${{ always() }} - # needs: [tests-swarm] - # uses: ./.github/workflows/staging-delete-infra.yml - # with: - # TYPE: swarm - # secrets: - # CICD_SECRETS: ${{ secrets.CICD_SECRETS }} - delete-infra-k8s: + delete-infra-swarm: if: ${{ always() }} - needs: [tests-k8s] + needs: [tests-swarm] uses: ./.github/workflows/staging-delete-infra.yml with: - TYPE: k8s + TYPE: swarm secrets: CICD_SECRETS: ${{ secrets.CICD_SECRETS }} + # delete-infra-k8s: + # if: ${{ always() }} + # needs: [tests-k8s] + # uses: ./.github/workflows/staging-delete-infra.yml + # with: + # TYPE: k8s + # secrets: + # CICD_SECRETS: ${{ secrets.CICD_SECRETS }} # delete-infra-linux: # if: ${{ always() }} # needs: [tests-linux] diff --git a/src/autoconf/SwarmController.py b/src/autoconf/SwarmController.py index 4f66ce8e2..39394976a 100644 --- a/src/autoconf/SwarmController.py +++ b/src/autoconf/SwarmController.py @@ -125,31 +125,49 @@ class SwarmController(Controller, ConfigCaller): ) def __event(self, event_type): - for _ in self.__client.events(decode=True, filters={"type": event_type}): - self.__internal_lock.acquire() - try: - self._instances = self.get_instances() - self._services = self.get_services() - self._configs = self.get_configs() - if not self._config.update_needed( - self._instances, self._services, configs=self._configs - ): - continue - self.__logger.info( - "Catched Swarm event, deploying new configuration ..." - ) - if not self.apply_config(): - self.__logger.error("Error while deploying new configuration") - else: - self.__logger.info( - "Successfully deployed new configuration 🚀", - ) - - except: + while True : + locked = False + error = False + try : + for _ in self.__client.events(decode=True, filters={"type": event_type}): + self.__internal_lock.acquire() + locked = True + try: + self._instances = self.get_instances() + self._services = self.get_services() + self._configs = self.get_configs() + if not self._config.update_needed( + self._instances, self._services, configs=self._configs + ): + self.__internal_lock.release() + locked = False + self.__logger.info( + f"Catched Swarm event ({event_type}), deploying new configuration ..." + ) + if not self.apply_config(): + self.__logger.error("Error while deploying new configuration") + else: + self.__logger.info( + "Successfully deployed new configuration 🚀", + ) + except: + self.__logger.error( + f"Exception while processing Swarm event ({event_type}) :\n{format_exc()}" + ) + self.__internal_lock.release() + locked = False + except : self.__logger.error( - f"Exception while processing events :\n{format_exc()}" + f"Exception while reading Swarm event ({event_type}) :\n{format_exc()}", ) - self.__internal_lock.release() + error = True + finally: + if locked: + self.__internal_lock.release() + locked = False + if error is True: + self.__logger.warning("Got exception, retrying in 10 seconds ...") + sleep(10) def process_events(self): self._set_autoconf_load_db() diff --git a/src/bw/loading/index.html b/src/bw/loading/index.html index 67cf5435c..4ad14674d 100644 --- a/src/bw/loading/index.html +++ b/src/bw/loading/index.html @@ -4,7 +4,7 @@ Generating... - +