Enhance instance status handling by adding 'failover' state and updating related logic in the database and UI templates

src/common/db/alembic/mariadb_versions/8a37fed772e9_upgrade_to_version_1_6_0_rc1.py

@@ -48,8 +48,14 @@ def upgrade() -> None:
     with op.batch_alter_table("bw_services_settings") as batch_op:
         batch_op.create_foreign_key("bw_services_settings_ibfk_1", "bw_services", ["service_id"], ["id"], onupdate="CASCADE", ondelete="CASCADE")
 
-    # Update bw_settings.default from String(4096) to TEXT
-    op.alter_column("bw_settings", "default", existing_type=mysql.VARCHAR(length=4096), type_=sa.TEXT(), existing_nullable=True)
+    # Update bw_instances.status from Enum("loading", "up", "down", name="instance_status_enum") to Enum("loading", "up", "down", "failover", name="instance_status_enum")
+    op.alter_column(
+        "bw_instances",
+        "status",
+        existing_type=mysql.ENUM("loading", "up", "down", name="instance_status_enum"),
+        type_=mysql.ENUM("loading", "up", "down", "failover", name="instance_status_enum"),
+        existing_nullable=False,
+    )
 
     # Drop bw_ui_users.id column
     op.drop_column("bw_ui_users", "id")
@@ -91,7 +97,13 @@ def downgrade() -> None:
     with op.batch_alter_table("bw_services_settings") as batch_op:
         batch_op.create_foreign_key("bw_services_settings_ibfk_1", "bw_services", ["service_id"], ["id"], onupdate="CASCADE", ondelete="CASCADE")
 
-    op.alter_column("bw_settings", "default", existing_type=sa.TEXT(), type_=mysql.VARCHAR(length=4096), existing_nullable=True)
+    op.alter_column(
+        "bw_instances",
+        "status",
+        existing_type=mysql.ENUM("loading", "up", "down", "failover", name="instance_status_enum"),
+        type_=mysql.ENUM("loading", "up", "down", name="instance_status_enum"),
+        existing_nullable=False,
+    )
 
     # Re-add bw_ui_users.id and index on username
     op.add_column("bw_ui_users", sa.Column("id", sa.Integer(), autoincrement=True, nullable=False))

src/common/db/alembic/mariadb_versions/bfa7869e34c3_upgrade_to_version_1_6_0_beta.py

@@ -228,6 +228,9 @@ def upgrade():
         batch_op.drop_column("obfuscation_file")
         batch_op.drop_column("obfuscation_checksum")
 
+    # Update bw_settings.default from String(4096) to TEXT
+    op.alter_column("bw_settings", "default", existing_type=mysql.VARCHAR(length=4096), type_=sa.TEXT(), existing_nullable=True)
+
     # bw_instances changes
     with op.batch_alter_table("bw_instances") as batch_op:
         batch_op.add_column(sa.Column("name", sa.String(256), nullable=True))
@@ -389,6 +392,9 @@ def downgrade():
     op.create_primary_key("bw_settings_pkey", "bw_settings", ["id", "name"])
     op.create_unique_constraint("id", "bw_settings", ["id"])
 
+    # Update bw_settings.default from TEXT to String(4096)
+    op.alter_column("bw_settings", "default", existing_type=sa.TEXT(), type_=mysql.VARCHAR(length=4096), existing_nullable=True)
+
     # bw_jobs revert: drop run_async, add success and last_run
     with op.batch_alter_table("bw_jobs") as batch_op:
         batch_op.drop_column("run_async")

src/common/db/alembic/mysql_versions/6307fa627563_upgrade_to_version_1_6_0_rc1.py

@@ -48,8 +48,14 @@ def upgrade() -> None:
     with op.batch_alter_table("bw_services_settings") as batch_op:
         batch_op.create_foreign_key("bw_services_settings_ibfk_1", "bw_services", ["service_id"], ["id"], onupdate="CASCADE", ondelete="CASCADE")
 
-    # Update bw_settings.default from String(4096) to TEXT
-    op.alter_column("bw_settings", "default", existing_type=mysql.VARCHAR(length=4096), type_=sa.TEXT(), existing_nullable=True)
+    # Update bw_instances.status from Enum("loading", "up", "down", name="instance_status_enum") to Enum("loading", "up", "down", "failover", name="instance_status_enum")
+    op.alter_column(
+        "bw_instances",
+        "status",
+        existing_type=mysql.ENUM("loading", "up", "down", name="instance_status_enum"),
+        type_=mysql.ENUM("loading", "up", "down", "failover", name="instance_status_enum"),
+        existing_nullable=False,
+    )
 
     # Drop bw_ui_users.id column
     op.drop_column("bw_ui_users", "id")
@@ -91,7 +97,13 @@ def downgrade() -> None:
     with op.batch_alter_table("bw_services_settings") as batch_op:
         batch_op.create_foreign_key("bw_services_settings_ibfk_1", "bw_services", ["service_id"], ["id"], onupdate="CASCADE", ondelete="CASCADE")
 
-    op.alter_column("bw_settings", "default", existing_type=sa.TEXT(), type_=mysql.VARCHAR(length=4096), existing_nullable=True)
+    op.alter_column(
+        "bw_instances",
+        "status",
+        existing_type=mysql.ENUM("loading", "up", "down", "failover", name="instance_status_enum"),
+        type_=mysql.ENUM("loading", "up", "down", name="instance_status_enum"),
+        existing_nullable=False,
+    )
 
     # Re-add bw_ui_users.id and index on username
     op.add_column("bw_ui_users", sa.Column("id", sa.Integer(), autoincrement=True, nullable=False))

src/common/db/alembic/mysql_versions/839424d81cf7_upgrade_to_version_1_6_0_beta.py

@@ -228,6 +228,9 @@ def upgrade():
         batch_op.drop_column("obfuscation_file")
         batch_op.drop_column("obfuscation_checksum")
 
+    # Update bw_settings.default from String(4096) to TEXT
+    op.alter_column("bw_settings", "default", existing_type=mysql.VARCHAR(length=4096), type_=sa.TEXT(), existing_nullable=True)
+
     # bw_instances changes
     with op.batch_alter_table("bw_instances") as batch_op:
         batch_op.add_column(sa.Column("name", sa.String(256), nullable=True))
@@ -389,6 +392,9 @@ def downgrade():
     op.create_primary_key("bw_settings_pkey", "bw_settings", ["id", "name"])
     op.create_unique_constraint("id", "bw_settings", ["id"])
 
+    # Update bw_settings.default from TEXT to String(4096)
+    op.alter_column("bw_settings", "default", existing_type=sa.TEXT(), type_=mysql.VARCHAR(length=4096), existing_nullable=True)
+
     # bw_jobs revert: drop run_async, add success and last_run
     with op.batch_alter_table("bw_jobs") as batch_op:
         batch_op.drop_column("run_async")

src/common/db/alembic/postgresql_versions/0b08c406d820_upgrade_to_version_1_6_0_beta.py

@@ -276,6 +276,9 @@ def upgrade():
         batch_op.alter_column("creation_date", existing_type=sa.DateTime(timezone=True), nullable=False)
         batch_op.alter_column("last_seen", existing_type=sa.DateTime(timezone=True), nullable=False)
 
+    # Update bw_settings.default from String(4096) to TEXT
+    op.alter_column("bw_settings", "default", existing_type=sa.String(4096), type_=sa.Text, existing_nullable=True)
+
     # Update version
     op.execute("UPDATE bw_metadata SET version = '1.6.0-beta' WHERE id = 1")
 
@@ -336,6 +339,9 @@ def downgrade():
     op.drop_table("bw_ui_users")
     op.rename_table("bw_ui_users_old", "bw_ui_users")
 
+    # Reverse bw_settings.default from TEXT to String(4096)
+    op.alter_column("bw_settings", "default", existing_type=sa.Text, type_=sa.String(4096), existing_nullable=True)
+
     # 4. Drop new UI and templates tables
     op.drop_table("bw_template_custom_configs")
     op.drop_table("bw_template_settings")

src/common/db/alembic/postgresql_versions/940350925f36_upgrade_to_version_1_6_0_rc1.py

@@ -147,6 +147,16 @@ def upgrade() -> None:
     with op.batch_alter_table("bw_jobs_cache") as batch_op:
         batch_op.create_foreign_key("bw_jobs_cache_service_id_fkey", "bw_services", ["service_id"], ["id"], onupdate="CASCADE", ondelete="CASCADE")
 
+    # Update bw_instances.status from Enum("loading", "up", "down", name="instance_status_enum") to Enum("loading", "up", "down", "failover", name="instance_status_enum")
+    op.execute("ALTER TYPE instance_status_enum ADD VALUE 'failover'")
+    op.alter_column(
+        "bw_instances",
+        "status",
+        existing_type=postgresql.ENUM("loading", "up", "down", name="instance_status_enum", create_type=False),
+        type_=postgresql.ENUM("loading", "up", "down", "failover", name="instance_status_enum", create_type=False),
+        existing_nullable=False,
+    )
+
     # Update the version in bw_metadata
     op.execute("UPDATE bw_metadata SET version = '1.6.0-rc1' WHERE id = 1")
 
@@ -283,5 +293,15 @@ def downgrade() -> None:
     with op.batch_alter_table("bw_jobs_cache") as batch_op:
         batch_op.create_foreign_key("bw_jobs_cache_service_id_fkey", "bw_services", ["service_id"], ["id"], onupdate="CASCADE", ondelete="CASCADE")
 
+    # Update bw_instances.status from Enum("loading", "up", "down", "failover", name="instance_status_enum") to Enum("loading", "up", "down", name="instance_status_enum")
+    op.execute("ALTER TYPE instance_status_enum DROP VALUE 'failover'")
+    op.alter_column(
+        "bw_instances",
+        "status",
+        existing_type=postgresql.ENUM("loading", "up", "down", "failover", name="instance_status_enum", create_type=False),
+        type_=postgresql.ENUM("loading", "up", "down", name="instance_status_enum", create_type=False),
+        existing_nullable=False,
+    )
+
     # Revert version
     op.execute("UPDATE bw_metadata SET version = '1.6.0-beta' WHERE id = 1")

src/common/db/alembic/sqlite_versions/5b0ea031ccfc_upgrade_to_version_1_6_0_rc1.py

@@ -148,6 +148,34 @@ def upgrade() -> None:
     op.drop_table("bw_jobs_cache")
     op.rename_table("bw_jobs_cache_new", "bw_jobs_cache")
 
+    # --- bw_instances ---
+    # Old schema:
+    # status: sa.Enum("loading", "up", "down", name="instance_status_enum")
+    # New schema:
+    # status: Enum("loading", "up", "down", "failover", name="instance_status_enum")
+    op.create_table(
+        "bw_instances_new",
+        sa.Column("hostname", sa.String(256), primary_key=True),
+        sa.Column("name", sa.String(256), nullable=False, default="manual instance"),
+        sa.Column("port", sa.Integer, nullable=False),
+        sa.Column("server_name", sa.String(256), nullable=False),
+        sa.Column("type", sa.Enum("static", "container", "pod", name="instance_type_enum"), nullable=False, default="static"),
+        sa.Column("status", sa.Enum("loading", "up", "down", "failover", name="instance_status_enum"), nullable=False, default="loading"),
+        sa.Column("method", sa.Enum("ui", "scheduler", "autoconf", "manual", "wizard", name="methods_enum"), nullable=False, default="manual"),
+        sa.Column("creation_date", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("last_seen", sa.DateTime(timezone=True), nullable=False),
+    )
+
+    op.execute(
+        """
+        INSERT INTO bw_instances_new (hostname, name, port, server_name, type, status, method, creation_date, last_seen)
+        SELECT hostname, name, port, server_name, type, status, method, creation_date, last_seen FROM bw_instances
+    """
+    )
+
+    op.drop_table("bw_instances")
+    op.rename_table("bw_instances_new", "bw_instances")
+
     # Re-enable foreign keys
     op.execute("PRAGMA foreign_keys=ON;")
 

src/common/db/model.py

@@ -37,7 +37,7 @@ STREAM_TYPES_ENUM = Enum("no", "yes", "partial", name="stream_types_enum")
 PLUGIN_TYPES_ENUM = Enum("core", "external", "ui", "pro", name="plugin_types_enum")
 PRO_STATUS_ENUM = Enum("active", "invalid", "expired", "suspended", name="pro_status_enum")
 INSTANCE_TYPE_ENUM = Enum("static", "container", "pod", name="instance_type_enum")
-INSTANCE_STATUS_ENUM = Enum("loading", "up", "down", name="instance_status_enum")
+INSTANCE_STATUS_ENUM = Enum("loading", "up", "down", "failover", name="instance_status_enum")
 Base = declarative_base()
 
 

src/scheduler/main.py

@@ -429,6 +429,10 @@ def healthcheck_job():
                 continue
 
             if resp["msg"] == "loading":
+                if db_instance["status"] == "failover":
+                    HEALTHCHECK_LOGGER.warning(f"Instance {db_instance['hostname']} is in failover mode, skipping sending config ...")
+                    continue
+
                 HEALTHCHECK_LOGGER.info(f"Instance {bw_instance.endpoint} is loading, sending config ...")
                 api_caller = ApiCaller([bw_instance])
                 api_caller.send_files(CUSTOM_CONFIGS_PATH, "/custom_configs")
@@ -809,9 +813,18 @@ if __name__ == "__main__":
                         status = responses.get(db_instance["hostname"], {"status": "down"}).get("status", "down")
                         if status == "success":
                             success = True
-                        ret = SCHEDULER.db.update_instance(db_instance["hostname"], "up" if status == "success" else "down")
+                        ret = SCHEDULER.db.update_instance(
+                            db_instance["hostname"],
+                            (
+                                "up"
+                                if status == "success"
+                                else ("failover" if responses.get(db_instance["hostname"], {}).get("msg") == "config check failed" else "down")
+                            ),
+                        )
                         if ret:
-                            LOGGER.error(f"Couldn't update instance {db_instance['hostname']} status to down in the database: {ret}")
+                            LOGGER.error(
+                                f"Couldn't update instance {db_instance['hostname']} status to {'up' if status == 'success' else 'down'} in the database: {ret}"
+                            )
 
                         if status == "success":
                             found = False

src/ui/app/templates/instances.html

@@ -77,12 +77,12 @@
                             {% if instance.status == "up" %}
                                 <span id="status-{{ instance.hostname }}"
                                       class="badge rounded-pill bg-label-primary">Up</span>
-                            {% elif instance.status == "down" %}
-                                <span id="status-{{ instance.hostname }}"
-                                      class="badge rounded-pill bg-label-danger">Down</span>
-                            {% else %}
+                            {% elif instance.status == "loading" %}
                                 <span id="status-{{ instance.hostname }}"
                                       class="badge rounded-pill bg-label-warning">Loading</span>
+                            {% else %}
+                                <span id="status-{{ instance.hostname }}"
+                                      class="badge rounded-pill bg-label-danger">{{ instance.status | title }}</span>
                             {% endif %}
                         </td>
                         <td>