diff --git a/core/cli/worker.go b/core/cli/worker.go index 186fe298e..ecc53bb72 100644 --- a/core/cli/worker.go +++ b/core/cli/worker.go @@ -21,6 +21,7 @@ import ( "github.com/mudler/LocalAI/core/cli/workerregistry" "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/core/services/galleryop" "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/core/services/nodes" "github.com/mudler/LocalAI/core/services/storage" @@ -597,12 +598,20 @@ func (s *backendSupervisor) installBackend(req messaging.BackendInstallRequest) // Try to find the backend binary backendPath := s.findBackend(req.Backend) if backendPath == "" { - // Backend not found locally — try auto-installing from gallery - xlog.Info("Backend not found locally, attempting gallery install", "backend", req.Backend) - if err := gallery.InstallBackendFromGallery( - context.Background(), galleries, s.systemState, s.ml, req.Backend, nil, false, - ); err != nil { - return "", fmt.Errorf("installing backend from gallery: %w", err) + if req.URI != "" { + xlog.Info("Backend not found locally, attempting external install", "backend", req.Backend, "uri", req.URI) + if err := galleryop.InstallExternalBackend( + context.Background(), galleries, s.systemState, s.ml, nil, req.URI, req.Name, req.Alias, + ); err != nil { + return "", fmt.Errorf("installing backend from gallery: %w", err) + } + } else { + xlog.Info("Backend not found locally, attempting gallery install", "backend", req.Backend) + if err := gallery.InstallBackendFromGallery( + context.Background(), galleries, s.systemState, s.ml, req.Backend, nil, false, + ); err != nil { + return "", fmt.Errorf("installing backend from gallery: %w", err) + } } // Re-register after install and retry gallery.RegisterBackends(s.systemState, s.ml) diff --git a/core/http/endpoints/localai/nodes.go b/core/http/endpoints/localai/nodes.go index 75eaafc60..8eb18bfbe 100644 --- a/core/http/endpoints/localai/nodes.go +++ b/core/http/endpoints/localai/nodes.go @@ -376,7 +376,7 @@ func InstallBackendOnNodeEndpoint(unloader nodes.NodeCommandSender) echo.Handler if err := c.Bind(&req); err != nil || req.Backend == "" { return c.JSON(http.StatusBadRequest, nodeError(http.StatusBadRequest, "backend name required")) } - reply, err := unloader.InstallBackend(nodeID, req.Backend, "", req.BackendGalleries) + reply, err := unloader.InstallBackend(nodeID, req.Backend, "", req.BackendGalleries, "", "", "") if err != nil { xlog.Error("Failed to install backend on node", "node", nodeID, "backend", req.Backend, "error", err) return c.JSON(http.StatusInternalServerError, nodeError(http.StatusInternalServerError, "failed to install backend on node")) diff --git a/core/services/messaging/subjects.go b/core/services/messaging/subjects.go index 3e9af53a9..abb632b3f 100644 --- a/core/services/messaging/subjects.go +++ b/core/services/messaging/subjects.go @@ -124,8 +124,13 @@ func SubjectNodeBackendInstall(nodeID string) string { // BackendInstallRequest is the payload for a backend.install NATS request. type BackendInstallRequest struct { Backend string `json:"backend"` - ModelID string `json:"model_id,omitempty"` // unique model identifier — each model gets its own gRPC process + ModelID string `json:"model_id,omitempty"` BackendGalleries string `json:"backend_galleries,omitempty"` + // URI is set for external installs (OCI image, URL, or path). When non-empty + // the worker routes to InstallExternalBackend instead of the gallery lookup. + URI string `json:"uri,omitempty"` + Name string `json:"name,omitempty"` + Alias string `json:"alias,omitempty"` } // BackendInstallReply is the response from a backend.install NATS request. diff --git a/core/services/nodes/managers_distributed.go b/core/services/nodes/managers_distributed.go index 373f00b6d..c626de7ee 100644 --- a/core/services/nodes/managers_distributed.go +++ b/core/services/nodes/managers_distributed.go @@ -293,7 +293,7 @@ func (d *DistributedBackendManager) InstallBackend(ctx context.Context, op *gall backendName := op.GalleryElementName _, err := d.enqueueAndDrainBackendOp(ctx, OpBackendInstall, backendName, galleriesJSON, func(node BackendNode) error { - reply, err := d.adapter.InstallBackend(node.ID, backendName, "", string(galleriesJSON)) + reply, err := d.adapter.InstallBackend(node.ID, backendName, "", string(galleriesJSON), op.ExternalURI, op.ExternalName, op.ExternalAlias) if err != nil { return err } @@ -311,7 +311,7 @@ func (d *DistributedBackendManager) UpgradeBackend(ctx context.Context, name str galleriesJSON, _ := json.Marshal(d.backendGalleries) _, err := d.enqueueAndDrainBackendOp(ctx, OpBackendUpgrade, name, galleriesJSON, func(node BackendNode) error { - reply, err := d.adapter.InstallBackend(node.ID, name, "", string(galleriesJSON)) + reply, err := d.adapter.InstallBackend(node.ID, name, "", string(galleriesJSON), "", "", "") if err != nil { return err } diff --git a/core/services/nodes/reconciler.go b/core/services/nodes/reconciler.go index 063a68d39..3ba897773 100644 --- a/core/services/nodes/reconciler.go +++ b/core/services/nodes/reconciler.go @@ -188,7 +188,7 @@ func (rc *ReplicaReconciler) drainPendingBackendOps(ctx context.Context) { case OpBackendDelete: _, applyErr = rc.adapter.DeleteBackend(op.NodeID, op.Backend) case OpBackendInstall, OpBackendUpgrade: - reply, err := rc.adapter.InstallBackend(op.NodeID, op.Backend, "", string(op.Galleries)) + reply, err := rc.adapter.InstallBackend(op.NodeID, op.Backend, "", string(op.Galleries), "", "", "") if err != nil { applyErr = err } else if !reply.Success { diff --git a/core/services/nodes/router.go b/core/services/nodes/router.go index fd28fc606..a9e899ea3 100644 --- a/core/services/nodes/router.go +++ b/core/services/nodes/router.go @@ -504,7 +504,7 @@ func (r *SmartRouter) installBackendOnNode(ctx context.Context, node *BackendNod return "", fmt.Errorf("no NATS connection for backend installation") } - reply, err := r.unloader.InstallBackend(node.ID, backendType, modelID, r.galleriesJSON) + reply, err := r.unloader.InstallBackend(node.ID, backendType, modelID, r.galleriesJSON, "", "", "") if err != nil { return "", err } diff --git a/core/services/nodes/router_test.go b/core/services/nodes/router_test.go index 7f2840ba6..8fb5f3bd9 100644 --- a/core/services/nodes/router_test.go +++ b/core/services/nodes/router_test.go @@ -244,7 +244,7 @@ type fakeUnloader struct { unloadErr error } -func (f *fakeUnloader) InstallBackend(_, _, _, _ string) (*messaging.BackendInstallReply, error) { +func (f *fakeUnloader) InstallBackend(_, _, _, _, _, _, _ string) (*messaging.BackendInstallReply, error) { return f.installReply, f.installErr } diff --git a/core/services/nodes/unloader.go b/core/services/nodes/unloader.go index 980960708..a1962ac8a 100644 --- a/core/services/nodes/unloader.go +++ b/core/services/nodes/unloader.go @@ -17,7 +17,7 @@ type backendStopRequest struct { // NodeCommandSender abstracts NATS-based commands to worker nodes. // Used by HTTP endpoint handlers to avoid coupling to the concrete RemoteUnloaderAdapter. type NodeCommandSender interface { - InstallBackend(nodeID, backendType, modelID, galleriesJSON string) (*messaging.BackendInstallReply, error) + InstallBackend(nodeID, backendType, modelID, galleriesJSON, uri, name, alias string) (*messaging.BackendInstallReply, error) DeleteBackend(nodeID, backendName string) (*messaging.BackendDeleteReply, error) ListBackends(nodeID string) (*messaging.BackendListReply, error) StopBackend(nodeID, backend string) error @@ -72,7 +72,7 @@ func (a *RemoteUnloaderAdapter) UnloadRemoteModel(modelName string) error { // The worker installs the backend from gallery (if not already installed), // starts the gRPC process, and replies when ready. // Timeout: 5 minutes (gallery install can take a while). -func (a *RemoteUnloaderAdapter) InstallBackend(nodeID, backendType, modelID, galleriesJSON string) (*messaging.BackendInstallReply, error) { +func (a *RemoteUnloaderAdapter) InstallBackend(nodeID, backendType, modelID, galleriesJSON, uri, name, alias string) (*messaging.BackendInstallReply, error) { subject := messaging.SubjectNodeBackendInstall(nodeID) xlog.Info("Sending NATS backend.install", "nodeID", nodeID, "backend", backendType, "modelID", modelID) @@ -80,6 +80,9 @@ func (a *RemoteUnloaderAdapter) InstallBackend(nodeID, backendType, modelID, gal Backend: backendType, ModelID: modelID, BackendGalleries: galleriesJSON, + URI: uri, + Name: name, + Alias: alias, }, 5*time.Minute) }