Merge pull request #8959 from appwrite/fix-send-migration-errors-individually

Send migration errors individually

app/worker.php

@@ -277,6 +277,50 @@ Server::setResource(
     fn () => fn (Document $project, string $resourceType, ?string $resourceId) => false
 );
 
+Server::setResource('logError', function (Registry $register, Document $project) {
+    return function (Throwable $error, string $namespace, string $action, ?array $extras) use ($register, $project) {
+        $logger = $register->get('logger');
+
+        if ($logger) {
+            $version = System::getEnv('_APP_VERSION', 'UNKNOWN');
+
+            $log = new Log();
+            $log->setNamespace($namespace);
+            $log->setServer(\gethostname());
+            $log->setVersion($version);
+            $log->setType(Log::TYPE_ERROR);
+            $log->setMessage($error->getMessage());
+
+            $log->addTag('code', $error->getCode());
+            $log->addTag('verboseType', get_class($error));
+            $log->addTag('projectId', $project->getId() ?? '');
+
+            $log->addExtra('file', $error->getFile());
+            $log->addExtra('line', $error->getLine());
+            $log->addExtra('trace', $error->getTraceAsString());
+
+
+            foreach ($extras as $key => $value) {
+                $log->addExtra($key, $value);
+            }
+
+            $log->setAction($action);
+
+            $isProduction = System::getEnv('_APP_ENV', 'development') === 'production';
+            $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING);
+
+            try {
+                $responseCode = $logger->addLog($log);
+                Console::info('Error log pushed with status code: ' . $responseCode);
+            } catch (Throwable $th) {
+                Console::error('Error pushing log: ' . $th->getMessage());
+            }
+        }
+
+        Console::warning("Failed: {$error->getMessage()}");
+        Console::warning($error->getTraceAsString());
+    };
+}, ['register', 'project']);
 
 $pools = $register->get('pools');
 $platform = new Appwrite();

src/Appwrite/Platform/Workers/Migrations.php

@@ -16,7 +16,6 @@ use Utopia\Database\Exception\Conflict;
 use Utopia\Database\Exception\Restricted;
 use Utopia\Database\Exception\Structure;
 use Utopia\Database\Helpers\ID;
-use Utopia\Logger\Log;
 use Utopia\Migration\Destination;
 use Utopia\Migration\Destinations\Appwrite as DestinationAppwrite;
 use Utopia\Migration\Exception as MigrationException;
@@ -37,6 +36,8 @@ class Migrations extends Action
 
     protected Document $project;
 
+    protected $logError;
+
     public static function getName(): string
     {
         return 'migrations';
@@ -52,14 +53,14 @@ class Migrations extends Action
             ->inject('message')
             ->inject('dbForProject')
             ->inject('dbForConsole')
-            ->inject('log')
-            ->callback(fn (Message $message, Database $dbForProject, Database $dbForConsole, Log $log) => $this->action($message, $dbForProject, $dbForConsole, $log));
+            ->inject('logError')
+            ->callback(fn (Message $message, Database $dbForProject, Database $dbForConsole, callable $logError) => $this->action($message, $dbForProject, $dbForConsole, $logError));
     }
 
     /**
      * @throws Exception
      */
-    public function action(Message $message, Database $dbForProject, Database $dbForConsole, Log $log): void
+    public function action(Message $message, Database $dbForProject, Database $dbForConsole, callable $logError): void
     {
         $payload = $message->getPayload() ?? [];
 
@@ -78,6 +79,7 @@ class Migrations extends Action
         $this->dbForProject = $dbForProject;
         $this->dbForConsole = $dbForConsole;
         $this->project = $project;
+        $this->logError = $logError;
 
         /**
          * Handle Event execution.
@@ -86,10 +88,7 @@ class Migrations extends Action
             return;
         }
 
-        $log->addTag('migrationId', $migration->getId());
-        $log->addTag('projectId', $project->getId());
-
-        $this->processMigration($migration, $log);
+        $this->processMigration($migration);
     }
 
     /**
@@ -259,7 +258,7 @@ class Migrations extends Action
      * @throws \Utopia\Database\Exception
      * @throws Exception
      */
-    protected function processMigration(Document $migration, Log $log): void
+    protected function processMigration(Document $migration): void
     {
         $project = $this->project;
         $projectDocument = $this->dbForConsole->getDocument('projects', $project->getId());
@@ -285,8 +284,6 @@ class Migrations extends Action
             $migration->setAttribute('status', 'processing');
             $this->updateMigrationDocument($migration, $projectDocument);
 
-            $log->addTag('type', $migration->getAttribute('source'));
-
             $source = $this->processSource($migration);
             $destination = $this->processDestination($migration, $tempAPIKey->getAttribute('secret'));
 
@@ -324,7 +321,6 @@ class Migrations extends Action
 
                 $errorMessages = [];
                 foreach ($sourceErrors as $error) {
-                    /** @var $sourceErrors $error */
                     $message = "Error occurred while fetching '{$error->getResourceName()}:{$error->getResourceId()}' from source with message: '{$error->getMessage()}'";
                     if ($error->getPrevious()) {
                         $message .= " Message: ".$error->getPrevious()->getMessage() . " File: ".$error->getPrevious()->getFile() . " Line: ".$error->getPrevious()->getLine();
@@ -344,7 +340,6 @@ class Migrations extends Action
                 }
 
                 $migration->setAttribute('errors', $errorMessages);
-                $log->addExtra('migrationErrors', json_encode($errorMessages));
                 $this->updateMigrationDocument($migration, $projectDocument);
 
                 return;
@@ -359,7 +354,6 @@ class Migrations extends Action
             if (! $migration->isEmpty()) {
                 $migration->setAttribute('status', 'failed');
                 $migration->setAttribute('stage', 'finished');
-                $migration->setAttribute('errors', [$th->getMessage()]);
 
                 return;
             }
@@ -379,7 +373,6 @@ class Migrations extends Action
                 }
 
                 $migration->setAttribute('errors', $errorMessages);
-                $log->addTag('migrationErrors', json_encode($errorMessages));
             }
         } finally {
             if (! $tempAPIKey->isEmpty()) {
@@ -394,7 +387,23 @@ class Migrations extends Action
                 $destination->error();
                 $source->error();
 
-                throw new Exception('Migration failed');
+                foreach ($source->getErrors() as $error) {
+                    call_user_func($this->logError, $error, 'appwrite-worker', 'appwrite-queue-' . self::getName(), [
+                        'migrationId' => $migration->getId() ?? '',
+                        'source' => $migration->getAttribute('source') ?? '',
+                        'resourceName' => $error->getResourceName(),
+                        'resourceGroup' => $error->getResourceGroup()
+                    ]);
+                }
+
+                foreach ($destination->getErrors() as $error) {
+                    call_user_func($this->logError, $error, 'appwrite-worker', 'appwrite-queue-' . self::getName(), [
+                        'migrationId' => $migration->getId() ?? '',
+                        'source' => $migration->getAttribute('source') ?? '',
+                        'resourceName' => $error->getResourceName(),
+                        'resourceGroup' => $error->getResourceGroup()
+                    ]);
+                }
             }
 
             if ($migration->getAttribute('status', '') === 'completed') {