From 830ad99111c854748d27effadc5e2edec61553d7 Mon Sep 17 00:00:00 2001 From: Fabian Gruber Date: Mon, 2 Dec 2024 10:47:17 +0100 Subject: [PATCH 01/74] chore: add audit labels to project resources --- app/controllers/api/projects.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 388c9e22ea..56e2b7900f 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -61,6 +61,7 @@ App::post('/v1/projects') ->desc('Create project') ->groups(['api', 'projects']) ->label('audits.event', 'projects.create') + ->label('audits.resource', 'project/{response.$id}') ->label('scope', 'projects.write') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'projects') @@ -382,6 +383,8 @@ App::patch('/v1/projects/:projectId') ->desc('Update project') ->groups(['api', 'projects']) ->label('scope', 'projects.write') + ->label('audits.event', 'projects.update') + ->label('audits.resource', 'project/{request.projectId}') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'projects') ->label('sdk.method', 'update') @@ -996,6 +999,7 @@ App::delete('/v1/projects/:projectId') ->desc('Delete project') ->groups(['api', 'projects']) ->label('audits.event', 'projects.delete') + ->label('audits.resource', 'project/{request.projectId}') ->label('scope', 'projects.write') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'projects') @@ -1529,6 +1533,7 @@ App::post('/v1/projects/:projectId/platforms') ->desc('Create platform') ->groups(['api', 'projects']) ->label('audits.event', 'platforms.create') + ->label('audits.resource', 'project/{request.projectId}') ->label('scope', 'platforms.write') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'projects') @@ -1693,6 +1698,7 @@ App::delete('/v1/projects/:projectId/platforms/:platformId') ->desc('Delete platform') ->groups(['api', 'projects']) ->label('audits.event', 'platforms.delete') + ->label('audits.resource', 'project/{request.projectId}/platform/${request.platformId}') ->label('scope', 'platforms.write') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'projects') From 297826a62a3b8a0cd4078e82e122c94d789d83f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Fri, 27 Dec 2024 13:13:14 +0100 Subject: [PATCH 02/74] Fix git identity collisions --- app/config/collections.php | 33 +++++++++++++ app/controllers/api/vcs.php | 97 ++++++++++++------------------------- 2 files changed, 65 insertions(+), 65 deletions(-) diff --git a/app/config/collections.php b/app/config/collections.php index a55ab1abd0..7a35dfe356 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -5417,6 +5417,39 @@ $consoleCollections = array_merge([ 'default' => false, 'array' => false, ], + [ + '$id' => ID::custom('personalAccessToken'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 16384, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => ['encrypt'], + ], + [ + '$id' => ID::custom('personalAccessTokenExpiry'), + 'type' => Database::VAR_DATETIME, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => ['datetime'], + ], + [ + '$id' => ID::custom('personalRefreshToken'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 16384, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => ['encrypt'], + ], ], 'indexes' => [ diff --git a/app/controllers/api/vcs.php b/app/controllers/api/vcs.php index 6e81c43ef8..eb3ef99495 100644 --- a/app/controllers/api/vcs.php +++ b/app/controllers/api/vcs.php @@ -355,53 +355,6 @@ App::get('/v1/vcs/github/callback') throw new Exception(Exception::PROJECT_NOT_FOUND, $error); } - $personalSlug = ''; - - // OAuth Authroization - if (!empty($code)) { - $oauth2 = new OAuth2Github(System::getEnv('_APP_VCS_GITHUB_CLIENT_ID', ''), System::getEnv('_APP_VCS_GITHUB_CLIENT_SECRET', ''), ""); - $accessToken = $oauth2->getAccessToken($code) ?? ''; - $refreshToken = $oauth2->getRefreshToken($code) ?? ''; - $accessTokenExpiry = $oauth2->getAccessTokenExpiry($code) ?? ''; - $personalSlug = $oauth2->getUserSlug($accessToken) ?? ''; - $email = $oauth2->getUserEmail($accessToken); - $oauth2ID = $oauth2->getUserID($accessToken); - - // Makes sure this email is not already used in another identity - $identity = $dbForPlatform->findOne('identities', [ - Query::equal('providerEmail', [$email]), - ]); - if (!$identity->isEmpty()) { - if ($identity->getAttribute('userInternalId', '') !== $user->getInternalId()) { - throw new Exception(Exception::USER_EMAIL_ALREADY_EXISTS); - } - - $identity = $identity - ->setAttribute('providerAccessToken', $accessToken) - ->setAttribute('providerRefreshToken', $refreshToken) - ->setAttribute('providerAccessTokenExpiry', DateTime::addSeconds(new \DateTime(), (int)$accessTokenExpiry)); - - $dbForPlatform->updateDocument('identities', $identity->getId(), $identity); - } else { - $identity = $dbForPlatform->createDocument('identities', new Document([ - '$id' => ID::unique(), - '$permissions' => [ - Permission::read(Role::any()), - Permission::update(Role::user($user->getId())), - Permission::delete(Role::user($user->getId())), - ], - 'userInternalId' => $user->getInternalId(), - 'userId' => $user->getId(), - 'provider' => 'github', - 'providerUid' => $oauth2ID, - 'providerEmail' => $email, - 'providerAccessToken' => $accessToken, - 'providerRefreshToken' => $refreshToken, - 'providerAccessTokenExpiry' => DateTime::addSeconds(new \DateTime(), (int)$accessTokenExpiry), - ])); - } - } - // Create / Update installation if (!empty($providerInstallationId)) { $privateKey = System::getEnv('_APP_VCS_GITHUB_PRIVATE_KEY'); @@ -416,6 +369,22 @@ App::get('/v1/vcs/github/callback') Query::equal('projectInternalId', [$projectInternalId]) ]); + $personal = false; + $refreshToken = null; + $accessToken = null; + $accessTokenExpiry = null; + + if (!empty($code)) { + $oauth2 = new OAuth2Github(System::getEnv('_APP_VCS_GITHUB_CLIENT_ID', ''), System::getEnv('_APP_VCS_GITHUB_CLIENT_SECRET', ''), ""); + $accessToken = $oauth2->getAccessToken($code) ?? ''; + $refreshToken = $oauth2->getRefreshToken($code) ?? ''; + $accessTokenExpiry = $oauth2->getAccessTokenExpiry($code) ?? ''; + + $personalSlug = $oauth2->getUserSlug($accessToken) ?? ''; + + $personal = $personalSlug === $owner; + } + if ($installation->isEmpty()) { $teamId = $project->getAttribute('teamId', ''); @@ -433,14 +402,20 @@ App::get('/v1/vcs/github/callback') 'projectInternalId' => $projectInternalId, 'provider' => 'github', 'organization' => $owner, - 'personal' => $personalSlug === $owner + 'personal' => $personal, + 'personalRefreshToken' => $refreshToken, + 'personalAccessToken' => $accessToken, + 'personalAccessTokenExpiry' => $accessTokenExpiry, ]); $installation = $dbForPlatform->createDocument('installations', $installation); } else { $installation = $installation ->setAttribute('organization', $owner) - ->setAttribute('personal', $personalSlug === $owner); + ->setAttribute('personal', $personal) + ->setAttribute('personalRefreshToken', $refreshToken) + ->setAttribute('personalAccessToken', $accessToken) + ->setAttribute('personalAccessTokenExpiry', $accessTokenExpiry); $installation = $dbForPlatform->updateDocument('installations', $installation->getId(), $installation); } } else { @@ -720,17 +695,9 @@ App::post('/v1/vcs/github/installations/:installationId/providerRepositories') if ($installation->getAttribute('personal', false) === true) { $oauth2 = new OAuth2Github(System::getEnv('_APP_VCS_GITHUB_CLIENT_ID', ''), System::getEnv('_APP_VCS_GITHUB_CLIENT_SECRET', ''), ""); - $identity = $dbForPlatform->findOne('identities', [ - Query::equal('provider', ['github']), - Query::equal('userInternalId', [$user->getInternalId()]), - ]); - if ($identity->isEmpty()) { - throw new Exception(Exception::USER_IDENTITY_NOT_FOUND); - } - - $accessToken = $identity->getAttribute('providerAccessToken'); - $refreshToken = $identity->getAttribute('providerRefreshToken'); - $accessTokenExpiry = $identity->getAttribute('providerAccessTokenExpiry'); + $accessToken = $installation->getAttribute('personalAccessToken'); + $refreshToken = $installation->getAttribute('personalRefreshToken'); + $accessTokenExpiry = $installation->getAttribute('personalAccessTokenExpiry'); $isExpired = new \DateTime($accessTokenExpiry) < new \DateTime('now'); if ($isExpired) { @@ -745,12 +712,12 @@ App::post('/v1/vcs/github/installations/:installationId/providerRepositories') throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, "Another request is currently refreshing OAuth token. Please try again."); } - $identity = $identity - ->setAttribute('providerAccessToken', $accessToken) - ->setAttribute('providerRefreshToken', $refreshToken) - ->setAttribute('providerAccessTokenExpiry', DateTime::addSeconds(new \DateTime(), (int)$oauth2->getAccessTokenExpiry(''))); + $installation = $installation + ->setAttribute('personalAccessToken', $accessToken) + ->setAttribute('personalRefreshToken', $refreshToken) + ->setAttribute('personalAccessTokenExpiry', DateTime::addSeconds(new \DateTime(), (int)$oauth2->getAccessTokenExpiry(''))); - $dbForPlatform->updateDocument('identities', $identity->getId(), $identity); + $dbForPlatform->updateDocument('installations', $installation->getId(), $installation); } try { From fcb517e679fb594220142713f3922604995ea6f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Fri, 27 Dec 2024 13:32:22 +0100 Subject: [PATCH 03/74] Fix type error --- app/controllers/api/vcs.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/vcs.php b/app/controllers/api/vcs.php index eb3ef99495..f90db24e98 100644 --- a/app/controllers/api/vcs.php +++ b/app/controllers/api/vcs.php @@ -376,12 +376,12 @@ App::get('/v1/vcs/github/callback') if (!empty($code)) { $oauth2 = new OAuth2Github(System::getEnv('_APP_VCS_GITHUB_CLIENT_ID', ''), System::getEnv('_APP_VCS_GITHUB_CLIENT_SECRET', ''), ""); + $accessToken = $oauth2->getAccessToken($code) ?? ''; $refreshToken = $oauth2->getRefreshToken($code) ?? ''; - $accessTokenExpiry = $oauth2->getAccessTokenExpiry($code) ?? ''; + $accessTokenExpiry = DateTime::addSeconds(new \DateTime(), \intval($oauth2->getAccessTokenExpiry($code))); $personalSlug = $oauth2->getUserSlug($accessToken) ?? ''; - $personal = $personalSlug === $owner; } From 58fd0d35bd13d034c79efc2d2ee6b4210ac75955 Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 29 Dec 2024 17:47:52 +0200 Subject: [PATCH 04/74] database crud usage addition --- src/Appwrite/Event/Usage.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Event/Usage.php b/src/Appwrite/Event/Usage.php index 89e900d2ab..d2e302f82b 100644 --- a/src/Appwrite/Event/Usage.php +++ b/src/Appwrite/Event/Usage.php @@ -69,7 +69,7 @@ class Usage extends Event 'reduce' => $this->reduce, 'metrics' => $this->metrics, ]); - + var_dump($this->metrics); $this->metrics = []; return $result; From c23af8168becf391dcc8403d10d727fe553fdeab Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 29 Dec 2024 17:48:05 +0200 Subject: [PATCH 05/74] database crud usage addition --- app/controllers/api/databases.php | 158 +++++++++++++++++++----------- app/init.php | 2 + 2 files changed, 104 insertions(+), 56 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index aad072c50a..80bc72c4b7 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -2885,7 +2885,11 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') throw new Exception(Exception::DOCUMENT_INVALID_STRUCTURE, '$id is not allowed for creating new documents, try update instead'); } - $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); + $database = Authorization::skip(function () use ($queueForUsage, $dbForProject, $databaseId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('databases', $databaseId); + }); + $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -2895,6 +2899,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') } $collection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId)); + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); if ($collection->isEmpty() || (!$collection->getAttribute('enabled', false) && !$isAPIKey && !$isPrivilegedUser)) { throw new Exception(Exception::COLLECTION_NOT_FOUND); @@ -2981,8 +2986,10 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip( - fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId) + $relatedCollection = Authorization::skip(function () use ($relatedCollectionId, $dbForProject, $queueForUsage) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); + } ); foreach ($relations as &$relation) { @@ -2995,8 +3002,10 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $relation = new Document($relation); } if ($relation instanceof Document) { - $current = Authorization::skip( - fn () => $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId(), $relation->getId()) + $current = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollection, $relation) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId(), $relation->getId()); + } ); if ($current->isEmpty()) { @@ -3028,6 +3037,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') try { $document = $dbForProject->createDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $document); + $queueForUsage->addMetric(METRIC_DATABASE_API_WRITE, 1); } catch (StructureException $e) { throw new Exception(Exception::DOCUMENT_INVALID_STRUCTURE, $e->getMessage()); } catch (DuplicateException $e) { @@ -3037,7 +3047,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') } // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, $queueForUsage) { $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3057,8 +3067,10 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip( - fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId) + $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); + } ); foreach ($related as $relation) { @@ -3116,8 +3128,11 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') ->inject('response') ->inject('dbForProject') ->inject('mode') - ->action(function (string $databaseId, string $collectionId, array $queries, Response $response, Database $dbForProject, string $mode) { + ->inject('queueForUsage') + ->action(function (string $databaseId, string $collectionId, array $queries, Response $response, Database $dbForProject, string $mode, Usage $queueForUsage) { $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -3126,6 +3141,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } $collection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId)); + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); if ($collection->isEmpty() || (!$collection->getAttribute('enabled', false) && !$isAPIKey && !$isPrivilegedUser)) { throw new Exception(Exception::COLLECTION_NOT_FOUND); @@ -3155,7 +3171,10 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') $documentId = $cursor->getValue(); - $cursorDocument = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId)); + $cursorDocument = Authorization::skip(function() use ($dbForProject, $queueForUsage, $collection, $database, $documentId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId); + }); if ($cursorDocument->isEmpty()) { throw new Exception(Exception::GENERAL_CURSOR_NOT_FOUND, "Document '{$documentId}' for the 'cursor' value not found."); @@ -3165,10 +3184,13 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } $documents = $dbForProject->find('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $queries); + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $total = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $queries, APP_LIMIT_COUNT); + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); // Add $collectionId and $databaseId for all documents - $processDocument = (function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database): bool { + $processDocument = (function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, $queueForUsage): bool { if ($document->isEmpty()) { return false; } @@ -3195,7 +3217,10 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId)); + $relatedCollection = Authorization::skip(function() use ($dbForProject, $queueForUsage, $database, $relatedCollectionId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); + }); foreach ($relations as $index => $doc) { if ($doc instanceof Document) { @@ -3274,8 +3299,11 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen ->inject('response') ->inject('dbForProject') ->inject('mode') - ->action(function (string $databaseId, string $collectionId, string $documentId, array $queries, Response $response, Database $dbForProject, string $mode) { + ->inject('queueForUsage') + ->action(function (string $databaseId, string $collectionId, string $documentId, array $queries, Response $response, Database $dbForProject, string $mode, Usage $queueForUsage) { + $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -3284,7 +3312,10 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen throw new Exception(Exception::DATABASE_NOT_FOUND); } - $collection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId)); + $collection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collectionId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId); + }); if ($collection->isEmpty() || (!$collection->getAttribute('enabled', false) && !$isAPIKey && !$isPrivilegedUser)) { throw new Exception(Exception::COLLECTION_NOT_FOUND); @@ -3293,6 +3324,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen try { $queries = Query::parseQueries($queries); $document = $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId, $queries); + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); } catch (AuthorizationException) { throw new Exception(Exception::USER_UNAUTHORIZED); } catch (QueryException $e) { @@ -3304,7 +3336,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen } // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, $queueForUsage) { if ($document->isEmpty()) { return; } @@ -3328,8 +3360,10 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip( - fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId) + $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); + } ); foreach ($related as $relation) { @@ -3481,7 +3515,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum ->inject('dbForProject') ->inject('queueForEvents') ->inject('mode') - ->action(function (string $databaseId, string $collectionId, string $documentId, string|array $data, ?array $permissions, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Event $queueForEvents, string $mode) { + ->inject('queueForUsage') + ->action(function (string $databaseId, string $collectionId, string $documentId, string|array $data, ?array $permissions, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Event $queueForEvents, string $mode, Usage $queueForUsage) { $data = (\is_string($data)) ? \json_decode($data, true) : $data; // Cast to JSON array @@ -3489,7 +3524,10 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum throw new Exception(Exception::DOCUMENT_MISSING_PAYLOAD); } - $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); + $database = Authorization::skip(function () use($queueForUsage, $dbForProject, $databaseId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('databases', $databaseId); + }); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -3498,7 +3536,10 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum throw new Exception(Exception::DATABASE_NOT_FOUND); } - $collection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId)); + $collection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $collectionId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId); + }); if ($collection->isEmpty() || (!$collection->getAttribute('enabled', false) && !$isAPIKey && !$isPrivilegedUser)) { throw new Exception(Exception::COLLECTION_NOT_FOUND); @@ -3506,7 +3547,10 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum // Read permission should not be required for update /** @var Document $document */ - $document = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId)); + $document = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collection, $documentId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId); + }); if ($document->isEmpty()) { throw new Exception(Exception::DOCUMENT_NOT_FOUND); @@ -3548,7 +3592,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $data['$permissions'] = $permissions; $newDocument = new Document($data); - $setCollection = (function (Document $collection, Document $document) use (&$setCollection, $dbForProject, $database) { + $setCollection = (function (Document $collection, Document $document) use (&$setCollection, $dbForProject, $database, $queueForUsage) { $relationships = \array_filter( $collection->getAttribute('attributes', []), fn ($attribute) => $attribute->getAttribute('type') === Database::VAR_RELATIONSHIP @@ -3570,9 +3614,10 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip( - fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId) - ); + $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); + }); foreach ($relations as &$relation) { // If the relation is an array it can be either update or create a child document. @@ -3585,17 +3630,15 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $relation = new Document($relation); } if ($relation instanceof Document) { - $oldDocument = Authorization::skip(fn () => $dbForProject->getDocument( - 'database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId(), - $relation->getId() - )); + $oldDocument = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollection, $relation) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId(), $relation->getId()); + }); + $relation->removeAttribute('$collectionId'); $relation->removeAttribute('$databaseId'); // Attribute $collection is required for Utopia. - $relation->setAttribute( - '$collection', - 'database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId() - ); + $relation->setAttribute('$collection', 'database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId()); if ($oldDocument->isEmpty()) { if (isset($relation['$id']) && $relation['$id'] === 'unique()') { @@ -3617,14 +3660,10 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $setCollection($collection, $newDocument); try { - $document = $dbForProject->withRequestTimestamp( - $requestTimestamp, - fn () => $dbForProject->updateDocument( - 'database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), - $document->getId(), - $newDocument - ) - ); + $document = $dbForProject->withRequestTimestamp($requestTimestamp, function () use ($queueForUsage, $dbForProject, $database, $collection, $document, $newDocument) { + $queueForUsage->addMetric(METRIC_DATABASE_API_WRITE, 1); + return $dbForProject->updateDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $document->getId(), $newDocument); + }); } catch (AuthorizationException) { throw new Exception(Exception::USER_UNAUTHORIZED); } catch (DuplicateException) { @@ -3636,7 +3675,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum } // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, $queueForUsage) { $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3656,9 +3695,10 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip( - fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId) - ); + $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); + }); foreach ($related as $relation) { if ($relation instanceof Document) { @@ -3720,6 +3760,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->inject('mode') ->action(function (string $databaseId, string $collectionId, string $documentId, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Event $queueForEvents, Usage $queueForUsage, string $mode) { $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -3728,32 +3769,36 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu throw new Exception(Exception::DATABASE_NOT_FOUND); } - $collection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId)); + $collection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collectionId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId); + }); if ($collection->isEmpty() || (!$collection->getAttribute('enabled', false) && !$isAPIKey && !$isPrivilegedUser)) { throw new Exception(Exception::COLLECTION_NOT_FOUND); } // Read permission should not be required for delete - $document = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId)); + $document = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collection, $documentId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId); + }); if ($document->isEmpty()) { throw new Exception(Exception::DOCUMENT_NOT_FOUND); } try { - $dbForProject->withRequestTimestamp($requestTimestamp, function () use ($dbForProject, $database, $collection, $documentId) { - $dbForProject->deleteDocument( - 'database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), - $documentId - ); + $dbForProject->withRequestTimestamp($requestTimestamp, function () use ($queueForUsage, $dbForProject, $database, $collection, $documentId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_WRITE, 1); + $dbForProject->deleteDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId); }); } catch (NotFoundException $e) { throw new Exception(Exception::COLLECTION_NOT_FOUND); } // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, $queueForUsage) { $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3773,9 +3818,10 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip( - fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId) - ); + $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); + }); foreach ($related as $relation) { if ($relation instanceof Document) { diff --git a/app/init.php b/app/init.php index c6847610c2..d5b8e4b235 100644 --- a/app/init.php +++ b/app/init.php @@ -232,6 +232,8 @@ const API_KEY_DYNAMIC = 'dynamic'; // Usage metrics const METRIC_TEAMS = 'teams'; const METRIC_USERS = 'users'; +const METRIC_DATABASE_API_READ = 'database.api.events.read'; +const METRIC_DATABASE_API_WRITE = 'database.api.events.write'; const METRIC_WEBHOOKS_SENT = 'webhooks.events.sent'; const METRIC_WEBHOOKS_FAILED = 'webhooks.events.failed'; const METRIC_WEBHOOK_ID_SENT = '{webhookInternalId}.webhooks.events.sent'; From 025c9aeeb297f6a0a96df401c6d0281321d17fb5 Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 29 Dec 2024 17:53:45 +0200 Subject: [PATCH 06/74] database crud usage addition --- src/Appwrite/Event/Usage.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Event/Usage.php b/src/Appwrite/Event/Usage.php index d2e302f82b..89e900d2ab 100644 --- a/src/Appwrite/Event/Usage.php +++ b/src/Appwrite/Event/Usage.php @@ -69,7 +69,7 @@ class Usage extends Event 'reduce' => $this->reduce, 'metrics' => $this->metrics, ]); - var_dump($this->metrics); + $this->metrics = []; return $result; From 8e17fc69b371861ee04da7e98a99fac294b42588 Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 29 Dec 2024 17:57:29 +0200 Subject: [PATCH 07/74] database crud usage addition --- app/controllers/api/databases.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 80bc72c4b7..b98df1382f 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -2898,8 +2898,10 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') throw new Exception(Exception::DATABASE_NOT_FOUND); } - $collection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId)); - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $collection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collectionId){ + $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + return $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId); + }); if ($collection->isEmpty() || (!$collection->getAttribute('enabled', false) && !$isAPIKey && !$isPrivilegedUser)) { throw new Exception(Exception::COLLECTION_NOT_FOUND); From dd13e8b3cf705be22540b93ff3be493a771fc399 Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 29 Dec 2024 18:02:19 +0200 Subject: [PATCH 08/74] database crud usage addition --- app/controllers/api/databases.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index b98df1382f..4275ab52ae 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3219,7 +3219,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip(function() use ($dbForProject, $queueForUsage, $database, $relatedCollectionId) { + $relatedCollection = Authorization::skip(function() use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); }); From ba04a7760b21579cd6dd534a894cfd9330180260 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 30 Dec 2024 14:43:28 +0100 Subject: [PATCH 09/74] Backwards-compatibility --- app/controllers/api/vcs.php | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/app/controllers/api/vcs.php b/app/controllers/api/vcs.php index f90db24e98..f24a0d97fe 100644 --- a/app/controllers/api/vcs.php +++ b/app/controllers/api/vcs.php @@ -699,6 +699,20 @@ App::post('/v1/vcs/github/installations/:installationId/providerRepositories') $refreshToken = $installation->getAttribute('personalRefreshToken'); $accessTokenExpiry = $installation->getAttribute('personalAccessTokenExpiry'); + if (empty($accessToken) || empty($refreshToken) || empty($accessTokenExpiry)) { + $identity = $dbForPlatform->findOne('identities', [ + Query::equal('provider', ['github']), + Query::equal('userInternalId', [$user->getInternalId()]), + ]); + if ($identity->isEmpty()) { + throw new Exception(Exception::USER_IDENTITY_NOT_FOUND); + } + + $accessToken = $accessToken ?? $identity->getAttribute('providerAccessToken'); + $refreshToken = $refreshToken ?? $identity->getAttribute('providerRefreshToken'); + $accessTokenExpiry = $accessTokenExpiry ?? $identity->getAttribute('providerAccessTokenExpiry'); + } + $isExpired = new \DateTime($accessTokenExpiry) < new \DateTime('now'); if ($isExpired) { $oauth2->refreshTokens($refreshToken); From d3b764a9e2e718b93a845979deca7a29ddeb8bd4 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Mon, 30 Dec 2024 15:10:27 +0000 Subject: [PATCH 10/74] Set base specification CPUs to 0.5 again --- app/config/runtimes/specifications.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/config/runtimes/specifications.php b/app/config/runtimes/specifications.php index 68880f4d4e..d3625db8a2 100644 --- a/app/config/runtimes/specifications.php +++ b/app/config/runtimes/specifications.php @@ -6,7 +6,7 @@ return [ Specification::S_05VCPU_512MB => [ 'slug' => Specification::S_05VCPU_512MB, 'memory' => 512, - 'cpus' => 1 // TODO: revert this, it's a temporary change to test function performance. + 'cpus' => 0.5 ], Specification::S_1VCPU_512MB => [ 'slug' => Specification::S_1VCPU_512MB, From 0f2985cec1a2e80fc325b59824eedad64519b5f2 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 2 Jan 2025 09:19:09 +0200 Subject: [PATCH 11/74] database crud usage addition --- app/controllers/api/databases.php | 56 +++++++++++++++---------------- app/init.php | 4 +-- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 4275ab52ae..e51201a467 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -2886,7 +2886,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') } $database = Authorization::skip(function () use ($queueForUsage, $dbForProject, $databaseId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('databases', $databaseId); }); @@ -2899,7 +2899,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') } $collection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collectionId){ - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId); }); @@ -2989,7 +2989,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $relatedCollectionId = $relationship->getAttribute('relatedCollection'); $relatedCollection = Authorization::skip(function () use ($relatedCollectionId, $dbForProject, $queueForUsage) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); } ); @@ -3005,7 +3005,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') } if ($relation instanceof Document) { $current = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollection, $relation) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId(), $relation->getId()); } ); @@ -3039,7 +3039,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') try { $document = $dbForProject->createDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $document); - $queueForUsage->addMetric(METRIC_DATABASE_API_WRITE, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_WRITES, 1); } catch (StructureException $e) { throw new Exception(Exception::DOCUMENT_INVALID_STRUCTURE, $e->getMessage()); } catch (DuplicateException $e) { @@ -3070,7 +3070,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $relatedCollectionId = $relationship->getAttribute('relatedCollection'); $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); } ); @@ -3133,7 +3133,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') ->inject('queueForUsage') ->action(function (string $databaseId, string $collectionId, array $queries, Response $response, Database $dbForProject, string $mode, Usage $queueForUsage) { $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -3143,7 +3143,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } $collection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId)); - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); if ($collection->isEmpty() || (!$collection->getAttribute('enabled', false) && !$isAPIKey && !$isPrivilegedUser)) { throw new Exception(Exception::COLLECTION_NOT_FOUND); @@ -3174,7 +3174,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') $documentId = $cursor->getValue(); $cursorDocument = Authorization::skip(function() use ($dbForProject, $queueForUsage, $collection, $database, $documentId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId); }); @@ -3186,10 +3186,10 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } $documents = $dbForProject->find('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $queries); - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); $total = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $queries, APP_LIMIT_COUNT); - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); // Add $collectionId and $databaseId for all documents $processDocument = (function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, $queueForUsage): bool { @@ -3220,7 +3220,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') $relatedCollectionId = $relationship->getAttribute('relatedCollection'); $relatedCollection = Authorization::skip(function() use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); }); @@ -3305,7 +3305,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen ->action(function (string $databaseId, string $collectionId, string $documentId, array $queries, Response $response, Database $dbForProject, string $mode, Usage $queueForUsage) { $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -3315,7 +3315,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen } $collection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId); }); @@ -3326,7 +3326,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen try { $queries = Query::parseQueries($queries); $document = $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId, $queries); - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); } catch (AuthorizationException) { throw new Exception(Exception::USER_UNAUTHORIZED); } catch (QueryException $e) { @@ -3363,7 +3363,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen $relatedCollectionId = $relationship->getAttribute('relatedCollection'); $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); } ); @@ -3527,7 +3527,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum } $database = Authorization::skip(function () use($queueForUsage, $dbForProject, $databaseId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('databases', $databaseId); }); @@ -3539,7 +3539,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum } $collection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $collectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId); }); @@ -3550,7 +3550,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum // Read permission should not be required for update /** @var Document $document */ $document = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collection, $documentId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId); }); @@ -3617,7 +3617,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $relatedCollectionId = $relationship->getAttribute('relatedCollection'); $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); }); @@ -3633,7 +3633,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum } if ($relation instanceof Document) { $oldDocument = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollection, $relation) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId(), $relation->getId()); }); @@ -3663,7 +3663,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum try { $document = $dbForProject->withRequestTimestamp($requestTimestamp, function () use ($queueForUsage, $dbForProject, $database, $collection, $document, $newDocument) { - $queueForUsage->addMetric(METRIC_DATABASE_API_WRITE, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_WRITES, 1); return $dbForProject->updateDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $document->getId(), $newDocument); }); } catch (AuthorizationException) { @@ -3698,7 +3698,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $relatedCollectionId = $relationship->getAttribute('relatedCollection'); $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); }); @@ -3762,7 +3762,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->inject('mode') ->action(function (string $databaseId, string $collectionId, string $documentId, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Event $queueForEvents, Usage $queueForUsage, string $mode) { $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -3772,7 +3772,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu } $collection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId); }); @@ -3782,7 +3782,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu // Read permission should not be required for delete $document = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collection, $documentId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId); }); @@ -3792,7 +3792,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu try { $dbForProject->withRequestTimestamp($requestTimestamp, function () use ($queueForUsage, $dbForProject, $database, $collection, $documentId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_WRITE, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_WRITES, 1); $dbForProject->deleteDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId); }); } catch (NotFoundException $e) { @@ -3821,7 +3821,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu $relatedCollectionId = $relationship->getAttribute('relatedCollection'); $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_API_READ, 1); + $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); }); diff --git a/app/init.php b/app/init.php index d5b8e4b235..d942e056db 100644 --- a/app/init.php +++ b/app/init.php @@ -232,8 +232,8 @@ const API_KEY_DYNAMIC = 'dynamic'; // Usage metrics const METRIC_TEAMS = 'teams'; const METRIC_USERS = 'users'; -const METRIC_DATABASE_API_READ = 'database.api.events.read'; -const METRIC_DATABASE_API_WRITE = 'database.api.events.write'; +const METRIC_DATABASE_OPERATIONS_READS = 'databases.operations.reads'; +const METRIC_DATABASE_OPERATIONS_WRITES = 'databases.operations.writes'; const METRIC_WEBHOOKS_SENT = 'webhooks.events.sent'; const METRIC_WEBHOOKS_FAILED = 'webhooks.events.failed'; const METRIC_WEBHOOK_ID_SENT = '{webhookInternalId}.webhooks.events.sent'; From 42589de95ef41114d247fdbceacce8ea3d708233 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 2 Jan 2025 09:56:14 +0200 Subject: [PATCH 12/74] database crud usage addition --- app/controllers/api/databases.php | 170 +++++++++++++----------------- app/init.php | 2 + 2 files changed, 75 insertions(+), 97 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index e51201a467..ea17065d45 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -2885,11 +2885,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') throw new Exception(Exception::DOCUMENT_INVALID_STRUCTURE, '$id is not allowed for creating new documents, try update instead'); } - $database = Authorization::skip(function () use ($queueForUsage, $dbForProject, $databaseId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('databases', $databaseId); - }); - + $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -2898,10 +2894,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') throw new Exception(Exception::DATABASE_NOT_FOUND); } - $collection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collectionId){ - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId); - }); + $collection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId)); if ($collection->isEmpty() || (!$collection->getAttribute('enabled', false) && !$isAPIKey && !$isPrivilegedUser)) { throw new Exception(Exception::COLLECTION_NOT_FOUND); @@ -2988,10 +2981,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip(function () use ($relatedCollectionId, $dbForProject, $queueForUsage) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); - } + $relatedCollection = Authorization::skip( + fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId) ); foreach ($relations as &$relation) { @@ -3004,10 +2995,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $relation = new Document($relation); } if ($relation instanceof Document) { - $current = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollection, $relation) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId(), $relation->getId()); - } + $current = Authorization::skip( + fn () => $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId(), $relation->getId()) ); if ($current->isEmpty()) { @@ -3039,7 +3028,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') try { $document = $dbForProject->createDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $document); - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_WRITES, 1); } catch (StructureException $e) { throw new Exception(Exception::DOCUMENT_INVALID_STRUCTURE, $e->getMessage()); } catch (DuplicateException $e) { @@ -3049,7 +3037,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') } // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, $queueForUsage) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3069,10 +3057,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); - } + $relatedCollection = Authorization::skip( + fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId) ); foreach ($related as $relation) { @@ -3107,6 +3093,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $queueForUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection }); @@ -3133,7 +3121,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') ->inject('queueForUsage') ->action(function (string $databaseId, string $collectionId, array $queries, Response $response, Database $dbForProject, string $mode, Usage $queueForUsage) { $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -3143,7 +3130,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } $collection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId)); - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); if ($collection->isEmpty() || (!$collection->getAttribute('enabled', false) && !$isAPIKey && !$isPrivilegedUser)) { throw new Exception(Exception::COLLECTION_NOT_FOUND); @@ -3173,10 +3159,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') $documentId = $cursor->getValue(); - $cursorDocument = Authorization::skip(function() use ($dbForProject, $queueForUsage, $collection, $database, $documentId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId); - }); + $cursorDocument = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId)); if ($cursorDocument->isEmpty()) { throw new Exception(Exception::GENERAL_CURSOR_NOT_FOUND, "Document '{$documentId}' for the 'cursor' value not found."); @@ -3186,13 +3169,10 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } $documents = $dbForProject->find('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $queries); - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - $total = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $queries, APP_LIMIT_COUNT); - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); // Add $collectionId and $databaseId for all documents - $processDocument = (function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, $queueForUsage): bool { + $processDocument = (function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database): bool { if ($document->isEmpty()) { return false; } @@ -3219,10 +3199,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip(function() use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); - }); + $relatedCollection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId)); foreach ($relations as $index => $doc) { if ($doc instanceof Document) { @@ -3273,6 +3250,11 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } } + $queueForUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_READS, 1) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), 1) + ; + $response->dynamic(new Document([ 'total' => $total, 'documents' => $documents, @@ -3305,7 +3287,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen ->action(function (string $databaseId, string $collectionId, string $documentId, array $queries, Response $response, Database $dbForProject, string $mode, Usage $queueForUsage) { $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -3314,10 +3295,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen throw new Exception(Exception::DATABASE_NOT_FOUND); } - $collection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId); - }); + $collection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId)); if ($collection->isEmpty() || (!$collection->getAttribute('enabled', false) && !$isAPIKey && !$isPrivilegedUser)) { throw new Exception(Exception::COLLECTION_NOT_FOUND); @@ -3326,7 +3304,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen try { $queries = Query::parseQueries($queries); $document = $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId, $queries); - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); } catch (AuthorizationException) { throw new Exception(Exception::USER_UNAUTHORIZED); } catch (QueryException $e) { @@ -3338,7 +3315,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen } // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, $queueForUsage) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { if ($document->isEmpty()) { return; } @@ -3362,10 +3339,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); - } + $relatedCollection = Authorization::skip( + fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId) ); foreach ($related as $relation) { @@ -3378,6 +3353,11 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen $processDocument($collection, $document); + $queueForUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_READS, 1) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), 1) + ; + $response->dynamic($document, Response::MODEL_DOCUMENT); }); @@ -3526,10 +3506,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum throw new Exception(Exception::DOCUMENT_MISSING_PAYLOAD); } - $database = Authorization::skip(function () use($queueForUsage, $dbForProject, $databaseId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('databases', $databaseId); - }); + $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -3538,10 +3515,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum throw new Exception(Exception::DATABASE_NOT_FOUND); } - $collection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $collectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId); - }); + $collection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId)); if ($collection->isEmpty() || (!$collection->getAttribute('enabled', false) && !$isAPIKey && !$isPrivilegedUser)) { throw new Exception(Exception::COLLECTION_NOT_FOUND); @@ -3549,10 +3523,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum // Read permission should not be required for update /** @var Document $document */ - $document = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collection, $documentId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId); - }); + $document = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId)); if ($document->isEmpty()) { throw new Exception(Exception::DOCUMENT_NOT_FOUND); @@ -3594,7 +3565,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $data['$permissions'] = $permissions; $newDocument = new Document($data); - $setCollection = (function (Document $collection, Document $document) use (&$setCollection, $dbForProject, $database, $queueForUsage) { + $setCollection = (function (Document $collection, Document $document) use (&$setCollection, $dbForProject, $database) { $relationships = \array_filter( $collection->getAttribute('attributes', []), fn ($attribute) => $attribute->getAttribute('type') === Database::VAR_RELATIONSHIP @@ -3616,10 +3587,9 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); - }); + $relatedCollection = Authorization::skip( + fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId) + ); foreach ($relations as &$relation) { // If the relation is an array it can be either update or create a child document. @@ -3632,15 +3602,17 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $relation = new Document($relation); } if ($relation instanceof Document) { - $oldDocument = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollection, $relation) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId(), $relation->getId()); - }); - + $oldDocument = Authorization::skip(fn () => $dbForProject->getDocument( + 'database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId(), + $relation->getId() + )); $relation->removeAttribute('$collectionId'); $relation->removeAttribute('$databaseId'); // Attribute $collection is required for Utopia. - $relation->setAttribute('$collection', 'database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId()); + $relation->setAttribute( + '$collection', + 'database_' . $database->getInternalId() . '_collection_' . $relatedCollection->getInternalId() + ); if ($oldDocument->isEmpty()) { if (isset($relation['$id']) && $relation['$id'] === 'unique()') { @@ -3662,10 +3634,14 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $setCollection($collection, $newDocument); try { - $document = $dbForProject->withRequestTimestamp($requestTimestamp, function () use ($queueForUsage, $dbForProject, $database, $collection, $document, $newDocument) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_WRITES, 1); - return $dbForProject->updateDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $document->getId(), $newDocument); - }); + $document = $dbForProject->withRequestTimestamp( + $requestTimestamp, + fn () => $dbForProject->updateDocument( + 'database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), + $document->getId(), + $newDocument + ) + ); } catch (AuthorizationException) { throw new Exception(Exception::USER_UNAUTHORIZED); } catch (DuplicateException) { @@ -3677,7 +3653,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum } // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, $queueForUsage) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3697,10 +3673,9 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); - }); + $relatedCollection = Authorization::skip( + fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId) + ); foreach ($related as $relation) { if ($relation instanceof Document) { @@ -3712,6 +3687,11 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $processDocument($collection, $document); + $queueForUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1) + ; + $response->dynamic($document, Response::MODEL_DOCUMENT); $relationships = \array_map( @@ -3762,7 +3742,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->inject('mode') ->action(function (string $databaseId, string $collectionId, string $documentId, ?\DateTime $requestTimestamp, Response $response, Database $dbForProject, Event $queueForEvents, Usage $queueForUsage, string $mode) { $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -3771,36 +3750,32 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu throw new Exception(Exception::DATABASE_NOT_FOUND); } - $collection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId); - }); + $collection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $collectionId)); if ($collection->isEmpty() || (!$collection->getAttribute('enabled', false) && !$isAPIKey && !$isPrivilegedUser)) { throw new Exception(Exception::COLLECTION_NOT_FOUND); } // Read permission should not be required for delete - $document = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $collection, $documentId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId); - }); + $document = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId)); if ($document->isEmpty()) { throw new Exception(Exception::DOCUMENT_NOT_FOUND); } try { - $dbForProject->withRequestTimestamp($requestTimestamp, function () use ($queueForUsage, $dbForProject, $database, $collection, $documentId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_WRITES, 1); - $dbForProject->deleteDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId); + $dbForProject->withRequestTimestamp($requestTimestamp, function () use ($dbForProject, $database, $collection, $documentId) { + $dbForProject->deleteDocument( + 'database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), + $documentId + ); }); } catch (NotFoundException $e) { throw new Exception(Exception::COLLECTION_NOT_FOUND); } // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, $queueForUsage) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3820,10 +3795,9 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); - $relatedCollection = Authorization::skip(function () use ($queueForUsage, $dbForProject, $database, $relatedCollectionId) { - $queueForUsage->addMetric(METRIC_DATABASE_OPERATIONS_READS, 1); - return $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId); - }); + $relatedCollection = Authorization::skip( + fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId) + ); foreach ($related as $relation) { if ($relation instanceof Document) { @@ -3852,6 +3826,8 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->setPayload($response->output($document, Response::MODEL_DOCUMENT), sensitive: $relationships); $queueForUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection $response->noContent(); diff --git a/app/init.php b/app/init.php index d942e056db..cf41bc8049 100644 --- a/app/init.php +++ b/app/init.php @@ -233,7 +233,9 @@ const API_KEY_DYNAMIC = 'dynamic'; const METRIC_TEAMS = 'teams'; const METRIC_USERS = 'users'; const METRIC_DATABASE_OPERATIONS_READS = 'databases.operations.reads'; +const METRIC_DATABASE_ID_OPERATIONS_READS = '{databaseInternalId}.databases.operations.reads'; const METRIC_DATABASE_OPERATIONS_WRITES = 'databases.operations.writes'; +const METRIC_DATABASE_ID_OPERATIONS_WRITES = '{databaseInternalId}.databases.operations.writes'; const METRIC_WEBHOOKS_SENT = 'webhooks.events.sent'; const METRIC_WEBHOOKS_FAILED = 'webhooks.events.failed'; const METRIC_WEBHOOK_ID_SENT = '{webhookInternalId}.webhooks.events.sent'; From c7841bb36ef0b503a6d1001ecfc78ca8ea360e44 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 2 Jan 2025 09:59:59 +0200 Subject: [PATCH 13/74] database crud usage addition --- app/controllers/api/databases.php | 3 --- 1 file changed, 3 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index ea17065d45..0ab84bf0fc 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3121,7 +3121,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') ->inject('queueForUsage') ->action(function (string $databaseId, string $collectionId, array $queries, Response $response, Database $dbForProject, string $mode, Usage $queueForUsage) { $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); - $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -3285,9 +3284,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen ->inject('mode') ->inject('queueForUsage') ->action(function (string $databaseId, string $collectionId, string $documentId, array $queries, Response $response, Database $dbForProject, string $mode, Usage $queueForUsage) { - $database = Authorization::skip(fn () => $dbForProject->getDocument('databases', $databaseId)); - $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); From fbbc9312eaf3ce85042efd74eb09f3e708394d7f Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 2 Jan 2025 10:09:44 +0200 Subject: [PATCH 14/74] database crud usage addition --- app/controllers/api/databases.php | 61 +++++++++++++++++++++++-------- 1 file changed, 46 insertions(+), 15 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 0ab84bf0fc..1249ccb2fa 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3036,8 +3036,10 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') throw new Exception(Exception::COLLECTION_NOT_FOUND); } + $operations = 1; + // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3049,6 +3051,10 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); + if (\in_array(\gettype($related), ['array', 'object'])) { + $operations++; + } + if (empty($related)) { continue; } @@ -3093,8 +3099,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $queueForUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1) + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection }); @@ -3170,8 +3176,10 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') $documents = $dbForProject->find('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $queries); $total = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $queries, APP_LIMIT_COUNT); + $operations = 1; + // Add $collectionId and $databaseId for all documents - $processDocument = (function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database): bool { + $processDocument = (function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations): bool { if ($document->isEmpty()) { return false; } @@ -3188,6 +3196,10 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); + if (\in_array(\gettype($related), ['array', 'object'])) { + $operations++; + } + if (empty($related)) { continue; } @@ -3198,6 +3210,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); + // todo: Use local cache for this getDocument $relatedCollection = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId)); foreach ($relations as $index => $doc) { @@ -3250,8 +3263,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } $queueForUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_READS, 1) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), 1) + ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ; $response->dynamic(new Document([ @@ -3311,8 +3324,10 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen throw new Exception(Exception::DOCUMENT_NOT_FOUND); } + $operations = 1; + // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { if ($document->isEmpty()) { return; } @@ -3328,6 +3343,10 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); + if (\in_array(\gettype($related), ['array', 'object'])) { + $operations++; + } + if (empty($related)) { continue; } @@ -3351,8 +3370,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen $processDocument($collection, $document); $queueForUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_READS, 1) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), 1) + ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ; $response->dynamic($document, Response::MODEL_DOCUMENT); @@ -3649,8 +3668,10 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum throw new Exception(Exception::COLLECTION_NOT_FOUND); } + $operations = 1; + // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3662,6 +3683,10 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); + if (\in_array(\gettype($related), ['array', 'object'])) { + $operations++; + } + if (empty($related)) { continue; } @@ -3685,8 +3710,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $processDocument($collection, $document); $queueForUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1) + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) ; $response->dynamic($document, Response::MODEL_DOCUMENT); @@ -3771,8 +3796,10 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu throw new Exception(Exception::COLLECTION_NOT_FOUND); } + $operations = 1; + // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3784,6 +3811,10 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); + if (\in_array(\gettype($related), ['array', 'object'])) { + $operations++; + } + if (empty($related)) { continue; } @@ -3823,8 +3854,8 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->setPayload($response->output($document, Response::MODEL_DOCUMENT), sensitive: $relationships); $queueForUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1) + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection $response->noContent(); From c0d48efe18b7ea40f654a7303c7f2192a4e5bf60 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 2 Jan 2025 10:29:55 +0200 Subject: [PATCH 15/74] composer --- app/controllers/api/databases.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 1249ccb2fa..a0fea5e293 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3265,7 +3265,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') $queueForUsage ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) - ; + ; $response->dynamic(new Document([ 'total' => $total, @@ -3372,7 +3372,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen $queueForUsage ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) - ; + ; $response->dynamic($document, Response::MODEL_DOCUMENT); }); From e7e095907cdf58a2a5627109f9b164e60a34e021 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 2 Jan 2025 10:44:56 +0200 Subject: [PATCH 16/74] composer --- app/controllers/api/databases.php | 4 ++-- app/init.php | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index a0fea5e293..1249ccb2fa 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3265,7 +3265,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') $queueForUsage ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) - ; + ; $response->dynamic(new Document([ 'total' => $total, @@ -3372,7 +3372,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen $queueForUsage ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) - ; + ; $response->dynamic($document, Response::MODEL_DOCUMENT); }); diff --git a/app/init.php b/app/init.php index cf41bc8049..2f83e7c3b2 100644 --- a/app/init.php +++ b/app/init.php @@ -232,9 +232,9 @@ const API_KEY_DYNAMIC = 'dynamic'; // Usage metrics const METRIC_TEAMS = 'teams'; const METRIC_USERS = 'users'; -const METRIC_DATABASE_OPERATIONS_READS = 'databases.operations.reads'; +const METRIC_DATABASES_OPERATIONS_READS = 'databases.operations.reads'; const METRIC_DATABASE_ID_OPERATIONS_READS = '{databaseInternalId}.databases.operations.reads'; -const METRIC_DATABASE_OPERATIONS_WRITES = 'databases.operations.writes'; +const METRIC_DATABASES_OPERATIONS_WRITES = 'databases.operations.writes'; const METRIC_DATABASE_ID_OPERATIONS_WRITES = '{databaseInternalId}.databases.operations.writes'; const METRIC_WEBHOOKS_SENT = 'webhooks.events.sent'; const METRIC_WEBHOOKS_FAILED = 'webhooks.events.failed'; From 504b7276cb3fc0c94967fd177151a629e629b50b Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 2 Jan 2025 11:02:53 +0200 Subject: [PATCH 17/74] composer --- app/controllers/api/databases.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 1249ccb2fa..a0fea5e293 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3265,7 +3265,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') $queueForUsage ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) - ; + ; $response->dynamic(new Document([ 'total' => $total, @@ -3372,7 +3372,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen $queueForUsage ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) - ; + ; $response->dynamic($document, Response::MODEL_DOCUMENT); }); From 63cf093020451c8fba1668443e4a5432621d2ebe Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 2 Jan 2025 13:53:57 +0200 Subject: [PATCH 18/74] file transformations usage --- app/controllers/api/storage.php | 15 ++++++++++++++- app/init.php | 10 ++++++---- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 7d4361a192..97fb5cc711 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -6,6 +6,7 @@ use Appwrite\Auth\Auth; use Appwrite\ClamAV\Network; use Appwrite\Event\Delete; use Appwrite\Event\Event; +use Appwrite\Event\Usage; use Appwrite\Extend\Exception; use Appwrite\OpenSSL\OpenSSL; use Appwrite\Utopia\Database\Validator\CustomId; @@ -886,7 +887,8 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') ->inject('mode') ->inject('deviceForFiles') ->inject('deviceForLocal') - ->action(function (string $bucketId, string $fileId, int $width, int $height, string $gravity, int $quality, int $borderWidth, string $borderColor, int $borderRadius, float $opacity, int $rotation, string $background, string $output, Request $request, Response $response, Document $project, Database $dbForProject, string $mode, Device $deviceForFiles, Device $deviceForLocal) { + ->inject('queueForUsage') + ->action(function (string $bucketId, string $fileId, int $width, int $height, string $gravity, int $quality, int $borderWidth, string $borderColor, int $borderRadius, float $opacity, int $rotation, string $background, string $output, Request $request, Response $response, Document $project, Database $dbForProject, string $mode, Device $deviceForFiles, Device $deviceForLocal, Usage $queueForUsage) { if (!\extension_loaded('imagick')) { throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Imagick extension is missing'); @@ -1014,6 +1016,17 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') $contentType = (\array_key_exists($output, $outputs)) ? $outputs[$output] : $outputs['jpg']; + foreach([$width, $height, $gravity, $quality, $borderWidth, $borderColor, $borderRadius, $opacity, $rotation, $background, $output] as $param){ + if(!empty($param)){ + $queueForUsage + ->addMetric(METRIC_FILES_TRANSFORMATIONS, 1) + ->addMetric(str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_TRANSFORMATIONS), 1) + ; + break; + } + } + + $response ->addHeader('Cache-Control', 'private, max-age=2592000') // 30 days ->setContentType($contentType) diff --git a/app/init.php b/app/init.php index 2f83e7c3b2..5e25fa905f 100644 --- a/app/init.php +++ b/app/init.php @@ -232,10 +232,6 @@ const API_KEY_DYNAMIC = 'dynamic'; // Usage metrics const METRIC_TEAMS = 'teams'; const METRIC_USERS = 'users'; -const METRIC_DATABASES_OPERATIONS_READS = 'databases.operations.reads'; -const METRIC_DATABASE_ID_OPERATIONS_READS = '{databaseInternalId}.databases.operations.reads'; -const METRIC_DATABASES_OPERATIONS_WRITES = 'databases.operations.writes'; -const METRIC_DATABASE_ID_OPERATIONS_WRITES = '{databaseInternalId}.databases.operations.writes'; const METRIC_WEBHOOKS_SENT = 'webhooks.events.sent'; const METRIC_WEBHOOKS_FAILED = 'webhooks.events.failed'; const METRIC_WEBHOOK_ID_SENT = '{webhookInternalId}.webhooks.events.sent'; @@ -263,9 +259,15 @@ const METRIC_DOCUMENTS = 'documents'; const METRIC_DATABASE_ID_DOCUMENTS = '{databaseInternalId}.documents'; const METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS = '{databaseInternalId}.{collectionInternalId}.documents'; const METRIC_DATABASE_ID_COLLECTION_ID_STORAGE = '{databaseInternalId}.{collectionInternalId}.databases.storage'; +const METRIC_DATABASES_OPERATIONS_READS = 'databases.operations.reads'; +const METRIC_DATABASE_ID_OPERATIONS_READS = '{databaseInternalId}.databases.operations.reads'; +const METRIC_DATABASES_OPERATIONS_WRITES = 'databases.operations.writes'; +const METRIC_DATABASE_ID_OPERATIONS_WRITES = '{databaseInternalId}.databases.operations.writes'; const METRIC_BUCKETS = 'buckets'; const METRIC_FILES = 'files'; const METRIC_FILES_STORAGE = 'files.storage'; +const METRIC_FILES_TRANSFORMATIONS = 'files.transformations'; +const METRIC_BUCKET_ID_FILES_TRANSFORMATIONS = '{bucketInternalId}.files.transformations'; const METRIC_BUCKET_ID_FILES = '{bucketInternalId}.files'; const METRIC_BUCKET_ID_FILES_STORAGE = '{bucketInternalId}.files.storage'; const METRIC_FUNCTIONS = 'functions'; From 6b45293f7828f36ee590019a1593c84a6b175747 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 2 Jan 2025 13:55:30 +0200 Subject: [PATCH 19/74] file transformations usage --- app/controllers/api/storage.php | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 97fb5cc711..8431fb19e3 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -1016,14 +1016,14 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') $contentType = (\array_key_exists($output, $outputs)) ? $outputs[$output] : $outputs['jpg']; - foreach([$width, $height, $gravity, $quality, $borderWidth, $borderColor, $borderRadius, $opacity, $rotation, $background, $output] as $param){ - if(!empty($param)){ - $queueForUsage - ->addMetric(METRIC_FILES_TRANSFORMATIONS, 1) - ->addMetric(str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_TRANSFORMATIONS), 1) - ; - break; - } + foreach ([$width, $height, $gravity, $quality, $borderWidth, $borderColor, $borderRadius, $opacity, $rotation, $background, $output] as $parameter) { + if (!empty($parameter)) { + $queueForUsage + ->addMetric(METRIC_FILES_TRANSFORMATIONS, 1) + ->addMetric(str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_TRANSFORMATIONS), 1) + ; + break; + } } From bdb93cd3234a7507e365af2f95210715b752fe84 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 2 Jan 2025 19:03:28 +0530 Subject: [PATCH 20/74] Update general.php --- app/controllers/general.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index bb60b01ddd..db9df71341 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -835,7 +835,7 @@ App::error() $adapter = new Sentry($projectId, $key, $host); $logger = new Logger($adapter); - $logger->setSample(0.04); + $logger->setSample(0.01); $publish = true; } else { throw new \Exception('Invalid experimental logging provider'); From c5a49cc5375ab4fdc512fe9207802bd6f4fe18df Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 2 Jan 2025 16:45:21 +0200 Subject: [PATCH 21/74] added operations debug header --- app/controllers/api/databases.php | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index a0fea5e293..c3f74389cc 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3077,6 +3077,13 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $processDocument($collection, $document); + $queueForUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) + ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection + + + $response->addHeader('X-Debug-Operations', $operations); $response ->setStatusCode(Response::STATUS_CODE_CREATED) ->dynamic($document, Response::MODEL_DOCUMENT); @@ -3098,10 +3105,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') ->setPayload($response->getPayload(), sensitive: $relationships); - $queueForUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) - ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection }); App::get('/v1/databases/:databaseId/collections/:collectionId/documents') @@ -3267,6 +3270,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ; + $response->addHeader('X-Debug-Operations', $operations); $response->dynamic(new Document([ 'total' => $total, 'documents' => $documents, @@ -3374,6 +3378,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ; + $response->addHeader('X-Debug-Operations', $operations); $response->dynamic($document, Response::MODEL_DOCUMENT); }); @@ -3714,6 +3719,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) ; + $response->addHeader('X-Debug-Operations', $operations); $response->dynamic($document, Response::MODEL_DOCUMENT); $relationships = \array_map( @@ -3858,6 +3864,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection + $response->addHeader('X-Debug-Operations', $operations); $response->noContent(); }); From 4acc2736e40a21b1c6ecfb9e97b1298086888b25 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 2 Jan 2025 16:46:38 +0200 Subject: [PATCH 22/74] added operations debug header --- app/controllers/api/storage.php | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 8431fb19e3..e92b55f140 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -1016,16 +1016,10 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview') $contentType = (\array_key_exists($output, $outputs)) ? $outputs[$output] : $outputs['jpg']; - foreach ([$width, $height, $gravity, $quality, $borderWidth, $borderColor, $borderRadius, $opacity, $rotation, $background, $output] as $parameter) { - if (!empty($parameter)) { - $queueForUsage - ->addMetric(METRIC_FILES_TRANSFORMATIONS, 1) - ->addMetric(str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_TRANSFORMATIONS), 1) - ; - break; - } - } - + $queueForUsage + ->addMetric(METRIC_FILES_TRANSFORMATIONS, 1) + ->addMetric(str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_TRANSFORMATIONS), 1) + ; $response ->addHeader('Cache-Control', 'private, max-age=2592000') // 30 days From 06a5876f35e86b48b4883f26651b27069d6e51e4 Mon Sep 17 00:00:00 2001 From: fogelito Date: Thu, 2 Jan 2025 17:06:22 +0200 Subject: [PATCH 23/74] No throwing Exception --- src/Appwrite/Platform/Workers/Databases.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Databases.php b/src/Appwrite/Platform/Workers/Databases.php index 9c11dd4090..74c1250505 100644 --- a/src/Appwrite/Platform/Workers/Databases.php +++ b/src/Appwrite/Platform/Workers/Databases.php @@ -284,8 +284,9 @@ class Databases extends Action $dbForProject->deleteDocument('attributes', $relatedAttribute->getId()); } - throw $e; - + /** + * No throwing Exception + */ } catch (\Throwable $e) { Console::error($e->getMessage()); From bd083ac2805aeb66ee5b11c65cbdccd946edb829 Mon Sep 17 00:00:00 2001 From: fogelito Date: Thu, 2 Jan 2025 17:54:47 +0200 Subject: [PATCH 24/74] database 0.53.29 --- composer.lock | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/composer.lock b/composer.lock index ce5f358619..dbe9e079b5 100644 --- a/composer.lock +++ b/composer.lock @@ -2453,16 +2453,16 @@ }, { "name": "symfony/http-client", - "version": "v7.2.1", + "version": "v7.2.2", "source": { "type": "git", "url": "https://github.com/symfony/http-client.git", - "reference": "ff4df2b68d1c67abb9fef146e6540ea16b58d99e" + "reference": "339ba21476eb184290361542f732ad12c97591ec" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-client/zipball/ff4df2b68d1c67abb9fef146e6540ea16b58d99e", - "reference": "ff4df2b68d1c67abb9fef146e6540ea16b58d99e", + "url": "https://api.github.com/repos/symfony/http-client/zipball/339ba21476eb184290361542f732ad12c97591ec", + "reference": "339ba21476eb184290361542f732ad12c97591ec", "shasum": "" }, "require": { @@ -2528,7 +2528,7 @@ "http" ], "support": { - "source": "https://github.com/symfony/http-client/tree/v7.2.1" + "source": "https://github.com/symfony/http-client/tree/v7.2.2" }, "funding": [ { @@ -2544,7 +2544,7 @@ "type": "tidelift" } ], - "time": "2024-12-07T08:50:44+00:00" + "time": "2024-12-30T18:35:15+00:00" }, { "name": "symfony/http-client-contracts", @@ -5126,16 +5126,16 @@ }, { "name": "laravel/pint", - "version": "v1.18.3", + "version": "v1.19.0", "source": { "type": "git", "url": "https://github.com/laravel/pint.git", - "reference": "cef51821608239040ab841ad6e1c6ae502ae3026" + "reference": "8169513746e1bac70c85d6ea1524d9225d4886f0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/pint/zipball/cef51821608239040ab841ad6e1c6ae502ae3026", - "reference": "cef51821608239040ab841ad6e1c6ae502ae3026", + "url": "https://api.github.com/repos/laravel/pint/zipball/8169513746e1bac70c85d6ea1524d9225d4886f0", + "reference": "8169513746e1bac70c85d6ea1524d9225d4886f0", "shasum": "" }, "require": { @@ -5146,10 +5146,10 @@ "php": "^8.1.0" }, "require-dev": { - "friendsofphp/php-cs-fixer": "^3.65.0", - "illuminate/view": "^10.48.24", - "larastan/larastan": "^2.9.11", - "laravel-zero/framework": "^10.4.0", + "friendsofphp/php-cs-fixer": "^3.66.0", + "illuminate/view": "^10.48.25", + "larastan/larastan": "^2.9.12", + "laravel-zero/framework": "^10.48.25", "mockery/mockery": "^1.6.12", "nunomaduro/termwind": "^1.17.0", "pestphp/pest": "^2.36.0" @@ -5188,7 +5188,7 @@ "issues": "https://github.com/laravel/pint/issues", "source": "https://github.com/laravel/pint" }, - "time": "2024-11-26T15:34:00+00:00" + "time": "2024-12-30T16:20:10+00:00" }, { "name": "matthiasmullie/minify", @@ -7735,16 +7735,16 @@ }, { "name": "symfony/finder", - "version": "v7.2.0", + "version": "v7.2.2", "source": { "type": "git", "url": "https://github.com/symfony/finder.git", - "reference": "6de263e5868b9a137602dd1e33e4d48bfae99c49" + "reference": "87a71856f2f56e4100373e92529eed3171695cfb" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/finder/zipball/6de263e5868b9a137602dd1e33e4d48bfae99c49", - "reference": "6de263e5868b9a137602dd1e33e4d48bfae99c49", + "url": "https://api.github.com/repos/symfony/finder/zipball/87a71856f2f56e4100373e92529eed3171695cfb", + "reference": "87a71856f2f56e4100373e92529eed3171695cfb", "shasum": "" }, "require": { @@ -7779,7 +7779,7 @@ "description": "Finds files and directories via an intuitive fluent interface", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/finder/tree/v7.2.0" + "source": "https://github.com/symfony/finder/tree/v7.2.2" }, "funding": [ { @@ -7795,7 +7795,7 @@ "type": "tidelift" } ], - "time": "2024-10-23T06:56:12+00:00" + "time": "2024-12-30T19:00:17+00:00" }, { "name": "symfony/options-resolver", @@ -8556,7 +8556,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": {}, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { From e8e39dbd568a202eb7305acc9364d761ae1d2c9b Mon Sep 17 00:00:00 2001 From: fogelito Date: Thu, 2 Jan 2025 17:57:46 +0200 Subject: [PATCH 25/74] Revert lock --- composer.lock | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/composer.lock b/composer.lock index dbe9e079b5..ce5f358619 100644 --- a/composer.lock +++ b/composer.lock @@ -2453,16 +2453,16 @@ }, { "name": "symfony/http-client", - "version": "v7.2.2", + "version": "v7.2.1", "source": { "type": "git", "url": "https://github.com/symfony/http-client.git", - "reference": "339ba21476eb184290361542f732ad12c97591ec" + "reference": "ff4df2b68d1c67abb9fef146e6540ea16b58d99e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-client/zipball/339ba21476eb184290361542f732ad12c97591ec", - "reference": "339ba21476eb184290361542f732ad12c97591ec", + "url": "https://api.github.com/repos/symfony/http-client/zipball/ff4df2b68d1c67abb9fef146e6540ea16b58d99e", + "reference": "ff4df2b68d1c67abb9fef146e6540ea16b58d99e", "shasum": "" }, "require": { @@ -2528,7 +2528,7 @@ "http" ], "support": { - "source": "https://github.com/symfony/http-client/tree/v7.2.2" + "source": "https://github.com/symfony/http-client/tree/v7.2.1" }, "funding": [ { @@ -2544,7 +2544,7 @@ "type": "tidelift" } ], - "time": "2024-12-30T18:35:15+00:00" + "time": "2024-12-07T08:50:44+00:00" }, { "name": "symfony/http-client-contracts", @@ -5126,16 +5126,16 @@ }, { "name": "laravel/pint", - "version": "v1.19.0", + "version": "v1.18.3", "source": { "type": "git", "url": "https://github.com/laravel/pint.git", - "reference": "8169513746e1bac70c85d6ea1524d9225d4886f0" + "reference": "cef51821608239040ab841ad6e1c6ae502ae3026" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/pint/zipball/8169513746e1bac70c85d6ea1524d9225d4886f0", - "reference": "8169513746e1bac70c85d6ea1524d9225d4886f0", + "url": "https://api.github.com/repos/laravel/pint/zipball/cef51821608239040ab841ad6e1c6ae502ae3026", + "reference": "cef51821608239040ab841ad6e1c6ae502ae3026", "shasum": "" }, "require": { @@ -5146,10 +5146,10 @@ "php": "^8.1.0" }, "require-dev": { - "friendsofphp/php-cs-fixer": "^3.66.0", - "illuminate/view": "^10.48.25", - "larastan/larastan": "^2.9.12", - "laravel-zero/framework": "^10.48.25", + "friendsofphp/php-cs-fixer": "^3.65.0", + "illuminate/view": "^10.48.24", + "larastan/larastan": "^2.9.11", + "laravel-zero/framework": "^10.4.0", "mockery/mockery": "^1.6.12", "nunomaduro/termwind": "^1.17.0", "pestphp/pest": "^2.36.0" @@ -5188,7 +5188,7 @@ "issues": "https://github.com/laravel/pint/issues", "source": "https://github.com/laravel/pint" }, - "time": "2024-12-30T16:20:10+00:00" + "time": "2024-11-26T15:34:00+00:00" }, { "name": "matthiasmullie/minify", @@ -7735,16 +7735,16 @@ }, { "name": "symfony/finder", - "version": "v7.2.2", + "version": "v7.2.0", "source": { "type": "git", "url": "https://github.com/symfony/finder.git", - "reference": "87a71856f2f56e4100373e92529eed3171695cfb" + "reference": "6de263e5868b9a137602dd1e33e4d48bfae99c49" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/finder/zipball/87a71856f2f56e4100373e92529eed3171695cfb", - "reference": "87a71856f2f56e4100373e92529eed3171695cfb", + "url": "https://api.github.com/repos/symfony/finder/zipball/6de263e5868b9a137602dd1e33e4d48bfae99c49", + "reference": "6de263e5868b9a137602dd1e33e4d48bfae99c49", "shasum": "" }, "require": { @@ -7779,7 +7779,7 @@ "description": "Finds files and directories via an intuitive fluent interface", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/finder/tree/v7.2.2" + "source": "https://github.com/symfony/finder/tree/v7.2.0" }, "funding": [ { @@ -7795,7 +7795,7 @@ "type": "tidelift" } ], - "time": "2024-12-30T19:00:17+00:00" + "time": "2024-10-23T06:56:12+00:00" }, { "name": "symfony/options-resolver", @@ -8556,7 +8556,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": {}, "prefer-stable": false, "prefer-lowest": false, "platform": { From 0203b67b28653e924418b5951150a6c25267bf72 Mon Sep 17 00:00:00 2001 From: Darshan Date: Fri, 3 Jan 2025 10:54:56 +0530 Subject: [PATCH 26/74] add: logs init. --- app/init.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/app/init.php b/app/init.php index c6847610c2..af83368082 100644 --- a/app/init.php +++ b/app/init.php @@ -869,6 +869,12 @@ $register->set('pools', function () { 'multiple' => true, 'schemes' => ['mariadb', 'mysql'], ], + 'logs' => [ + 'type' => 'database', + 'dsns' => System::getEnv('_APP_CONNECTIONS_DB_LOGS', $fallbackForDB), + 'multiple' => true, + 'schemes' => ['mariadb', 'mysql'], + ], 'queue' => [ 'type' => 'queue', 'dsns' => $fallbackForRedis, From 31f8950b45c5daa754aefb9416cc77e3b49e4329 Mon Sep 17 00:00:00 2001 From: Darshan Date: Fri, 3 Jan 2025 15:30:55 +0530 Subject: [PATCH 27/74] add: hostname to audits. --- app/controllers/shared/api.php | 1 + src/Appwrite/Event/Audit.php | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 012dd13c73..182151a6c3 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -506,6 +506,7 @@ App::init() ->setMode($mode) ->setUserAgent($request->getUserAgent('')) ->setIP($request->getIP()) + ->setHostname($request->getHostname()) ->setEvent($route->getLabel('audits.event', '')) ->setProject($project) ->setUser($user); diff --git a/src/Appwrite/Event/Audit.php b/src/Appwrite/Event/Audit.php index 17506bfe6c..406f64b370 100644 --- a/src/Appwrite/Event/Audit.php +++ b/src/Appwrite/Event/Audit.php @@ -11,6 +11,7 @@ class Audit extends Event protected string $mode = ''; protected string $userAgent = ''; protected string $ip = ''; + protected string $hostname = ''; public function __construct(protected Connection $connection) { @@ -113,6 +114,30 @@ class Audit extends Event return $this->ip; } + /** + * Set the hostname. + * + * @param string $hostname + * + * @return self + */ + public function setHostname(string $hostname): self + { + $this->hostname = $hostname; + + return $this; + } + + /** + * Get the hostname. + * + * @return string + */ + public function getHostname(): string + { + return $this->hostname; + } + /** * Executes the event and sends it to the audit worker. * @@ -136,6 +161,7 @@ class Audit extends Event 'ip' => $this->ip, 'userAgent' => $this->userAgent, 'event' => $this->event, + 'hostname' => $this->hostname ]); } } From 1a73847c4798e77923da52a1f231f9fc7bb0cf70 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 6 Jan 2025 11:28:26 +0200 Subject: [PATCH 28/74] added operations debug header --- app/controllers/api/databases.php | 67 ++++++++++++++++++++++--------- 1 file changed, 49 insertions(+), 18 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index c3f74389cc..16258e9972 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -2944,7 +2944,9 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $data['$permissions'] = $permissions; $document = new Document($data); - $checkPermissions = function (Document $collection, Document $document, string $permission) use (&$checkPermissions, $dbForProject, $database) { + $operations = 1; + + $checkPermissions = function (Document $collection, Document $document, string $permission) use (&$checkPermissions, $dbForProject, $database, &$operations) { $documentSecurity = $collection->getAttribute('documentSecurity', false); $validator = new Authorization($permission); @@ -2976,10 +2978,13 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') if ($isList) { $relations = $related; + $operations += count($related); } else { $relations = [$related]; + $operations++; } + $relatedCollectionId = $relationship->getAttribute('relatedCollection'); $relatedCollection = Authorization::skip( fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId) @@ -3026,6 +3031,11 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $checkPermissions($collection, $document, Database::PERMISSION_CREATE); + $queueForUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) + ; + try { $document = $dbForProject->createDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $document); } catch (StructureException $e) { @@ -3036,7 +3046,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') throw new Exception(Exception::COLLECTION_NOT_FOUND); } - $operations = 1; + $operations = 0; // Add $collectionId and $databaseId for all documents $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { @@ -3078,8 +3088,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $processDocument($collection, $document); $queueForUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) + ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection @@ -3238,6 +3248,11 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') $processDocument($collection, $document); } + $queueForUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) + ; + $select = \array_reduce($queries, function ($result, $query) { return $result || ($query->getMethod() === Query::TYPE_SELECT); }, false); @@ -3265,11 +3280,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } } - $queueForUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) - ; - $response->addHeader('X-Debug-Operations', $operations); $response->dynamic(new Document([ 'total' => $total, @@ -3586,7 +3596,10 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $data['$permissions'] = $permissions; $newDocument = new Document($data); - $setCollection = (function (Document $collection, Document $document) use (&$setCollection, $dbForProject, $database) { + + $operations = 1; + + $setCollection = (function (Document $collection, Document $document) use (&$setCollection, $dbForProject, $database, &$operations) { $relationships = \array_filter( $collection->getAttribute('attributes', []), fn ($attribute) => $attribute->getAttribute('type') === Database::VAR_RELATIONSHIP @@ -3595,6 +3608,13 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); + /** + * Write relationship counts + */ + if (\in_array(\gettype($related), ['array', 'object'])) { + + } + if (empty($related)) { continue; } @@ -3603,8 +3623,10 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum if ($isList) { $relations = $related; + $operations += count($related); } else { $relations = [$related]; + $operations++; } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); @@ -3654,6 +3676,11 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $setCollection($collection, $newDocument); + $queueForUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) + ; + try { $document = $dbForProject->withRequestTimestamp( $requestTimestamp, @@ -3673,7 +3700,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum throw new Exception(Exception::COLLECTION_NOT_FOUND); } - $operations = 1; + $operations = 0; // Add $collectionId and $databaseId for all documents $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { @@ -3715,8 +3742,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $processDocument($collection, $document); $queueForUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) + ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ; $response->addHeader('X-Debug-Operations', $operations); @@ -3843,6 +3870,15 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu $processDocument($collection, $document); + $queueForUsage + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1) + ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) + ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection + + + $relationships = \array_map( fn ($document) => $document->getAttribute('key'), \array_filter( @@ -3859,11 +3895,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->setContext('database', $database) ->setPayload($response->output($document, Response::MODEL_DOCUMENT), sensitive: $relationships); - $queueForUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) - ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection - $response->addHeader('X-Debug-Operations', $operations); $response->noContent(); }); From 895e79e7369715be206e01cfdf8fe3eca3376450 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 6 Jan 2025 12:25:56 +0200 Subject: [PATCH 29/74] Operations --- app/controllers/api/databases.php | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 16258e9972..ac41f2d2fe 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -2944,7 +2944,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $data['$permissions'] = $permissions; $document = new Document($data); - $operations = 1; + $operations = 1; // For root createDocument $checkPermissions = function (Document $collection, Document $document, string $permission) use (&$checkPermissions, $dbForProject, $database, &$operations) { $documentSecurity = $collection->getAttribute('documentSecurity', false); @@ -3046,7 +3046,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') throw new Exception(Exception::COLLECTION_NOT_FOUND); } - $operations = 0; + $operations = 0; // Since the Document is sent through post no getDocument used // Add $collectionId and $databaseId for all documents $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { @@ -3597,7 +3597,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $newDocument = new Document($data); - $operations = 1; + $operations = 1; // Root updateDocument $setCollection = (function (Document $collection, Document $document) use (&$setCollection, $dbForProject, $database, &$operations) { $relationships = \array_filter( @@ -3700,7 +3700,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum throw new Exception(Exception::COLLECTION_NOT_FOUND); } - $operations = 0; + $operations = 1; // Since we read original Document // Add $collectionId and $databaseId for all documents $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { @@ -3811,6 +3811,8 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu throw new Exception(Exception::COLLECTION_NOT_FOUND); } + $operations = 1; // Read original Document + // Read permission should not be required for delete $document = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId)); @@ -3829,8 +3831,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu throw new Exception(Exception::COLLECTION_NOT_FOUND); } - $operations = 1; - // Add $collectionId and $databaseId for all documents $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { $document->setAttribute('$databaseId', $database->getId()); @@ -3877,8 +3877,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection - - $relationships = \array_map( fn ($document) => $document->getAttribute('key'), \array_filter( From 1bc419b6a1dc697cb106b47ca674e4f5b661db48 Mon Sep 17 00:00:00 2001 From: Fabian Gruber Date: Mon, 6 Jan 2025 13:48:01 +0100 Subject: [PATCH 30/74] feat(swoole): allow configuration override of available cpus --- app/http.php | 2 +- app/init.php | 2 +- app/realtime.php | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/http.php b/app/http.php index 61afce3eae..3a7562ffd1 100644 --- a/app/http.php +++ b/app/http.php @@ -42,7 +42,7 @@ $http = new Server( ); $payloadSize = 12 * (1024 * 1024); // 12MB - adding slight buffer for headers and other data that might be sent with the payload - update later with valid testing -$totalWorkers = swoole_cpu_num() * intval(System::getEnv('_APP_WORKER_PER_CORE', 6)); +$totalWorkers = intval(System::getEnv('_APP_CPU_NUM', swoole_cpu_num())) * intval(System::getEnv('_APP_WORKER_PER_CORE', 6)); $http ->set([ diff --git a/app/init.php b/app/init.php index c6847610c2..7f95ac4c25 100644 --- a/app/init.php +++ b/app/init.php @@ -895,7 +895,7 @@ $register->set('pools', function () { $multiprocessing = System::getEnv('_APP_SERVER_MULTIPROCESS', 'disabled') === 'enabled'; if ($multiprocessing) { - $workerCount = swoole_cpu_num() * intval(System::getEnv('_APP_WORKER_PER_CORE', 6)); + $workerCount = intval(System::getEnv('_APP_CPU_NUM', swoole_cpu_num())) * intval(System::getEnv('_APP_WORKER_PER_CORE', 6)); } else { $workerCount = 1; } diff --git a/app/realtime.php b/app/realtime.php index 4f87e4dea1..86f9c85fdd 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -193,7 +193,7 @@ $stats->create(); $containerId = uniqid(); $statsDocument = null; -$workerNumber = swoole_cpu_num() * intval(System::getEnv('_APP_WORKER_PER_CORE', 6)); +$workerNumber = intval(System::getEnv('_APP_CPU_NUM', swoole_cpu_num())) * intval(System::getEnv('_APP_WORKER_PER_CORE', 6)); $adapter = new Adapter\Swoole(port: System::getEnv('PORT', 80)); $adapter From 0814fe49e204221ac86f021f325cc1c0fd92cd25 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 6 Jan 2025 16:50:23 +0200 Subject: [PATCH 31/74] added operations debug header --- app/controllers/api/databases.php | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 16258e9972..8f59033795 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3031,11 +3031,14 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $checkPermissions($collection, $document, Database::PERMISSION_CREATE); + $queueForUsage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) ; + $response->addHeader('X-Debug-operations-write', $operations); + try { $document = $dbForProject->createDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $document); } catch (StructureException $e) { @@ -3092,8 +3095,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection - - $response->addHeader('X-Debug-Operations', $operations); + $response->addHeader('X-Debug-operations-read', $operations); $response ->setStatusCode(Response::STATUS_CODE_CREATED) ->dynamic($document, Response::MODEL_DOCUMENT); @@ -3253,6 +3255,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ; + $response->addHeader('X-Debug-operations-reads', $operations); + $select = \array_reduce($queries, function ($result, $query) { return $result || ($query->getMethod() === Query::TYPE_SELECT); }, false); @@ -3280,7 +3284,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') } } - $response->addHeader('X-Debug-Operations', $operations); $response->dynamic(new Document([ 'total' => $total, 'documents' => $documents, @@ -3388,7 +3391,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ; - $response->addHeader('X-Debug-Operations', $operations); + $response->addHeader('X-Debug-operations-reads', $operations); $response->dynamic($document, Response::MODEL_DOCUMENT); }); @@ -3681,6 +3684,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) ; + $response->addHeader('X-Debug-operations-writes', $operations); + try { $document = $dbForProject->withRequestTimestamp( $requestTimestamp, @@ -3746,7 +3751,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ; - $response->addHeader('X-Debug-Operations', $operations); + $response->addHeader('X-Debug-operations-reads', $operations); + $response->dynamic($document, Response::MODEL_DOCUMENT); $relationships = \array_map( From f70222d41a532d00bb2db34a767bc64c7f2e96f4 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 6 Jan 2025 17:19:39 +0200 Subject: [PATCH 32/74] added operations debug header --- app/controllers/api/databases.php | 35 ++++++++++--------------------- 1 file changed, 11 insertions(+), 24 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 3f8fe7cfdc..fa12087463 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -2944,8 +2944,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $data['$permissions'] = $permissions; $document = new Document($data); - $operations = 1; // For root createDocument - + $operations = 1; $checkPermissions = function (Document $collection, Document $document, string $permission) use (&$checkPermissions, $dbForProject, $database, &$operations) { $documentSecurity = $collection->getAttribute('documentSecurity', false); $validator = new Authorization($permission); @@ -3031,14 +3030,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $checkPermissions($collection, $document, Database::PERMISSION_CREATE); - - $queueForUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) - ; - - $response->addHeader('X-Debug-operations-write', $operations); - try { $document = $dbForProject->createDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $document); } catch (StructureException $e) { @@ -3049,10 +3040,9 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') throw new Exception(Exception::COLLECTION_NOT_FOUND); } - $operations = 0; // Since the Document is sent through post no getDocument used // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3064,10 +3054,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); - if (\in_array(\gettype($related), ['array', 'object'])) { - $operations++; - } - if (empty($related)) { continue; } @@ -3091,11 +3077,12 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $processDocument($collection, $document); $queueForUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) + ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASES_OPERATIONS_WRITES), $operations) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection - $response->addHeader('X-Debug-operations-read', $operations); + $response->addHeader('X-Debug-Operations', $operations); + $response ->setStatusCode(Response::STATUS_CODE_CREATED) ->dynamic($document, Response::MODEL_DOCUMENT); @@ -3255,7 +3242,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ; - $response->addHeader('X-Debug-operations-reads', $operations); + $response->addHeader('X-Debug-Operations', $operations); $select = \array_reduce($queries, function ($result, $query) { return $result || ($query->getMethod() === Query::TYPE_SELECT); @@ -3391,7 +3378,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ; - $response->addHeader('X-Debug-operations-reads', $operations); + $response->addHeader('X-Debug-Operations', $operations); + $response->dynamic($document, Response::MODEL_DOCUMENT); }); @@ -3600,8 +3588,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $newDocument = new Document($data); - $operations = 1; // Root updateDocument - + $operations = 1; $setCollection = (function (Document $collection, Document $document) use (&$setCollection, $dbForProject, $database, &$operations) { $relationships = \array_filter( $collection->getAttribute('attributes', []), @@ -3684,7 +3671,7 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) ; - $response->addHeader('X-Debug-operations-writes', $operations); + $response->addHeader('X-Debug-Operations', $operations); try { $document = $dbForProject->withRequestTimestamp( From fc03075ef895d4818029445d6af32fd55cfd4ae0 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 6 Jan 2025 18:01:44 +0200 Subject: [PATCH 33/74] Update counts --- app/controllers/api/databases.php | 28 +++++++++++--------- composer.lock | 44 +++++++++++++++---------------- 2 files changed, 37 insertions(+), 35 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index fa12087463..f6de0d7d0f 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3163,7 +3163,6 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') throw new Exception(Exception::GENERAL_QUERY_INVALID, $validator->getDescription()); } - $documentId = $cursor->getValue(); $cursorDocument = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId)); @@ -3178,7 +3177,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') $documents = $dbForProject->find('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $queries); $total = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $queries, APP_LIMIT_COUNT); - $operations = 1; + $operations = 0; // Add $collectionId and $databaseId for all documents $processDocument = (function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations): bool { @@ -3186,6 +3185,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') return false; } + $operations++; + $document->removeAttribute('$collection'); $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3198,13 +3199,16 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); - if (\in_array(\gettype($related), ['array', 'object'])) { + if (\gettype($related) === 'object'){ $operations++; + } else if(\gettype($related) === 'array'){ + $operations += max(1, count($related)); } if (empty($related)) { continue; } + if (!\is_array($related)) { $relations = [$related]; } else { @@ -3328,7 +3332,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen throw new Exception(Exception::DOCUMENT_NOT_FOUND); } - $operations = 1; + $operations = 0; // Add $collectionId and $databaseId for all documents $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { @@ -3336,6 +3340,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen return; } + $operations++; + $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3347,13 +3353,16 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); - if (\in_array(\gettype($related), ['array', 'object'])) { + if (\gettype($related) === 'object'){ $operations++; + } else if(\gettype($related) === 'array'){ + $operations += max(1, count($related)); } if (empty($related)) { continue; } + if (!\is_array($related)) { $related = [$related]; } @@ -3587,8 +3596,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $data['$permissions'] = $permissions; $newDocument = new Document($data); - $operations = 1; + $setCollection = (function (Document $collection, Document $document) use (&$setCollection, $dbForProject, $database, &$operations) { $relationships = \array_filter( $collection->getAttribute('attributes', []), @@ -3598,13 +3607,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); - /** - * Write relationship counts - */ - if (\in_array(\gettype($related), ['array', 'object'])) { - - } - if (empty($related)) { continue; } diff --git a/composer.lock b/composer.lock index ce5f358619..dbe9e079b5 100644 --- a/composer.lock +++ b/composer.lock @@ -2453,16 +2453,16 @@ }, { "name": "symfony/http-client", - "version": "v7.2.1", + "version": "v7.2.2", "source": { "type": "git", "url": "https://github.com/symfony/http-client.git", - "reference": "ff4df2b68d1c67abb9fef146e6540ea16b58d99e" + "reference": "339ba21476eb184290361542f732ad12c97591ec" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-client/zipball/ff4df2b68d1c67abb9fef146e6540ea16b58d99e", - "reference": "ff4df2b68d1c67abb9fef146e6540ea16b58d99e", + "url": "https://api.github.com/repos/symfony/http-client/zipball/339ba21476eb184290361542f732ad12c97591ec", + "reference": "339ba21476eb184290361542f732ad12c97591ec", "shasum": "" }, "require": { @@ -2528,7 +2528,7 @@ "http" ], "support": { - "source": "https://github.com/symfony/http-client/tree/v7.2.1" + "source": "https://github.com/symfony/http-client/tree/v7.2.2" }, "funding": [ { @@ -2544,7 +2544,7 @@ "type": "tidelift" } ], - "time": "2024-12-07T08:50:44+00:00" + "time": "2024-12-30T18:35:15+00:00" }, { "name": "symfony/http-client-contracts", @@ -5126,16 +5126,16 @@ }, { "name": "laravel/pint", - "version": "v1.18.3", + "version": "v1.19.0", "source": { "type": "git", "url": "https://github.com/laravel/pint.git", - "reference": "cef51821608239040ab841ad6e1c6ae502ae3026" + "reference": "8169513746e1bac70c85d6ea1524d9225d4886f0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/pint/zipball/cef51821608239040ab841ad6e1c6ae502ae3026", - "reference": "cef51821608239040ab841ad6e1c6ae502ae3026", + "url": "https://api.github.com/repos/laravel/pint/zipball/8169513746e1bac70c85d6ea1524d9225d4886f0", + "reference": "8169513746e1bac70c85d6ea1524d9225d4886f0", "shasum": "" }, "require": { @@ -5146,10 +5146,10 @@ "php": "^8.1.0" }, "require-dev": { - "friendsofphp/php-cs-fixer": "^3.65.0", - "illuminate/view": "^10.48.24", - "larastan/larastan": "^2.9.11", - "laravel-zero/framework": "^10.4.0", + "friendsofphp/php-cs-fixer": "^3.66.0", + "illuminate/view": "^10.48.25", + "larastan/larastan": "^2.9.12", + "laravel-zero/framework": "^10.48.25", "mockery/mockery": "^1.6.12", "nunomaduro/termwind": "^1.17.0", "pestphp/pest": "^2.36.0" @@ -5188,7 +5188,7 @@ "issues": "https://github.com/laravel/pint/issues", "source": "https://github.com/laravel/pint" }, - "time": "2024-11-26T15:34:00+00:00" + "time": "2024-12-30T16:20:10+00:00" }, { "name": "matthiasmullie/minify", @@ -7735,16 +7735,16 @@ }, { "name": "symfony/finder", - "version": "v7.2.0", + "version": "v7.2.2", "source": { "type": "git", "url": "https://github.com/symfony/finder.git", - "reference": "6de263e5868b9a137602dd1e33e4d48bfae99c49" + "reference": "87a71856f2f56e4100373e92529eed3171695cfb" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/finder/zipball/6de263e5868b9a137602dd1e33e4d48bfae99c49", - "reference": "6de263e5868b9a137602dd1e33e4d48bfae99c49", + "url": "https://api.github.com/repos/symfony/finder/zipball/87a71856f2f56e4100373e92529eed3171695cfb", + "reference": "87a71856f2f56e4100373e92529eed3171695cfb", "shasum": "" }, "require": { @@ -7779,7 +7779,7 @@ "description": "Finds files and directories via an intuitive fluent interface", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/finder/tree/v7.2.0" + "source": "https://github.com/symfony/finder/tree/v7.2.2" }, "funding": [ { @@ -7795,7 +7795,7 @@ "type": "tidelift" } ], - "time": "2024-10-23T06:56:12+00:00" + "time": "2024-12-30T19:00:17+00:00" }, { "name": "symfony/options-resolver", @@ -8556,7 +8556,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": {}, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { From d64ea97ee6ff6f895477cb4b54166e9f6261c0e3 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 6 Jan 2025 18:06:18 +0200 Subject: [PATCH 34/74] added operations debug header --- app/controllers/api/databases.php | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index fa12087463..c65c76297d 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3692,10 +3692,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum throw new Exception(Exception::COLLECTION_NOT_FOUND); } - $operations = 1; // Since we read original Document - // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3707,10 +3705,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); - if (\in_array(\gettype($related), ['array', 'object'])) { - $operations++; - } - if (empty($related)) { continue; } @@ -3733,13 +3727,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $processDocument($collection, $document); - $queueForUsage - ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) - ; - - $response->addHeader('X-Debug-operations-reads', $operations); - $response->dynamic($document, Response::MODEL_DOCUMENT); $relationships = \array_map( From 40b407027efb1ddcb453b6570cd033ed004ff717 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 6 Jan 2025 18:51:39 +0200 Subject: [PATCH 35/74] Count changes --- app/controllers/api/databases.php | 49 ++++++++++++------------------- 1 file changed, 18 insertions(+), 31 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index b9614f5ef9..ca81519834 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -2944,8 +2944,11 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $data['$permissions'] = $permissions; $document = new Document($data); - $operations = 1; + $operations = 0; + $checkPermissions = function (Document $collection, Document $document, string $permission) use (&$checkPermissions, $dbForProject, $database, &$operations) { + $operations++; + $documentSecurity = $collection->getAttribute('documentSecurity', false); $validator = new Authorization($permission); @@ -2977,13 +2980,10 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') if ($isList) { $relations = $related; - $operations += count($related); } else { $relations = [$related]; - $operations++; } - $relatedCollectionId = $relationship->getAttribute('relatedCollection'); $relatedCollection = Authorization::skip( fn () => $dbForProject->getDocument('database_' . $database->getInternalId(), $relatedCollectionId) @@ -3102,8 +3102,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') ->setContext('collection', $collection) ->setContext('database', $database) ->setPayload($response->getPayload(), sensitive: $relationships); - - }); App::get('/v1/databases/:databaseId/collections/:collectionId/documents') @@ -3199,13 +3197,11 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); - if (\gettype($related) === 'object'){ - $operations++; - } else if(\gettype($related) === 'array'){ - $operations += max(1, count($related)); - } - if (empty($related)) { + if (\in_array(\gettype($related), ['array', 'object'])) { + $operations++; + } + continue; } @@ -3353,13 +3349,11 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); - if (\gettype($related) === 'object'){ - $operations++; - } else if(\gettype($related) === 'array'){ - $operations += max(1, count($related)); - } - if (empty($related)) { + if (\in_array(\gettype($related), ['array', 'object'])) { + $operations++; + } + continue; } @@ -3596,9 +3590,12 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $data['$permissions'] = $permissions; $newDocument = new Document($data); - $operations = 1; + $operations = 0; $setCollection = (function (Document $collection, Document $document) use (&$setCollection, $dbForProject, $database, &$operations) { + + $operations++; + $relationships = \array_filter( $collection->getAttribute('attributes', []), fn ($attribute) => $attribute->getAttribute('type') === Database::VAR_RELATIONSHIP @@ -3615,10 +3612,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum if ($isList) { $relations = $related; - $operations += count($related); } else { $relations = [$related]; - $operations++; } $relatedCollectionId = $relationship->getAttribute('relatedCollection'); @@ -3793,8 +3788,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu throw new Exception(Exception::COLLECTION_NOT_FOUND); } - $operations = 1; // Read original Document - // Read permission should not be required for delete $document = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId)); @@ -3814,7 +3807,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu } // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3826,10 +3819,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); - if (\in_array(\gettype($related), ['array', 'object'])) { - $operations++; - } - if (empty($related)) { continue; } @@ -3855,8 +3844,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu $queueForUsage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1) - ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection $relationships = \array_map( @@ -3875,7 +3862,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->setContext('database', $database) ->setPayload($response->output($document, Response::MODEL_DOCUMENT), sensitive: $relationships); - $response->addHeader('X-Debug-Operations', $operations); + $response->addHeader('X-Debug-Operations', 1); $response->noContent(); }); From a481e958e345b220a98b252277043cd4de8eac47 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 6 Jan 2025 18:57:35 +0200 Subject: [PATCH 36/74] added operations debug header --- app/controllers/api/databases.php | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index b9614f5ef9..5503a78051 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3078,7 +3078,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') $queueForUsage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, $operations) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASES_OPERATIONS_WRITES), $operations) + ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection $response->addHeader('X-Debug-Operations', $operations); @@ -3102,8 +3102,6 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents') ->setContext('collection', $collection) ->setContext('database', $database) ->setPayload($response->getPayload(), sensitive: $relationships); - - }); App::get('/v1/databases/:databaseId/collections/:collectionId/documents') @@ -3177,9 +3175,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents') $documents = $dbForProject->find('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $queries); $total = $dbForProject->count('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $queries, APP_LIMIT_COUNT); - $operations = 0; - // Add $collectionId and $databaseId for all documents + $operations = 1; $processDocument = (function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations): bool { if ($document->isEmpty()) { return false; @@ -3332,9 +3329,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen throw new Exception(Exception::DOCUMENT_NOT_FOUND); } - $operations = 0; - // Add $collectionId and $databaseId for all documents + $operations = 1; $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { if ($document->isEmpty()) { return; @@ -3491,6 +3487,7 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen $output[$i]['countryName'] = $locale->getText('locale.country.unknown'); } } + $response->dynamic(new Document([ 'total' => $audit->countLogsByResource($resource), 'logs' => $output, @@ -3597,7 +3594,6 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum $newDocument = new Document($data); $operations = 1; - $setCollection = (function (Document $collection, Document $document) use (&$setCollection, $dbForProject, $database, &$operations) { $relationships = \array_filter( $collection->getAttribute('attributes', []), @@ -3793,8 +3789,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu throw new Exception(Exception::COLLECTION_NOT_FOUND); } - $operations = 1; // Read original Document - // Read permission should not be required for delete $document = Authorization::skip(fn () => $dbForProject->getDocument('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId(), $documentId)); @@ -3814,7 +3808,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu } // Add $collectionId and $databaseId for all documents - $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database, &$operations) { + $processDocument = function (Document $collection, Document $document) use (&$processDocument, $dbForProject, $database) { $document->setAttribute('$databaseId', $database->getId()); $document->setAttribute('$collectionId', $collection->getId()); @@ -3826,10 +3820,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu foreach ($relationships as $relationship) { $related = $document->getAttribute($relationship->getAttribute('key')); - if (\in_array(\gettype($related), ['array', 'object'])) { - $operations++; - } - if (empty($related)) { continue; } @@ -3855,10 +3845,10 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu $queueForUsage ->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1) ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1) - ->addMetric(METRIC_DATABASES_OPERATIONS_READS, $operations) - ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_READS), $operations) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection + $response->addHeader('X-Debug-Operations', $operations); + $relationships = \array_map( fn ($document) => $document->getAttribute('key'), \array_filter( @@ -3875,7 +3865,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->setContext('database', $database) ->setPayload($response->output($document, Response::MODEL_DOCUMENT), sensitive: $relationships); - $response->addHeader('X-Debug-Operations', $operations); $response->noContent(); }); From 5c097df56a1710925fbb3bbe0973380147975b21 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 6 Jan 2025 19:07:18 +0200 Subject: [PATCH 37/74] added operations debug header --- app/controllers/api/databases.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/databases.php b/app/controllers/api/databases.php index 92e61ad1d5..347b4ebbef 100644 --- a/app/controllers/api/databases.php +++ b/app/controllers/api/databases.php @@ -3847,7 +3847,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu ->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1) ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection - $response->addHeader('X-Debug-Operations', $operations); + $response->addHeader('X-Debug-Operations', 1); $relationships = \array_map( fn ($document) => $document->getAttribute('key'), From f2cbdbae870babc3c92fca3d10d0c4a255e4ecdf Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 6 Jan 2025 19:32:19 +0200 Subject: [PATCH 38/74] added operations debug header --- composer.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.lock b/composer.lock index dbe9e079b5..b906af3a81 100644 --- a/composer.lock +++ b/composer.lock @@ -8580,5 +8580,5 @@ "platform-overrides": { "php": "8.3" }, - "plugin-api-version": "2.6.0" + "plugin-api-version": "2.2.0" } From 077313fe2d40be73756c4334c150b248841cbad6 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 8 Jan 2025 15:46:35 +1300 Subject: [PATCH 39/74] Update src/Appwrite/Platform/Workers/Databases.php --- src/Appwrite/Platform/Workers/Databases.php | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Databases.php b/src/Appwrite/Platform/Workers/Databases.php index 74c1250505..c16fba36c1 100644 --- a/src/Appwrite/Platform/Workers/Databases.php +++ b/src/Appwrite/Platform/Workers/Databases.php @@ -284,9 +284,6 @@ class Databases extends Action $dbForProject->deleteDocument('attributes', $relatedAttribute->getId()); } - /** - * No throwing Exception - */ } catch (\Throwable $e) { Console::error($e->getMessage()); From 2c04ace02236fb01c2aed9229d5da1ecab63e813 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 8 Jan 2025 18:38:24 +1300 Subject: [PATCH 40/74] Add connection for project --- app/init.php | 31 +++++++++++++++++++++++-------- app/worker.php | 39 +++++++++++++++++++++------------------ 2 files changed, 44 insertions(+), 26 deletions(-) diff --git a/app/init.php b/app/init.php index b4e60d7924..51d9c38239 100644 --- a/app/init.php +++ b/app/init.php @@ -74,6 +74,7 @@ use Utopia\Logger\Adapter\Raygun; use Utopia\Logger\Adapter\Sentry; use Utopia\Logger\Log; use Utopia\Logger\Logger; +use Utopia\Pools\Connection as PoolConnection; use Utopia\Pools\Group; use Utopia\Pools\Pool; use Utopia\Queue; @@ -1412,7 +1413,26 @@ App::setResource('console', function () { ]); }, []); -App::setResource('dbForProject', function (Group $pools, Database $dbForPlatform, Cache $cache, Document $project) { +App::setResource('connectionForProject', function (Group $pools, Document $project) { + if ($project->isEmpty() || $project->getId() === 'console') { + return $pools + ->get('console') + ->pop(); + } + + try { + $dsn = new DSN($project->getAttribute('database')); + } catch (\InvalidArgumentException) { + // TODO: Temporary until all projects are using shared tables + $dsn = new DSN('mysql://' . $project->getAttribute('database')); + } + + return $pools + ->get($dsn->getHost()) + ->pop(); +}, ['pools', 'project']); + +App::setResource('dbForProject', function (Group $pools, PoolConnection $connectionForProject, Database $dbForPlatform, Cache $cache, Document $project) { if ($project->isEmpty() || $project->getId() === 'console') { return $dbForPlatform; } @@ -1424,12 +1444,7 @@ App::setResource('dbForProject', function (Group $pools, Database $dbForPlatform $dsn = new DSN('mysql://' . $project->getAttribute('database')); } - $dbAdapter = $pools - ->get($dsn->getHost()) - ->pop() - ->getResource(); - - $database = new Database($dbAdapter, $cache); + $database = new Database($connectionForProject->getResource(), $cache); $database ->setMetadata('host', \gethostname()) @@ -1452,7 +1467,7 @@ App::setResource('dbForProject', function (Group $pools, Database $dbForPlatform } return $database; -}, ['pools', 'dbForPlatform', 'cache', 'project']); +}, ['pools', 'connectionForProject', 'dbForPlatform', 'cache', 'project']); App::setResource('dbForPlatform', function (Group $pools, Cache $cache) { $dbAdapter = $pools diff --git a/app/worker.php b/app/worker.php index 6eb1363e9b..845af12efc 100644 --- a/app/worker.php +++ b/app/worker.php @@ -16,6 +16,7 @@ use Appwrite\Event\Migration; use Appwrite\Event\Usage; use Appwrite\Event\UsageDump; use Appwrite\Platform\Appwrite; +use Swoole\Database\DetectsLostConnections; use Swoole\Runtime; use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis; use Utopia\Cache\Adapter\Sharding; @@ -25,11 +26,13 @@ use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; +use Utopia\Database\Exception as DatabaseException; use Utopia\Database\Validator\Authorization; use Utopia\DSN\DSN; use Utopia\Logger\Log; use Utopia\Logger\Logger; use Utopia\Platform\Service; +use Utopia\Pools\Connection as PoolConnection; use Utopia\Pools\Group; use Utopia\Queue\Connection; use Utopia\Queue\Message; @@ -66,13 +69,13 @@ Server::setResource('project', function (Message $message, Database $dbForPlatfo return $dbForPlatform->getDocument('projects', $project->getId()); }, ['message', 'dbForPlatform']); -Server::setResource('dbForProject', function (Cache $cache, Registry $register, Message $message, Document $project, Database $dbForPlatform) { +Server::setResource('connectionForProject', function (Group $pools, Document $project) { if ($project->isEmpty() || $project->getId() === 'console') { - return $dbForPlatform; + return $pools + ->get('console') + ->pop(); } - $pools = $register->get('pools'); - try { $dsn = new DSN($project->getAttribute('database')); } catch (\InvalidArgumentException) { @@ -80,12 +83,17 @@ Server::setResource('dbForProject', function (Cache $cache, Registry $register, $dsn = new DSN('mysql://' . $project->getAttribute('database')); } - $adapter = $pools + return $pools ->get($dsn->getHost()) - ->pop() - ->getResource(); + ->pop(); +}, ['pools', 'project']); - $database = new Database($adapter, $cache); +Server::setResource('dbForProject', function (PoolConnection $connectionForProject, Cache $cache, Registry $register, Message $message, Document $project, Database $dbForPlatform) { + if ($project->isEmpty() || $project->getId() === 'console') { + return $dbForPlatform; + } + + $database = new Database($connectionForProject->getResource(), $cache); try { $dsn = new DSN($project->getAttribute('database')); @@ -109,12 +117,12 @@ Server::setResource('dbForProject', function (Cache $cache, Registry $register, } return $database; -}, ['cache', 'register', 'message', 'project', 'dbForPlatform']); +}, ['connectionForProject', 'cache', 'register', 'message', 'project', 'dbForPlatform']); -Server::setResource('getProjectDB', function (Group $pools, Database $dbForPlatform, $cache) { +Server::setResource('getProjectDB', function (Group $pools, PoolConnection $connectionForProject, Database $dbForPlatform, $cache) { $databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools - return function (Document $project) use ($pools, $dbForPlatform, $cache, &$databases): Database { + return function (Document $project) use ($pools, $connectionForProject, $dbForPlatform, $cache, &$databases): Database { if ($project->isEmpty() || $project->getId() === 'console') { return $dbForPlatform; } @@ -146,12 +154,7 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForPlatf return $database; } - $dbAdapter = $pools - ->get($dsn->getHost()) - ->pop() - ->getResource(); - - $database = new Database($dbAdapter, $cache); + $database = new Database($connectionForProject->getResource(), $cache); $databases[$dsn->getHost()] = $database; @@ -171,7 +174,7 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForPlatf return $database; }; -}, ['pools', 'dbForPlatform', 'cache']); +}, ['pools', 'connectionForProject', 'dbForPlatform', 'cache']); Server::setResource('abuseRetention', function () { return time() - (int) System::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', 86400); From 71b892132bc4ef7d57ee813bd0d1a6cc307dc3d7 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 8 Jan 2025 18:39:06 +1300 Subject: [PATCH 41/74] Mark connections as unhealthy on lost connection error from DB --- app/controllers/general.php | 14 +++++++++++++- app/worker.php | 11 ++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index db9df71341..90cb3293e6 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -21,6 +21,7 @@ use Appwrite\Utopia\Response\Filters\V18 as ResponseV18; use Appwrite\Utopia\View; use Executor\Executor; use MaxMind\Db\Reader; +use Swoole\Database\DetectsLostConnections; use Swoole\Http\Request as SwooleRequest; use Utopia\App; use Utopia\CLI\Console; @@ -28,6 +29,7 @@ use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; +use Utopia\Database\Exception as DatabaseException; use Utopia\Database\Helpers\ID; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; @@ -38,6 +40,7 @@ use Utopia\Logger\Adapter\Sentry; use Utopia\Logger\Log; use Utopia\Logger\Log\User; use Utopia\Logger\Logger; +use Utopia\Pools\Connection; use Utopia\System\System; use Utopia\Validator\Hostname; use Utopia\Validator\Text; @@ -746,7 +749,16 @@ App::error() ->inject('logger') ->inject('log') ->inject('queueForUsage') - ->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, Log $log, Usage $queueForUsage) { + ->inject('connectionForProject') + ->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, Log $log, Usage $queueForUsage, Connection $connectionForProject) { + if ( + ($error instanceof PDOException || $error instanceof DatabaseException) + && DetectsLostConnections::causedByLostConnection($error) + ) { + // Mark connection as unhealthy so it will be recycled on next reclaim. + $connectionForProject->setHealthy(false); + } + $version = System::getEnv('_APP_VERSION', 'UNKNOWN'); $route = $utopia->getRoute(); $class = \get_class($error); diff --git a/app/worker.php b/app/worker.php index 845af12efc..62882b32b1 100644 --- a/app/worker.php +++ b/app/worker.php @@ -413,7 +413,16 @@ $worker ->inject('log') ->inject('pools') ->inject('project') - ->action(function (Throwable $error, ?Logger $logger, Log $log, Group $pools, Document $project) use ($queueName) { + ->inject('connectionForProject') + ->action(function (Throwable $error, ?Logger $logger, Log $log, Group $pools, Document $project, PoolConnection $connectionForProject) use ($queueName) { + if ( + ($error instanceof PDOException || $error instanceof DatabaseException) + && DetectsLostConnections::causedByLostConnection($error) + ) { + // Mark connection as unhealthy, it will be recycled on next reclaim. + $connectionForProject->setHealthy(false); + } + $pools->reclaim(); $version = System::getEnv('_APP_VERSION', 'UNKNOWN'); From 38ee612164c3be3206fd1eaf6288cc37eadede94 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 8 Jan 2025 18:39:26 +1300 Subject: [PATCH 42/74] Update pools --- composer.json | 2 +- composer.lock | 49 +++++++++++++++++++++++++++++-------------------- 2 files changed, 30 insertions(+), 21 deletions(-) diff --git a/composer.json b/composer.json index e44dd0bffc..ebbcad4673 100644 --- a/composer.json +++ b/composer.json @@ -63,7 +63,7 @@ "utopia-php/migration": "0.6.*", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.7.*", - "utopia-php/pools": "0.5.*", + "utopia-php/pools": "dev-feat-connection-health as 0.5.0", "utopia-php/preloader": "0.2.*", "utopia-php/queue": "0.7.*", "utopia-php/registry": "0.5.*", diff --git a/composer.lock b/composer.lock index b906af3a81..90d2d3fe71 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "6b136b5490c0d5d331eac0d70bb3e198", + "content-hash": "59c7d90abfa068a55426b786a244d985", "packages": [ { "name": "adhocore/jwt", @@ -3929,16 +3929,16 @@ }, { "name": "utopia-php/migration", - "version": "0.6.13", + "version": "0.6.14", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "68d9b0a9477755afcda607e7e8109785cae17a13" + "reference": "59a19f09ded0ccab4c8cca35b1242c01e2b9cfd2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/68d9b0a9477755afcda607e7e8109785cae17a13", - "reference": "68d9b0a9477755afcda607e7e8109785cae17a13", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/59a19f09ded0ccab4c8cca35b1242c01e2b9cfd2", + "reference": "59a19f09ded0ccab4c8cca35b1242c01e2b9cfd2", "shasum": "" }, "require": { @@ -3979,9 +3979,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/0.6.13" + "source": "https://github.com/utopia-php/migration/tree/0.6.14" }, - "time": "2024-11-26T13:57:53+00:00" + "time": "2025-01-08T01:07:25+00:00" }, { "name": "utopia-php/mongo", @@ -4145,25 +4145,25 @@ }, { "name": "utopia-php/pools", - "version": "0.5.0", + "version": "dev-feat-connection-health", "source": { "type": "git", "url": "https://github.com/utopia-php/pools.git", - "reference": "6f716a213a08db95eda1b5dddfa90983c1834817" + "reference": "b1d35f023b3ccf0d171fad88cc833cdace0e1e67" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/pools/zipball/6f716a213a08db95eda1b5dddfa90983c1834817", - "reference": "6f716a213a08db95eda1b5dddfa90983c1834817", + "url": "https://api.github.com/repos/utopia-php/pools/zipball/b1d35f023b3ccf0d171fad88cc833cdace0e1e67", + "reference": "b1d35f023b3ccf0d171fad88cc833cdace0e1e67", "shasum": "" }, "require": { - "php": ">=8.0" + "php": ">=8.3" }, "require-dev": { - "laravel/pint": "1.2.*", - "phpstan/phpstan": "1.8.*", - "phpunit/phpunit": "^9.3" + "laravel/pint": "1.*", + "phpstan/phpstan": "1.*", + "phpunit/phpunit": "11.*" }, "type": "library", "autoload": { @@ -4190,9 +4190,9 @@ ], "support": { "issues": "https://github.com/utopia-php/pools/issues", - "source": "https://github.com/utopia-php/pools/tree/0.5.0" + "source": "https://github.com/utopia-php/pools/tree/feat-connection-health" }, - "time": "2024-04-19T11:11:54+00:00" + "time": "2025-01-08T03:56:11+00:00" }, { "name": "utopia-php/preloader", @@ -8554,9 +8554,18 @@ "time": "2024-03-07T20:33:40+00:00" } ], - "aliases": [], + "aliases": [ + { + "package": "utopia-php/pools", + "version": "dev-feat-connection-health", + "alias": "0.5.0", + "alias_normalized": "0.5.0.0" + } + ], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": { + "utopia-php/pools": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { @@ -8580,5 +8589,5 @@ "platform-overrides": { "php": "8.3" }, - "plugin-api-version": "2.2.0" + "plugin-api-version": "2.6.0" } From 39f34135f0780babc4ee69f9bfe0ef7754ccc36a Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 8 Jan 2025 20:43:45 +1300 Subject: [PATCH 43/74] Update pools --- composer.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/composer.lock b/composer.lock index 90d2d3fe71..5aa40556e2 100644 --- a/composer.lock +++ b/composer.lock @@ -4149,12 +4149,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/pools.git", - "reference": "b1d35f023b3ccf0d171fad88cc833cdace0e1e67" + "reference": "3b1913e7e9cb01acac26215abb37eb767c0112ee" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/pools/zipball/b1d35f023b3ccf0d171fad88cc833cdace0e1e67", - "reference": "b1d35f023b3ccf0d171fad88cc833cdace0e1e67", + "url": "https://api.github.com/repos/utopia-php/pools/zipball/3b1913e7e9cb01acac26215abb37eb767c0112ee", + "reference": "3b1913e7e9cb01acac26215abb37eb767c0112ee", "shasum": "" }, "require": { @@ -4192,7 +4192,7 @@ "issues": "https://github.com/utopia-php/pools/issues", "source": "https://github.com/utopia-php/pools/tree/feat-connection-health" }, - "time": "2025-01-08T03:56:11+00:00" + "time": "2025-01-08T07:40:13+00:00" }, { "name": "utopia-php/preloader", From 7f6a43cf7b3bde76d4f4ce47dddbed74a7608a03 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 8 Jan 2025 21:00:02 +1300 Subject: [PATCH 44/74] Update to release --- composer.json | 2 +- composer.lock | 27 +++++++++------------------ 2 files changed, 10 insertions(+), 19 deletions(-) diff --git a/composer.json b/composer.json index ebbcad4673..2612403c1a 100644 --- a/composer.json +++ b/composer.json @@ -63,7 +63,7 @@ "utopia-php/migration": "0.6.*", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.7.*", - "utopia-php/pools": "dev-feat-connection-health as 0.5.0", + "utopia-php/pools": "0.6.*", "utopia-php/preloader": "0.2.*", "utopia-php/queue": "0.7.*", "utopia-php/registry": "0.5.*", diff --git a/composer.lock b/composer.lock index 5aa40556e2..045ef31ffa 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "59c7d90abfa068a55426b786a244d985", + "content-hash": "7b5b5926b452186543903eb539f59c2d", "packages": [ { "name": "adhocore/jwt", @@ -4145,16 +4145,16 @@ }, { "name": "utopia-php/pools", - "version": "dev-feat-connection-health", + "version": "0.6.0", "source": { "type": "git", "url": "https://github.com/utopia-php/pools.git", - "reference": "3b1913e7e9cb01acac26215abb37eb767c0112ee" + "reference": "59414ab7b57728edfde6d5ccc5a2583b7967ac18" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/pools/zipball/3b1913e7e9cb01acac26215abb37eb767c0112ee", - "reference": "3b1913e7e9cb01acac26215abb37eb767c0112ee", + "url": "https://api.github.com/repos/utopia-php/pools/zipball/59414ab7b57728edfde6d5ccc5a2583b7967ac18", + "reference": "59414ab7b57728edfde6d5ccc5a2583b7967ac18", "shasum": "" }, "require": { @@ -4190,9 +4190,9 @@ ], "support": { "issues": "https://github.com/utopia-php/pools/issues", - "source": "https://github.com/utopia-php/pools/tree/feat-connection-health" + "source": "https://github.com/utopia-php/pools/tree/0.6.0" }, - "time": "2025-01-08T07:40:13+00:00" + "time": "2025-01-08T07:58:42+00:00" }, { "name": "utopia-php/preloader", @@ -8554,18 +8554,9 @@ "time": "2024-03-07T20:33:40+00:00" } ], - "aliases": [ - { - "package": "utopia-php/pools", - "version": "dev-feat-connection-health", - "alias": "0.5.0", - "alias_normalized": "0.5.0.0" - } - ], + "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/pools": 20 - }, + "stability-flags": {}, "prefer-stable": false, "prefer-lowest": false, "platform": { From 941c9f19055928e4eb7b3fc5761b7602df98a99a Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 8 Jan 2025 21:15:33 +1300 Subject: [PATCH 45/74] Add connection health check to http error handler --- app/http.php | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/app/http.php b/app/http.php index 3a7562ffd1..b988c7d8ac 100644 --- a/app/http.php +++ b/app/http.php @@ -5,6 +5,7 @@ require_once __DIR__ . '/../vendor/autoload.php'; use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; use Swoole\Constant; +use Swoole\Database\DetectsLostConnections; use Swoole\Http\Request as SwooleRequest; use Swoole\Http\Response as SwooleResponse; use Swoole\Http\Server; @@ -17,6 +18,7 @@ use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; +use Utopia\Database\Exception as DatabaseException; use Utopia\Database\Exception\Duplicate; use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Permission; @@ -346,6 +348,15 @@ $http->on(Constant::EVENT_REQUEST, function (SwooleRequest $swooleRequest, Swool $app->run($request, $response); } catch (\Throwable $th) { + if ( + ($th instanceof PDOException || $th instanceof DatabaseException) + && DetectsLostConnections::causedByLostConnection($th) + ) { + // Mark connection as unhealthy so it will be recycled on next reclaim. + $connectionForProject = $app->getResource('connectionForProject'); + $connectionForProject->setHealthy(false); + } + $version = System::getEnv('_APP_VERSION', 'UNKNOWN'); $logger = $app->getResource("logger"); From ff056fa1457f3096ff6fc7ef7456d0bb95979911 Mon Sep 17 00:00:00 2001 From: ChiragAgg5k Date: Wed, 8 Jan 2025 17:01:01 +0530 Subject: [PATCH 46/74] chore: shifted authphone usage tracking to api calls --- app/controllers/api/account.php | 84 ++++++++++++++++++++- app/controllers/api/teams.php | 29 ++++++- src/Appwrite/Event/Messaging.php | 2 +- src/Appwrite/Platform/Workers/Messaging.php | 18 +---- 4 files changed, 113 insertions(+), 20 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 76a3ef8b61..40e248836b 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -17,6 +17,7 @@ use Appwrite\Event\Delete; use Appwrite\Event\Event; use Appwrite\Event\Mail; use Appwrite\Event\Messaging; +use Appwrite\Event\Usage; use Appwrite\Extend\Exception; use Appwrite\Hooks\Hooks; use Appwrite\Network\Validator\Email; @@ -27,7 +28,9 @@ use Appwrite\Utopia\Database\Validator\CustomId; use Appwrite\Utopia\Database\Validator\Queries\Identities; use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; +use libphonenumber\PhoneNumberUtil; use MaxMind\Db\Reader; +use Utopia\Abuse\Abuse; use Utopia\App; use Utopia\Audit\Audit as EventAudit; use Utopia\Config\Config; @@ -2315,7 +2318,10 @@ App::post('/v1/account/tokens/phone') ->inject('queueForEvents') ->inject('queueForMessaging') ->inject('locale') - ->action(function (string $userId, string $phone, Request $request, Response $response, Document $user, Document $project, Database $dbForProject, Event $queueForEvents, Messaging $queueForMessaging, Locale $locale) { + ->inject('timelimit') + ->inject('queueForUsage') + ->inject('plan') + ->action(function (string $userId, string $phone, Request $request, Response $response, Document $user, Document $project, Database $dbForProject, Event $queueForEvents, Messaging $queueForMessaging, Locale $locale, callable $timelimit, Usage $queueForUsage, array $plan) { if (empty(System::getEnv('_APP_SMS_PROVIDER'))) { throw new Exception(Exception::GENERAL_PHONE_DISABLED, 'Phone provider not configured'); } @@ -2452,6 +2458,27 @@ App::post('/v1/account/tokens/phone') ->setMessage($messageDoc) ->setRecipients([$phone]) ->setProviderType(MESSAGE_TYPE_SMS); + + if (isset($plan['authPhone'])) { + $timelimit = $timelimit('organization:{organizationId}', $plan['authPhone'], 30 * 24 * 60 * 60); // 30 days + $timelimit + ->setParam('{organizationId}', $project->getAttribute('organizationId')); + + $abuse = new Abuse($timelimit); + if ($abuse->check() && System::getEnv('_APP_OPTIONS_ABUSE', 'enabled') === 'enabled') { + $helper = PhoneNumberUtil::getInstance(); + $countryCode = $helper->parse($phone)->getCountryCode(); + + if (!empty($countryCode)) { + $queueForUsage + ->addMetric(str_replace('{countryCode}', $countryCode, METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE), 1); + } + $queueForUsage + ->addMetric(METRIC_AUTH_METHOD_PHONE, 1) + ->setProject($project) + ->trigger(); + } + } } // Set to unhashed secret for events and server responses @@ -3465,7 +3492,10 @@ App::post('/v1/account/verification/phone') ->inject('queueForMessaging') ->inject('project') ->inject('locale') - ->action(function (Request $request, Response $response, Document $user, Database $dbForProject, Event $queueForEvents, Messaging $queueForMessaging, Document $project, Locale $locale) { + ->inject('timelimit') + ->inject('queueForUsage') + ->inject('plan') + ->action(function (Request $request, Response $response, Document $user, Database $dbForProject, Event $queueForEvents, Messaging $queueForMessaging, Document $project, Locale $locale, callable $timelimit, Usage $queueForUsage, array $plan) { if (empty(System::getEnv('_APP_SMS_PROVIDER'))) { throw new Exception(Exception::GENERAL_PHONE_DISABLED, 'Phone provider not configured'); } @@ -3548,6 +3578,27 @@ App::post('/v1/account/verification/phone') ->setMessage($messageDoc) ->setRecipients([$user->getAttribute('phone')]) ->setProviderType(MESSAGE_TYPE_SMS); + + if (isset($plan['authPhone'])) { + $timelimit = $timelimit('organization:{organizationId}', $plan['authPhone'], 30 * 24 * 60 * 60); // 30 days + $timelimit + ->setParam('{organizationId}', $project->getAttribute('organizationId')); + + $abuse = new Abuse($timelimit); + if ($abuse->check() && System::getEnv('_APP_OPTIONS_ABUSE', 'enabled') === 'enabled') { + $helper = PhoneNumberUtil::getInstance(); + $countryCode = $helper->parse($phone)->getCountryCode(); + + if (!empty($countryCode)) { + $queueForUsage + ->addMetric(str_replace('{countryCode}', $countryCode, METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE), 1); + } + $queueForUsage + ->addMetric(METRIC_AUTH_METHOD_PHONE, 1) + ->setProject($project) + ->trigger(); + } + } } // Set to unhashed secret for events and server responses @@ -4026,7 +4077,10 @@ App::post('/v1/account/mfa/challenge') ->inject('queueForEvents') ->inject('queueForMessaging') ->inject('queueForMails') - ->action(function (string $factor, Response $response, Database $dbForProject, Document $user, Locale $locale, Document $project, Request $request, Event $queueForEvents, Messaging $queueForMessaging, Mail $queueForMails) { + ->inject('timelimit') + ->inject('queueForUsage') + ->inject('plan') + ->action(function (string $factor, Response $response, Database $dbForProject, Document $user, Locale $locale, Document $project, Request $request, Event $queueForEvents, Messaging $queueForMessaging, Mail $queueForMails, callable $timelimit, Usage $queueForUsage, array $plan) { $expire = DateTime::addSeconds(new \DateTime(), Auth::TOKEN_EXPIRATION_CONFIRM); $code = Auth::codeGenerator(); @@ -4074,6 +4128,7 @@ App::post('/v1/account/mfa/challenge') $message = $message->render(); + $phone = $user->getAttribute('phone'); $queueForMessaging ->setType(MESSAGE_SEND_TYPE_INTERNAL) ->setMessage(new Document([ @@ -4082,8 +4137,29 @@ App::post('/v1/account/mfa/challenge') 'content' => $code, ], ])) - ->setRecipients([$user->getAttribute('phone')]) + ->setRecipients([$phone]) ->setProviderType(MESSAGE_TYPE_SMS); + + if (isset($plan['authPhone'])) { + $timelimit = $timelimit('organization:{organizationId}', $plan['authPhone'], 30 * 24 * 60 * 60); // 30 days + $timelimit + ->setParam('{organizationId}', $project->getAttribute('organizationId')); + + $abuse = new Abuse($timelimit); + if ($abuse->check() && System::getEnv('_APP_OPTIONS_ABUSE', 'enabled') === 'enabled') { + $helper = PhoneNumberUtil::getInstance(); + $countryCode = $helper->parse($phone)->getCountryCode(); + + if (!empty($countryCode)) { + $queueForUsage + ->addMetric(str_replace('{countryCode}', $countryCode, METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE), 1); + } + $queueForUsage + ->addMetric(METRIC_AUTH_METHOD_PHONE, 1) + ->setProject($project) + ->trigger(); + } + } break; case Type::EMAIL: if (empty(System::getEnv('_APP_SMTP_HOST'))) { diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index f829800b98..216807fecf 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -8,6 +8,7 @@ use Appwrite\Event\Delete; use Appwrite\Event\Event; use Appwrite\Event\Mail; use Appwrite\Event\Messaging; +use Appwrite\Event\Usage; use Appwrite\Extend\Exception; use Appwrite\Network\Validator\Email; use Appwrite\Platform\Workers\Deletes; @@ -17,7 +18,9 @@ use Appwrite\Utopia\Database\Validator\Queries\Memberships; use Appwrite\Utopia\Database\Validator\Queries\Teams; use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; +use libphonenumber\PhoneNumberUtil; use MaxMind\Db\Reader; +use Utopia\Abuse\Abuse; use Utopia\App; use Utopia\Audit\Audit; use Utopia\Config\Config; @@ -423,7 +426,10 @@ App::post('/v1/teams/:teamId/memberships') ->inject('queueForMails') ->inject('queueForMessaging') ->inject('queueForEvents') - ->action(function (string $teamId, string $email, string $userId, string $phone, array $roles, string $url, string $name, Response $response, Document $project, Document $user, Database $dbForProject, Locale $locale, Mail $queueForMails, Messaging $queueForMessaging, Event $queueForEvents) { + ->inject('timelimit') + ->inject('queueForUsage') + ->inject('plan') + ->action(function (string $teamId, string $email, string $userId, string $phone, array $roles, string $url, string $name, Response $response, Document $project, Document $user, Database $dbForProject, Locale $locale, Mail $queueForMails, Messaging $queueForMessaging, Event $queueForEvents, callable $timelimit, Usage $queueForUsage, array $plan) { $isAPIKey = Auth::isAppUser(Authorization::getRoles()); $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles()); @@ -691,6 +697,27 @@ App::post('/v1/teams/:teamId/memberships') ->setMessage($messageDoc) ->setRecipients([$phone]) ->setProviderType('SMS'); + + if (isset($plan['authPhone'])) { + $timelimit = $timelimit('organization:{organizationId}', $plan['authPhone'], 30 * 24 * 60 * 60); // 30 days + $timelimit + ->setParam('{organizationId}', $project->getAttribute('organizationId')); + + $abuse = new Abuse($timelimit); + if ($abuse->check() && System::getEnv('_APP_OPTIONS_ABUSE', 'enabled') === 'enabled') { + $helper = PhoneNumberUtil::getInstance(); + $countryCode = $helper->parse($phone)->getCountryCode(); + + if (!empty($countryCode)) { + $queueForUsage + ->addMetric(str_replace('{countryCode}', $countryCode, METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE), 1); + } + $queueForUsage + ->addMetric(METRIC_AUTH_METHOD_PHONE, 1) + ->setProject($project) + ->trigger(); + } + } } } diff --git a/src/Appwrite/Event/Messaging.php b/src/Appwrite/Event/Messaging.php index ab9b1bee6b..755b8c9158 100644 --- a/src/Appwrite/Event/Messaging.php +++ b/src/Appwrite/Event/Messaging.php @@ -27,7 +27,7 @@ class Messaging extends Event /** * Sets type for the build event. * - * @param string $type Can be `MESSAGE_TYPE_INTERNAL` or `MESSAGE_TYPE_EXTERNAL`. + * @param string $type Can be `MESSAGE_SEND_TYPE_INTERNAL` or `MESSAGE_SEND_TYPE_EXTERNAL`. * @return self */ public function setType(string $type): self diff --git a/src/Appwrite/Platform/Workers/Messaging.php b/src/Appwrite/Platform/Workers/Messaging.php index 58f6265ff4..1ff032c3e1 100644 --- a/src/Appwrite/Platform/Workers/Messaging.php +++ b/src/Appwrite/Platform/Workers/Messaging.php @@ -96,7 +96,7 @@ class Messaging extends Action $message = new Document($payload['message'] ?? []); $recipients = $payload['recipients'] ?? []; - $this->sendInternalSMSMessage($message, $project, $recipients, $queueForUsage, $log); + $this->sendInternalSMSMessage($message, $project, $recipients, $log); break; case MESSAGE_SEND_TYPE_EXTERNAL: $message = $dbForProject->getDocument('messages', $payload['messageId']); @@ -377,7 +377,7 @@ class Messaging extends Action } } - private function sendInternalSMSMessage(Document $message, Document $project, array $recipients, Usage $queueForUsage, Log $log): void + private function sendInternalSMSMessage(Document $message, Document $project, array $recipients, Log $log): void { if (empty(System::getEnv('_APP_SMS_PROVIDER')) || empty(System::getEnv('_APP_SMS_FROM'))) { throw new \Exception('Skipped SMS processing. Missing "_APP_SMS_PROVIDER" or "_APP_SMS_FROM" environment variables.'); @@ -456,24 +456,14 @@ class Messaging extends Action $adapter->getMaxMessagesPerRequest() ); - batch(\array_map(function ($batch) use ($message, $provider, $adapter, $project, $queueForUsage) { - return function () use ($batch, $message, $provider, $adapter, $project, $queueForUsage) { + batch(\array_map(function ($batch) use ($message, $provider, $adapter) { + return function () use ($batch, $message, $provider, $adapter) { $message->setAttribute('to', $batch); $data = $this->buildSmsMessage($message, $provider); try { $adapter->send($data); - - $countryCode = $adapter->getCountryCode($message['to'][0] ?? ''); - if (!empty($countryCode)) { - $queueForUsage - ->addMetric(str_replace('{countryCode}', $countryCode, METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE), 1); - } - $queueForUsage - ->addMetric(METRIC_AUTH_METHOD_PHONE, 1) - ->setProject($project) - ->trigger(); } catch (\Throwable $th) { throw new \Exception('Failed sending to targets with error: ' . $th->getMessage()); } From 1f240ca27aa765727249beb20cac9e5e7bf5c254 Mon Sep 17 00:00:00 2001 From: ChiragAgg5k Date: Wed, 8 Jan 2025 17:09:55 +0530 Subject: [PATCH 47/74] chore: moved non-countrycode tracking outside abuse --- app/controllers/api/account.php | 24 ++++++++++++------------ app/controllers/api/teams.php | 8 ++++---- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 40e248836b..b80cf0fd64 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -2473,11 +2473,11 @@ App::post('/v1/account/tokens/phone') $queueForUsage ->addMetric(str_replace('{countryCode}', $countryCode, METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE), 1); } - $queueForUsage - ->addMetric(METRIC_AUTH_METHOD_PHONE, 1) - ->setProject($project) - ->trigger(); } + $queueForUsage + ->addMetric(METRIC_AUTH_METHOD_PHONE, 1) + ->setProject($project) + ->trigger(); } } @@ -3593,11 +3593,11 @@ App::post('/v1/account/verification/phone') $queueForUsage ->addMetric(str_replace('{countryCode}', $countryCode, METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE), 1); } - $queueForUsage - ->addMetric(METRIC_AUTH_METHOD_PHONE, 1) - ->setProject($project) - ->trigger(); } + $queueForUsage + ->addMetric(METRIC_AUTH_METHOD_PHONE, 1) + ->setProject($project) + ->trigger(); } } @@ -4154,11 +4154,11 @@ App::post('/v1/account/mfa/challenge') $queueForUsage ->addMetric(str_replace('{countryCode}', $countryCode, METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE), 1); } - $queueForUsage - ->addMetric(METRIC_AUTH_METHOD_PHONE, 1) - ->setProject($project) - ->trigger(); } + $queueForUsage + ->addMetric(METRIC_AUTH_METHOD_PHONE, 1) + ->setProject($project) + ->trigger(); } break; case Type::EMAIL: diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index 216807fecf..fc9f959ed2 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -712,11 +712,11 @@ App::post('/v1/teams/:teamId/memberships') $queueForUsage ->addMetric(str_replace('{countryCode}', $countryCode, METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE), 1); } - $queueForUsage - ->addMetric(METRIC_AUTH_METHOD_PHONE, 1) - ->setProject($project) - ->trigger(); } + $queueForUsage + ->addMetric(METRIC_AUTH_METHOD_PHONE, 1) + ->setProject($project) + ->trigger(); } } } From 2178bb44cd01d3c8e0934e6266bf76379a97cd68 Mon Sep 17 00:00:00 2001 From: ChiragAgg5k Date: Wed, 8 Jan 2025 17:14:07 +0530 Subject: [PATCH 48/74] fix: attribute --- app/controllers/api/account.php | 6 +++--- app/controllers/api/teams.php | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index b80cf0fd64..ff96acfc57 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -2462,7 +2462,7 @@ App::post('/v1/account/tokens/phone') if (isset($plan['authPhone'])) { $timelimit = $timelimit('organization:{organizationId}', $plan['authPhone'], 30 * 24 * 60 * 60); // 30 days $timelimit - ->setParam('{organizationId}', $project->getAttribute('organizationId')); + ->setParam('{organizationId}', $project->getAttribute('teamId')); $abuse = new Abuse($timelimit); if ($abuse->check() && System::getEnv('_APP_OPTIONS_ABUSE', 'enabled') === 'enabled') { @@ -3582,7 +3582,7 @@ App::post('/v1/account/verification/phone') if (isset($plan['authPhone'])) { $timelimit = $timelimit('organization:{organizationId}', $plan['authPhone'], 30 * 24 * 60 * 60); // 30 days $timelimit - ->setParam('{organizationId}', $project->getAttribute('organizationId')); + ->setParam('{organizationId}', $project->getAttribute('teamId')); $abuse = new Abuse($timelimit); if ($abuse->check() && System::getEnv('_APP_OPTIONS_ABUSE', 'enabled') === 'enabled') { @@ -4143,7 +4143,7 @@ App::post('/v1/account/mfa/challenge') if (isset($plan['authPhone'])) { $timelimit = $timelimit('organization:{organizationId}', $plan['authPhone'], 30 * 24 * 60 * 60); // 30 days $timelimit - ->setParam('{organizationId}', $project->getAttribute('organizationId')); + ->setParam('{organizationId}', $project->getAttribute('teamId')); $abuse = new Abuse($timelimit); if ($abuse->check() && System::getEnv('_APP_OPTIONS_ABUSE', 'enabled') === 'enabled') { diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index fc9f959ed2..461e90da58 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -701,7 +701,7 @@ App::post('/v1/teams/:teamId/memberships') if (isset($plan['authPhone'])) { $timelimit = $timelimit('organization:{organizationId}', $plan['authPhone'], 30 * 24 * 60 * 60); // 30 days $timelimit - ->setParam('{organizationId}', $project->getAttribute('organizationId')); + ->setParam('{organizationId}', $project->getAttribute('teamId')); $abuse = new Abuse($timelimit); if ($abuse->check() && System::getEnv('_APP_OPTIONS_ABUSE', 'enabled') === 'enabled') { From 88755e6d3863f8d10be1d2e101cb4bc32537860a Mon Sep 17 00:00:00 2001 From: Darshan Date: Wed, 8 Jan 2025 18:21:35 +0530 Subject: [PATCH 49/74] update: add `sleep` to tests. --- tests/e2e/Services/Messaging/MessagingConsoleClientTest.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/e2e/Services/Messaging/MessagingConsoleClientTest.php b/tests/e2e/Services/Messaging/MessagingConsoleClientTest.php index 1b0d840f96..a4b7750b58 100644 --- a/tests/e2e/Services/Messaging/MessagingConsoleClientTest.php +++ b/tests/e2e/Services/Messaging/MessagingConsoleClientTest.php @@ -54,6 +54,9 @@ class MessagingConsoleClientTest extends Scope $this->assertEquals(200, $response['headers']['status-code']); + // required for Cloud x Audits + sleep(10); + $logs = $this->client->call(Client::METHOD_GET, '/messaging/providers/' . $provider['body']['$id'] . '/logs', \array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], From addb56ab7da144b6c55d79bd37c717f63ebb49e8 Mon Sep 17 00:00:00 2001 From: Darshan Date: Wed, 8 Jan 2025 18:24:53 +0530 Subject: [PATCH 50/74] update: shift comment. --- tests/e2e/Services/Messaging/MessagingConsoleClientTest.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/e2e/Services/Messaging/MessagingConsoleClientTest.php b/tests/e2e/Services/Messaging/MessagingConsoleClientTest.php index a4b7750b58..72cdb34714 100644 --- a/tests/e2e/Services/Messaging/MessagingConsoleClientTest.php +++ b/tests/e2e/Services/Messaging/MessagingConsoleClientTest.php @@ -54,8 +54,7 @@ class MessagingConsoleClientTest extends Scope $this->assertEquals(200, $response['headers']['status-code']); - // required for Cloud x Audits - sleep(10); + sleep(10); // required for Cloud x Audits $logs = $this->client->call(Client::METHOD_GET, '/messaging/providers/' . $provider['body']['$id'] . '/logs', \array_merge([ 'content-type' => 'application/json', From 4226e923878802aff19e7fff88c4288aa052d53a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Wed, 8 Jan 2025 13:05:49 +0000 Subject: [PATCH 51/74] Improve DB schema --- app/config/collections.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/config/collections.php b/app/config/collections.php index 7a35dfe356..780870b382 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -5421,7 +5421,7 @@ $consoleCollections = array_merge([ '$id' => ID::custom('personalAccessToken'), 'type' => Database::VAR_STRING, 'format' => '', - 'size' => 16384, + 'size' => 256, 'signed' => true, 'required' => false, 'default' => null, @@ -5443,7 +5443,7 @@ $consoleCollections = array_merge([ '$id' => ID::custom('personalRefreshToken'), 'type' => Database::VAR_STRING, 'format' => '', - 'size' => 16384, + 'size' => 256, 'signed' => true, 'required' => false, 'default' => null, From e3637e4ba5bedaf7d9f55dd0e1a34cc55d447eb0 Mon Sep 17 00:00:00 2001 From: Darshan Date: Wed, 8 Jan 2025 18:40:00 +0530 Subject: [PATCH 52/74] update: lets clear this test too, it has been a while now. --- .github/workflows/tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3ab240c1b9..fc03891b35 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -175,6 +175,8 @@ jobs: name: Benchmark runs-on: ubuntu-latest needs: setup + permissions: + pull-requests: write steps: - name: Checkout repository uses: actions/checkout@v4 From 0daf4057f36b062bd80df21f50d1ebeaf3271d96 Mon Sep 17 00:00:00 2001 From: Darshan Date: Wed, 8 Jan 2025 19:10:30 +0530 Subject: [PATCH 53/74] fix: benchmark test. --- .github/workflows/tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3ab240c1b9..fc03891b35 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -175,6 +175,8 @@ jobs: name: Benchmark runs-on: ubuntu-latest needs: setup + permissions: + pull-requests: write steps: - name: Checkout repository uses: actions/checkout@v4 From bce68b22028b0699e55ed006114b54bfa2ccbe8d Mon Sep 17 00:00:00 2001 From: Darshan Date: Thu, 9 Jan 2025 19:45:03 +0530 Subject: [PATCH 54/74] fix: benchmark test. --- .../Messaging/MessagingConsoleClientTest.php | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/tests/e2e/Services/Messaging/MessagingConsoleClientTest.php b/tests/e2e/Services/Messaging/MessagingConsoleClientTest.php index 72cdb34714..245eb3e8de 100644 --- a/tests/e2e/Services/Messaging/MessagingConsoleClientTest.php +++ b/tests/e2e/Services/Messaging/MessagingConsoleClientTest.php @@ -2,6 +2,7 @@ namespace Tests\E2E\Services\Messaging; +use Appwrite\Tests\Async; use Tests\E2E\Client; use Tests\E2E\Scopes\ProjectCustom; use Tests\E2E\Scopes\Scope; @@ -11,6 +12,8 @@ use Utopia\Database\Query; class MessagingConsoleClientTest extends Scope { + use Async; + use MessagingBase; use ProjectCustom; use SideConsole; @@ -54,17 +57,18 @@ class MessagingConsoleClientTest extends Scope $this->assertEquals(200, $response['headers']['status-code']); - sleep(10); // required for Cloud x Audits + // required for Cloud x Audits + $this->assertEventually(function () use ($provider) { + $logs = $this->client->call(Client::METHOD_GET, '/messaging/providers/' . $provider['body']['$id'] . '/logs', \array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); - $logs = $this->client->call(Client::METHOD_GET, '/messaging/providers/' . $provider['body']['$id'] . '/logs', \array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - ], $this->getHeaders())); - - $this->assertEquals($logs['headers']['status-code'], 200); - $this->assertIsArray($logs['body']['logs']); - $this->assertIsNumeric($logs['body']['total']); - $this->assertCount(2, $logs['body']['logs']); + $this->assertEquals($logs['headers']['status-code'], 200); + $this->assertIsArray($logs['body']['logs']); + $this->assertIsNumeric($logs['body']['total']); + $this->assertCount(2, $logs['body']['logs']); + }); $logs = $this->client->call(Client::METHOD_GET, '/messaging/providers/' . $provider['body']['$id'] . '/logs', \array_merge([ 'content-type' => 'application/json', From d068067a985e1bafc9eace696cc2c6f75307e852 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Thu, 9 Jan 2025 16:35:40 +0000 Subject: [PATCH 55/74] feat: project sms usage --- app/controllers/api/project.php | 46 ++++++++++++++++++- .../Utopia/Response/Model/MetricBreakdown.php | 8 ++++ .../Utopia/Response/Model/UsageProject.php | 19 ++++++++ 3 files changed, 72 insertions(+), 1 deletion(-) diff --git a/app/controllers/api/project.php b/app/controllers/api/project.php index 45c8f2c32b..7f3b442727 100644 --- a/app/controllers/api/project.php +++ b/app/controllers/api/project.php @@ -31,7 +31,8 @@ App::get('/v1/project/usage') ->param('period', '1d', new WhiteList(['1h', '1d']), 'Period used', true) ->inject('response') ->inject('dbForProject') - ->action(function (string $startDate, string $endDate, string $period, Response $response, Database $dbForProject) { + ->inject('getSmsPrice') + ->action(function (string $startDate, string $endDate, string $period, Response $response, Database $dbForProject, callable $getSmsPrice) { $stats = $total = $usage = []; $format = 'Y-m-d 00:00:00'; $firstDay = (new DateTime($startDate))->format($format); @@ -257,6 +258,46 @@ App::get('/v1/project/usage') ]; }, $dbForProject->find('functions')); + // This total is includes free and paid SMS usage + $authPhoneTotal = Authorization::skip(fn () => $dbForProject->sum('stats', 'value', [ + Query::equal('metric', [METRIC_AUTH_METHOD_PHONE]), + Query::equal('period', ['1d']), + Query::greaterThanEqual('time', $firstDay), + Query::lessThan('time', $lastDay), + ])); + + // This estimate is only for paid SMS usage + $authPhoneMetrics = Authorization::skip(fn () => $dbForProject->find('stats', [ + Query::startsWith('metric', METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE . '.'), + Query::equal('period', ['1d']), + Query::greaterThanEqual('time', $firstDay), + Query::lessThan('time', $lastDay), + ])); + + $authPhoneEstimate = 0.0; + $authPhoneCountryBreakdown = []; + foreach ($authPhoneMetrics as $metric) { + $parts = explode('.', $metric->getAttribute('metric')); + $countryCode = $parts[3] ?? null; + if ($countryCode === null) { + continue; + } + + $value = $metric->getAttribute('value', 0); + + if (is_callable($getSmsPrice)) { + $authPhoneEstimate += $value * $getSmsPrice($countryCode); + } + + $authPhoneCountryBreakdown[] = [ + 'name' => $countryCode, + 'value' => $value, + 'estimate' => is_callable($getSmsPrice) + ? $value * $getSmsPrice($countryCode) + : 0.0, + ]; + } + // merge network inbound + outbound $projectBandwidth = []; foreach ($usage[METRIC_NETWORK_INBOUND] as $item) { @@ -303,6 +344,9 @@ App::get('/v1/project/usage') 'executionsMbSecondsBreakdown' => $executionsMbSecondsBreakdown, 'buildsMbSecondsBreakdown' => $buildsMbSecondsBreakdown, 'functionsStorageBreakdown' => $functionsStorageBreakdown, + 'authPhoneTotal' => $authPhoneTotal, + 'authPhoneEstimate' => $authPhoneEstimate, + 'authPhoneCountryBreakdown' => $authPhoneCountryBreakdown, ]), Response::MODEL_USAGE_PROJECT); }); diff --git a/src/Appwrite/Utopia/Response/Model/MetricBreakdown.php b/src/Appwrite/Utopia/Response/Model/MetricBreakdown.php index 29d5621532..ba46afcb77 100644 --- a/src/Appwrite/Utopia/Response/Model/MetricBreakdown.php +++ b/src/Appwrite/Utopia/Response/Model/MetricBreakdown.php @@ -15,6 +15,7 @@ class MetricBreakdown extends Model 'description' => 'Resource ID.', 'default' => '', 'example' => '5e5ea5c16897e', + 'required' => false, ]) ->addRule('name', [ 'type' => self::TYPE_STRING, @@ -27,6 +28,13 @@ class MetricBreakdown extends Model 'description' => 'The value of this metric at the timestamp.', 'default' => 0, 'example' => 1, + ]) + ->addRule('estimate', [ + 'type' => self::TYPE_FLOAT, + 'description' => 'The estimated value of this metric at the end of the period.', + 'default' => 0, + 'example' => 1, + 'required' => false, ]); } diff --git a/src/Appwrite/Utopia/Response/Model/UsageProject.php b/src/Appwrite/Utopia/Response/Model/UsageProject.php index 17d9271f04..39c245a542 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageProject.php +++ b/src/Appwrite/Utopia/Response/Model/UsageProject.php @@ -152,6 +152,25 @@ class UsageProject extends Model 'example' => [], 'array' => true ]) + ->addRule('authPhoneTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of phone auth.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('authPhoneEstimate', [ + 'type' => self::TYPE_FLOAT, + 'description' => 'Estimated total aggregated cost of phone auth.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('authPhoneCountryBreakdown', [ + 'type' => Response::MODEL_METRIC_BREAKDOWN, + 'description' => 'Aggregated breakdown in totals of phone auth by country.', + 'default' => [], + 'example' => [], + 'array' => true + ]) ; } From 651a3d1d622e7ab6dfe3a65ca2e06f1548495773 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Thu, 9 Jan 2025 16:37:26 +0000 Subject: [PATCH 56/74] test: fix --- tests/e2e/Services/Projects/ProjectsConsoleClientTest.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php index 326443afcf..c21056f727 100644 --- a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php +++ b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php @@ -485,6 +485,8 @@ class ProjectsConsoleClientTest extends Scope $this->assertIsNumeric($response['body']['usersTotal']); $this->assertIsNumeric($response['body']['filesStorageTotal']); $this->assertIsNumeric($response['body']['deploymentStorageTotal']); + $this->assertIsNumeric($response['body']['authPhoneTotal']); + $this->assertIsNumeric($response['body']['authPhoneEstimate']); /** From 76e1b27cefb32b64913d4720b96ab8805a500cef Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Thu, 9 Jan 2025 16:41:24 +0000 Subject: [PATCH 57/74] fix: create dummy resource --- app/init.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/init.php b/app/init.php index 51d9c38239..d1749be8f3 100644 --- a/app/init.php +++ b/app/init.php @@ -1844,6 +1844,10 @@ App::setResource('plan', function (array $plan = []) { return []; }); +App::setResource('getSmsPrice', function () { + return []; +}); + App::setResource('team', function (Document $project, Database $dbForPlatform, App $utopia, Request $request) { $teamInternalId = ''; if ($project->getId() !== 'console') { From 7c9c3d9214cbd044ed09934906f062e9d7ca0054 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Thu, 9 Jan 2025 18:33:07 +0000 Subject: [PATCH 58/74] feat: sms rates --- app/controllers/api/project.php | 12 ++++++------ app/init.php | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/app/controllers/api/project.php b/app/controllers/api/project.php index 7f3b442727..d12e35d9d7 100644 --- a/app/controllers/api/project.php +++ b/app/controllers/api/project.php @@ -31,8 +31,8 @@ App::get('/v1/project/usage') ->param('period', '1d', new WhiteList(['1h', '1d']), 'Period used', true) ->inject('response') ->inject('dbForProject') - ->inject('getSmsPrice') - ->action(function (string $startDate, string $endDate, string $period, Response $response, Database $dbForProject, callable $getSmsPrice) { + ->inject('smsRates') + ->action(function (string $startDate, string $endDate, string $period, Response $response, Database $dbForProject, array $smsRates) { $stats = $total = $usage = []; $format = 'Y-m-d 00:00:00'; $firstDay = (new DateTime($startDate))->format($format); @@ -285,15 +285,15 @@ App::get('/v1/project/usage') $value = $metric->getAttribute('value', 0); - if (is_callable($getSmsPrice)) { - $authPhoneEstimate += $value * $getSmsPrice($countryCode); + if (!empty($smsRates[$countryCode])) { + $authPhoneEstimate += $value * $smsRates[$countryCode]; } $authPhoneCountryBreakdown[] = [ 'name' => $countryCode, 'value' => $value, - 'estimate' => is_callable($getSmsPrice) - ? $value * $getSmsPrice($countryCode) + 'estimate' => !empty($smsRates[$countryCode]) + ? $value * $smsRates[$countryCode] : 0.0, ]; } diff --git a/app/init.php b/app/init.php index d1749be8f3..acaf3daaca 100644 --- a/app/init.php +++ b/app/init.php @@ -1844,7 +1844,7 @@ App::setResource('plan', function (array $plan = []) { return []; }); -App::setResource('getSmsPrice', function () { +App::setResource('smsRates', function () { return []; }); From ee9e4f90c1586a9f9d78283047312869e028edd8 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Thu, 9 Jan 2025 18:57:00 +0000 Subject: [PATCH 59/74] chore: specs --- app/config/specs/open-api3-1.6.x-client.json | 8 +- app/config/specs/open-api3-1.6.x-console.json | 453 ++++------------- app/config/specs/open-api3-1.6.x-server.json | 104 ++-- .../specs/open-api3-latest-console.json | 36 +- app/config/specs/swagger2-1.6.x-client.json | 8 +- app/config/specs/swagger2-1.6.x-console.json | 469 ++++-------------- app/config/specs/swagger2-1.6.x-server.json | 104 ++-- app/config/specs/swagger2-latest-console.json | 37 +- 8 files changed, 378 insertions(+), 841 deletions(-) diff --git a/app/config/specs/open-api3-1.6.x-client.json b/app/config/specs/open-api3-1.6.x-client.json index e3cd909b4e..af7303c985 100644 --- a/app/config/specs/open-api3-1.6.x-client.json +++ b/app/config/specs/open-api3-1.6.x-client.json @@ -1,7 +1,7 @@ { "openapi": "3.0.0", "info": { - "version": "1.6.0", + "version": "1.6.1", "title": "Appwrite", "description": "Appwrite backend as a service cuts up to 70% of the time and costs required for building a modern application. We abstract and simplify common development tasks behind a REST APIs, to help you develop your app in a fast and secure way. For full API documentation and tutorials go to [https:\/\/appwrite.io\/docs](https:\/\/appwrite.io\/docs)", "termsOfService": "https:\/\/appwrite.io\/policy\/terms", @@ -2784,7 +2784,7 @@ "tags": [ "account" ], - "description": "Sends the user an email with a secret key for creating a session. If the provided user ID has not been registered, a new user will be created. When the user clicks the link in the email, the user is redirected back to the URL you provided with the secret key and userId values attached to the URL query string. Use the query string parameters to submit a request to the [POST \/v1\/account\/sessions\/token](https:\/\/appwrite.io\/docs\/references\/cloud\/client-web\/account#createSession) endpoint to complete the login process. The link sent to the user's email address is valid for 1 hour. If you are on a mobile device you can leave the URL parameter empty, so that the login completion will be handled by your Appwrite instance by default.\n\nA user is limited to 10 active sessions at a time by default. [Learn more about session limits](https:\/\/appwrite.io\/docs\/authentication-security#limits).\n", + "description": "Sends the user an email with a secret key for creating a session. If the provided user ID has not been registered, a new user will be created. When the user clicks the link in the email, the user is redirected back to the URL you provided with the secret key and userId values attached to the URL query string. Use the query string parameters to submit a request to the [POST \/v1\/account\/sessions\/token](https:\/\/appwrite.io\/docs\/references\/cloud\/client-web\/account#createSession) endpoint to complete the login process. The link sent to the user's email address is valid for 1 hour.\n\nA user is limited to 10 active sessions at a time by default. [Learn more about session limits](https:\/\/appwrite.io\/docs\/authentication-security#limits).\n", "responses": { "201": { "description": "Token", @@ -5766,7 +5766,7 @@ }, "x-appwrite": { "method": "createSubscriber", - "weight": 382, + "weight": 376, "cookies": false, "type": "", "deprecated": false, @@ -5851,7 +5851,7 @@ }, "x-appwrite": { "method": "deleteSubscriber", - "weight": 386, + "weight": 380, "cookies": false, "type": "", "deprecated": false, diff --git a/app/config/specs/open-api3-1.6.x-console.json b/app/config/specs/open-api3-1.6.x-console.json index 369fea741a..5b0e5376f5 100644 --- a/app/config/specs/open-api3-1.6.x-console.json +++ b/app/config/specs/open-api3-1.6.x-console.json @@ -1,7 +1,7 @@ { "openapi": "3.0.0", "info": { - "version": "1.6.0", + "version": "1.6.1", "title": "Appwrite", "description": "Appwrite backend as a service cuts up to 70% of the time and costs required for building a modern application. We abstract and simplify common development tasks behind a REST APIs, to help you develop your app in a fast and secure way. For full API documentation and tutorials go to [https:\/\/appwrite.io\/docs](https:\/\/appwrite.io\/docs)", "termsOfService": "https:\/\/appwrite.io\/policy\/terms", @@ -2795,7 +2795,7 @@ "tags": [ "account" ], - "description": "Sends the user an email with a secret key for creating a session. If the provided user ID has not been registered, a new user will be created. When the user clicks the link in the email, the user is redirected back to the URL you provided with the secret key and userId values attached to the URL query string. Use the query string parameters to submit a request to the [POST \/v1\/account\/sessions\/token](https:\/\/appwrite.io\/docs\/references\/cloud\/client-web\/account#createSession) endpoint to complete the login process. The link sent to the user's email address is valid for 1 hour. If you are on a mobile device you can leave the URL parameter empty, so that the login completion will be handled by your Appwrite instance by default.\n\nA user is limited to 10 active sessions at a time by default. [Learn more about session limits](https:\/\/appwrite.io\/docs\/authentication-security#limits).\n", + "description": "Sends the user an email with a secret key for creating a session. If the provided user ID has not been registered, a new user will be created. When the user clicks the link in the email, the user is redirected back to the URL you provided with the secret key and userId values attached to the URL query string. Use the query string parameters to submit a request to the [POST \/v1\/account\/sessions\/token](https:\/\/appwrite.io\/docs\/references\/cloud\/client-web\/account#createSession) endpoint to complete the login process. The link sent to the user's email address is valid for 1 hour.\n\nA user is limited to 10 active sessions at a time by default. [Learn more about session limits](https:\/\/appwrite.io\/docs\/authentication-security#limits).\n", "responses": { "201": { "description": "Token", @@ -9488,7 +9488,8 @@ "bun-1.0", "bun-1.1", "go-1.23", - "static-1" + "static-1", + "flutter-3.24" ], "x-enum-name": null, "x-enum-keys": [] @@ -10145,7 +10146,8 @@ "bun-1.0", "bun-1.1", "go-1.23", - "static-1" + "static-1", + "flutter-3.24" ], "x-enum-name": null, "x-enum-keys": [] @@ -13673,7 +13675,7 @@ }, "x-appwrite": { "method": "listMessages", - "weight": 390, + "weight": 384, "cookies": false, "type": "", "deprecated": false, @@ -13751,7 +13753,7 @@ }, "x-appwrite": { "method": "createEmail", - "weight": 387, + "weight": 381, "cookies": false, "type": "", "deprecated": false, @@ -13897,7 +13899,7 @@ }, "x-appwrite": { "method": "updateEmail", - "weight": 394, + "weight": 388, "cookies": false, "type": "", "deprecated": false, @@ -14045,7 +14047,7 @@ }, "x-appwrite": { "method": "createPush", - "weight": 389, + "weight": 383, "cookies": false, "type": "", "deprecated": false, @@ -14202,7 +14204,7 @@ }, "x-appwrite": { "method": "updatePush", - "weight": 396, + "weight": 390, "cookies": false, "type": "", "deprecated": false, @@ -14361,7 +14363,7 @@ }, "x-appwrite": { "method": "createSms", - "weight": 388, + "weight": 382, "cookies": false, "type": "", "deprecated": false, @@ -14472,7 +14474,7 @@ }, "x-appwrite": { "method": "updateSms", - "weight": 395, + "weight": 389, "cookies": false, "type": "", "deprecated": false, @@ -14586,7 +14588,7 @@ }, "x-appwrite": { "method": "getMessage", - "weight": 393, + "weight": 387, "cookies": false, "type": "", "deprecated": false, @@ -14641,7 +14643,7 @@ }, "x-appwrite": { "method": "delete", - "weight": 397, + "weight": 391, "cookies": false, "type": "", "deprecated": false, @@ -14705,7 +14707,7 @@ }, "x-appwrite": { "method": "listMessageLogs", - "weight": 391, + "weight": 385, "cookies": false, "type": "", "deprecated": false, @@ -14782,7 +14784,7 @@ }, "x-appwrite": { "method": "listTargets", - "weight": 392, + "weight": 386, "cookies": false, "type": "", "deprecated": false, @@ -14859,7 +14861,7 @@ }, "x-appwrite": { "method": "listProviders", - "weight": 362, + "weight": 356, "cookies": false, "type": "", "deprecated": false, @@ -14937,7 +14939,7 @@ }, "x-appwrite": { "method": "createApnsProvider", - "weight": 361, + "weight": 355, "cookies": false, "type": "", "deprecated": false, @@ -15044,7 +15046,7 @@ }, "x-appwrite": { "method": "updateApnsProvider", - "weight": 374, + "weight": 368, "cookies": false, "type": "", "deprecated": false, @@ -15154,7 +15156,7 @@ }, "x-appwrite": { "method": "createFcmProvider", - "weight": 360, + "weight": 354, "cookies": false, "type": "", "deprecated": false, @@ -15241,7 +15243,7 @@ }, "x-appwrite": { "method": "updateFcmProvider", - "weight": 373, + "weight": 367, "cookies": false, "type": "", "deprecated": false, @@ -15331,7 +15333,7 @@ }, "x-appwrite": { "method": "createMailgunProvider", - "weight": 352, + "weight": 346, "cookies": false, "type": "", "deprecated": false, @@ -15448,7 +15450,7 @@ }, "x-appwrite": { "method": "updateMailgunProvider", - "weight": 365, + "weight": 359, "cookies": false, "type": "", "deprecated": false, @@ -15568,7 +15570,7 @@ }, "x-appwrite": { "method": "createMsg91Provider", - "weight": 355, + "weight": 349, "cookies": false, "type": "", "deprecated": false, @@ -15665,7 +15667,7 @@ }, "x-appwrite": { "method": "updateMsg91Provider", - "weight": 368, + "weight": 362, "cookies": false, "type": "", "deprecated": false, @@ -15765,7 +15767,7 @@ }, "x-appwrite": { "method": "createSendgridProvider", - "weight": 353, + "weight": 347, "cookies": false, "type": "", "deprecated": false, @@ -15872,7 +15874,7 @@ }, "x-appwrite": { "method": "updateSendgridProvider", - "weight": 366, + "weight": 360, "cookies": false, "type": "", "deprecated": false, @@ -15982,7 +15984,7 @@ }, "x-appwrite": { "method": "createSmtpProvider", - "weight": 354, + "weight": 348, "cookies": false, "type": "", "deprecated": false, @@ -16127,7 +16129,7 @@ }, "x-appwrite": { "method": "updateSmtpProvider", - "weight": 367, + "weight": 361, "cookies": false, "type": "", "deprecated": false, @@ -16274,7 +16276,7 @@ }, "x-appwrite": { "method": "createTelesignProvider", - "weight": 356, + "weight": 350, "cookies": false, "type": "", "deprecated": false, @@ -16371,7 +16373,7 @@ }, "x-appwrite": { "method": "updateTelesignProvider", - "weight": 369, + "weight": 363, "cookies": false, "type": "", "deprecated": false, @@ -16471,7 +16473,7 @@ }, "x-appwrite": { "method": "createTextmagicProvider", - "weight": 357, + "weight": 351, "cookies": false, "type": "", "deprecated": false, @@ -16568,7 +16570,7 @@ }, "x-appwrite": { "method": "updateTextmagicProvider", - "weight": 370, + "weight": 364, "cookies": false, "type": "", "deprecated": false, @@ -16668,7 +16670,7 @@ }, "x-appwrite": { "method": "createTwilioProvider", - "weight": 358, + "weight": 352, "cookies": false, "type": "", "deprecated": false, @@ -16765,7 +16767,7 @@ }, "x-appwrite": { "method": "updateTwilioProvider", - "weight": 371, + "weight": 365, "cookies": false, "type": "", "deprecated": false, @@ -16865,7 +16867,7 @@ }, "x-appwrite": { "method": "createVonageProvider", - "weight": 359, + "weight": 353, "cookies": false, "type": "", "deprecated": false, @@ -16962,7 +16964,7 @@ }, "x-appwrite": { "method": "updateVonageProvider", - "weight": 372, + "weight": 366, "cookies": false, "type": "", "deprecated": false, @@ -17062,7 +17064,7 @@ }, "x-appwrite": { "method": "getProvider", - "weight": 364, + "weight": 358, "cookies": false, "type": "", "deprecated": false, @@ -17117,7 +17119,7 @@ }, "x-appwrite": { "method": "deleteProvider", - "weight": 375, + "weight": 369, "cookies": false, "type": "", "deprecated": false, @@ -17181,7 +17183,7 @@ }, "x-appwrite": { "method": "listProviderLogs", - "weight": 363, + "weight": 357, "cookies": false, "type": "", "deprecated": false, @@ -17258,7 +17260,7 @@ }, "x-appwrite": { "method": "listSubscriberLogs", - "weight": 384, + "weight": 378, "cookies": false, "type": "", "deprecated": false, @@ -17335,7 +17337,7 @@ }, "x-appwrite": { "method": "listTopics", - "weight": 377, + "weight": 371, "cookies": false, "type": "", "deprecated": false, @@ -17411,7 +17413,7 @@ }, "x-appwrite": { "method": "createTopic", - "weight": 376, + "weight": 370, "cookies": false, "type": "", "deprecated": false, @@ -17496,7 +17498,7 @@ }, "x-appwrite": { "method": "getTopic", - "weight": 379, + "weight": 373, "cookies": false, "type": "", "deprecated": false, @@ -17558,7 +17560,7 @@ }, "x-appwrite": { "method": "updateTopic", - "weight": 380, + "weight": 374, "cookies": false, "type": "", "deprecated": false, @@ -17637,7 +17639,7 @@ }, "x-appwrite": { "method": "deleteTopic", - "weight": 381, + "weight": 375, "cookies": false, "type": "", "deprecated": false, @@ -17701,7 +17703,7 @@ }, "x-appwrite": { "method": "listTopicLogs", - "weight": 378, + "weight": 372, "cookies": false, "type": "", "deprecated": false, @@ -17778,7 +17780,7 @@ }, "x-appwrite": { "method": "listSubscribers", - "weight": 383, + "weight": 377, "cookies": false, "type": "", "deprecated": false, @@ -17864,7 +17866,7 @@ }, "x-appwrite": { "method": "createSubscriber", - "weight": 382, + "weight": 376, "cookies": false, "type": "", "deprecated": false, @@ -17956,7 +17958,7 @@ }, "x-appwrite": { "method": "getSubscriber", - "weight": 385, + "weight": 379, "cookies": false, "type": "", "deprecated": false, @@ -18021,7 +18023,7 @@ }, "x-appwrite": { "method": "deleteSubscriber", - "weight": 386, + "weight": 380, "cookies": false, "type": "", "deprecated": false, @@ -18098,7 +18100,7 @@ }, "x-appwrite": { "method": "list", - "weight": 339, + "weight": 338, "cookies": false, "type": "", "deprecated": false, @@ -18264,7 +18266,7 @@ }, "x-appwrite": { "method": "getAppwriteReport", - "weight": 341, + "weight": 340, "cookies": false, "type": "", "deprecated": false, @@ -18339,7 +18341,7 @@ }, "\/migrations\/firebase": { "post": { - "summary": "Migrate Firebase data (Service Account)", + "summary": "Migrate Firebase data", "operationId": "migrationsCreateFirebaseMigration", "tags": [ "migrations" @@ -18359,7 +18361,7 @@ }, "x-appwrite": { "method": "createFirebaseMigration", - "weight": 336, + "weight": 335, "cookies": false, "type": "", "deprecated": false, @@ -18415,177 +18417,6 @@ } } }, - "\/migrations\/firebase\/deauthorize": { - "get": { - "summary": "Revoke Appwrite's authorization to access Firebase projects", - "operationId": "migrationsDeleteFirebaseAuth", - "tags": [ - "migrations" - ], - "description": "", - "responses": { - "200": { - "description": "File" - } - }, - "x-appwrite": { - "method": "deleteFirebaseAuth", - "weight": 347, - "cookies": false, - "type": "", - "deprecated": false, - "demo": "migrations\/delete-firebase-auth.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master", - "rate-limit": 0, - "rate-time": 3600, - "rate-key": "url:{url},ip:{ip}", - "scope": "migrations.write", - "platforms": [ - "console" - ], - "packaging": false, - "offline-model": "", - "offline-key": "", - "offline-response-key": "$id", - "auth": { - "Project": [] - } - }, - "security": [ - { - "Project": [] - } - ] - } - }, - "\/migrations\/firebase\/oauth": { - "post": { - "summary": "Migrate Firebase data (OAuth)", - "operationId": "migrationsCreateFirebaseOAuthMigration", - "tags": [ - "migrations" - ], - "description": "", - "responses": { - "202": { - "description": "Migration", - "content": { - "application\/json": { - "schema": { - "$ref": "#\/components\/schemas\/migration" - } - } - } - } - }, - "x-appwrite": { - "method": "createFirebaseOAuthMigration", - "weight": 335, - "cookies": false, - "type": "", - "deprecated": false, - "demo": "migrations\/create-firebase-o-auth-migration.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/migrations\/migration-firebase.md", - "rate-limit": 0, - "rate-time": 3600, - "rate-key": "url:{url},ip:{ip}", - "scope": "migrations.write", - "platforms": [ - "console" - ], - "packaging": false, - "offline-model": "", - "offline-key": "", - "offline-response-key": "$id", - "auth": { - "Project": [] - } - }, - "security": [ - { - "Project": [] - } - ], - "requestBody": { - "content": { - "application\/json": { - "schema": { - "type": "object", - "properties": { - "resources": { - "type": "array", - "description": "List of resources to migrate", - "x-example": null, - "items": { - "type": "string" - } - }, - "projectId": { - "type": "string", - "description": "Project ID of the Firebase Project", - "x-example": "" - } - }, - "required": [ - "resources", - "projectId" - ] - } - } - } - } - } - }, - "\/migrations\/firebase\/projects": { - "get": { - "summary": "List Firebase projects", - "operationId": "migrationsListFirebaseProjects", - "tags": [ - "migrations" - ], - "description": "", - "responses": { - "200": { - "description": "Migrations Firebase Projects List", - "content": { - "application\/json": { - "schema": { - "$ref": "#\/components\/schemas\/firebaseProjectList" - } - } - } - } - }, - "x-appwrite": { - "method": "listFirebaseProjects", - "weight": 346, - "cookies": false, - "type": "", - "deprecated": false, - "demo": "migrations\/list-firebase-projects.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master", - "rate-limit": 0, - "rate-time": 3600, - "rate-key": "url:{url},ip:{ip}", - "scope": "migrations.read", - "platforms": [ - "console" - ], - "packaging": false, - "offline-model": "", - "offline-key": "", - "offline-response-key": "$id", - "auth": { - "Project": [] - } - }, - "security": [ - { - "Project": [] - } - ] - } - }, "\/migrations\/firebase\/report": { "get": { "summary": "Generate a report on Firebase data", @@ -18608,7 +18439,7 @@ }, "x-appwrite": { "method": "getFirebaseReport", - "weight": 342, + "weight": 341, "cookies": false, "type": "", "deprecated": false, @@ -18660,80 +18491,6 @@ ] } }, - "\/migrations\/firebase\/report\/oauth": { - "get": { - "summary": "Generate a report on Firebase data using OAuth", - "operationId": "migrationsGetFirebaseReportOAuth", - "tags": [ - "migrations" - ], - "description": "", - "responses": { - "200": { - "description": "Migration Report", - "content": { - "application\/json": { - "schema": { - "$ref": "#\/components\/schemas\/migrationReport" - } - } - } - } - }, - "x-appwrite": { - "method": "getFirebaseReportOAuth", - "weight": 343, - "cookies": false, - "type": "", - "deprecated": false, - "demo": "migrations\/get-firebase-report-o-auth.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/migrations\/migration-firebase-report.md", - "rate-limit": 0, - "rate-time": 3600, - "rate-key": "url:{url},ip:{ip}", - "scope": "migrations.write", - "platforms": [ - "console" - ], - "packaging": false, - "offline-model": "", - "offline-key": "", - "offline-response-key": "$id", - "auth": { - "Project": [] - } - }, - "security": [ - { - "Project": [] - } - ], - "parameters": [ - { - "name": "resources", - "description": "List of resources to migrate", - "required": true, - "schema": { - "type": "array", - "items": { - "type": "string" - } - }, - "in": "query" - }, - { - "name": "projectId", - "description": "Project ID", - "required": true, - "schema": { - "type": "string", - "x-example": "" - }, - "in": "query" - } - ] - } - }, "\/migrations\/nhost": { "post": { "summary": "Migrate NHost data", @@ -18756,7 +18513,7 @@ }, "x-appwrite": { "method": "createNHostMigration", - "weight": 338, + "weight": 337, "cookies": false, "type": "", "deprecated": false, @@ -18869,7 +18626,7 @@ }, "x-appwrite": { "method": "getNHostReport", - "weight": 349, + "weight": 343, "cookies": false, "type": "", "deprecated": false, @@ -19004,7 +18761,7 @@ }, "x-appwrite": { "method": "createSupabaseMigration", - "weight": 337, + "weight": 336, "cookies": false, "type": "", "deprecated": false, @@ -19111,7 +18868,7 @@ }, "x-appwrite": { "method": "getSupabaseReport", - "weight": 348, + "weight": 342, "cookies": false, "type": "", "deprecated": false, @@ -19237,7 +18994,7 @@ }, "x-appwrite": { "method": "get", - "weight": 340, + "weight": 339, "cookies": false, "type": "", "deprecated": false, @@ -19297,7 +19054,7 @@ }, "x-appwrite": { "method": "retry", - "weight": 350, + "weight": 344, "cookies": false, "type": "", "deprecated": false, @@ -19350,7 +19107,7 @@ }, "x-appwrite": { "method": "delete", - "weight": 351, + "weight": 345, "cookies": false, "type": "", "deprecated": false, @@ -32749,30 +32506,6 @@ "migrations" ] }, - "firebaseProjectList": { - "description": "Migrations Firebase Projects List", - "type": "object", - "properties": { - "total": { - "type": "integer", - "description": "Total number of projects documents that matched your query.", - "x-example": 5, - "format": "int32" - }, - "projects": { - "type": "array", - "description": "List of projects.", - "items": { - "$ref": "#\/components\/schemas\/firebaseProject" - }, - "x-example": "" - } - }, - "required": [ - "total", - "projects" - ] - }, "specificationList": { "description": "Specifications List", "type": "object", @@ -35109,7 +34842,7 @@ }, "schedule": { "type": "string", - "description": "Function execution schedult in CRON format.", + "description": "Function execution schedule in CRON format.", "x-example": "5 4 * * *" }, "timeout": { @@ -36976,7 +36709,8 @@ "resourceId": { "type": "string", "description": "Resource ID.", - "x-example": "5e5ea5c16897e" + "x-example": "5e5ea5c16897e", + "nullable": true }, "name": { "type": "string", @@ -36988,10 +36722,16 @@ "description": "The value of this metric at the timestamp.", "x-example": 1, "format": "int32" + }, + "estimate": { + "type": "number", + "description": "The estimated value of this metric at the end of the period.", + "x-example": 1, + "format": "double", + "nullable": true } }, "required": [ - "resourceId", "name", "value" ] @@ -37807,6 +37547,26 @@ "$ref": "#\/components\/schemas\/metricBreakdown" }, "x-example": [] + }, + "authPhoneTotal": { + "type": "integer", + "description": "Total aggregated number of phone auth.", + "x-example": 0, + "format": "int32" + }, + "authPhoneEstimate": { + "type": "number", + "description": "Estimated total aggregated cost of phone auth.", + "x-example": 0, + "format": "double" + }, + "authPhoneCountryBreakdown": { + "type": "array", + "description": "Aggregated breakdown in totals of phone auth by country.", + "items": { + "$ref": "#\/components\/schemas\/metricBreakdown" + }, + "x-example": [] } }, "required": [ @@ -37831,7 +37591,10 @@ "databasesStorageBreakdown", "executionsMbSecondsBreakdown", "buildsMbSecondsBreakdown", - "functionsStorageBreakdown" + "functionsStorageBreakdown", + "authPhoneTotal", + "authPhoneEstimate", + "authPhoneCountryBreakdown" ] }, "headers": { @@ -38707,26 +38470,6 @@ "size", "version" ] - }, - "firebaseProject": { - "description": "MigrationFirebaseProject", - "type": "object", - "properties": { - "projectId": { - "type": "string", - "description": "Project ID.", - "x-example": "my-project" - }, - "displayName": { - "type": "string", - "description": "Project display name.", - "x-example": "My Project" - } - }, - "required": [ - "projectId", - "displayName" - ] } }, "securitySchemes": { diff --git a/app/config/specs/open-api3-1.6.x-server.json b/app/config/specs/open-api3-1.6.x-server.json index e84b751743..8900d6ddf2 100644 --- a/app/config/specs/open-api3-1.6.x-server.json +++ b/app/config/specs/open-api3-1.6.x-server.json @@ -1,7 +1,7 @@ { "openapi": "3.0.0", "info": { - "version": "1.6.0", + "version": "1.6.1", "title": "Appwrite", "description": "Appwrite backend as a service cuts up to 70% of the time and costs required for building a modern application. We abstract and simplify common development tasks behind a REST APIs, to help you develop your app in a fast and secure way. For full API documentation and tutorials go to [https:\/\/appwrite.io\/docs](https:\/\/appwrite.io\/docs)", "termsOfService": "https:\/\/appwrite.io\/policy\/terms", @@ -2451,7 +2451,7 @@ "tags": [ "account" ], - "description": "Sends the user an email with a secret key for creating a session. If the provided user ID has not been registered, a new user will be created. When the user clicks the link in the email, the user is redirected back to the URL you provided with the secret key and userId values attached to the URL query string. Use the query string parameters to submit a request to the [POST \/v1\/account\/sessions\/token](https:\/\/appwrite.io\/docs\/references\/cloud\/client-web\/account#createSession) endpoint to complete the login process. The link sent to the user's email address is valid for 1 hour. If you are on a mobile device you can leave the URL parameter empty, so that the login completion will be handled by your Appwrite instance by default.\n\nA user is limited to 10 active sessions at a time by default. [Learn more about session limits](https:\/\/appwrite.io\/docs\/authentication-security#limits).\n", + "description": "Sends the user an email with a secret key for creating a session. If the provided user ID has not been registered, a new user will be created. When the user clicks the link in the email, the user is redirected back to the URL you provided with the secret key and userId values attached to the URL query string. Use the query string parameters to submit a request to the [POST \/v1\/account\/sessions\/token](https:\/\/appwrite.io\/docs\/references\/cloud\/client-web\/account#createSession) endpoint to complete the login process. The link sent to the user's email address is valid for 1 hour.\n\nA user is limited to 10 active sessions at a time by default. [Learn more about session limits](https:\/\/appwrite.io\/docs\/authentication-security#limits).\n", "responses": { "201": { "description": "Token", @@ -8596,7 +8596,8 @@ "bun-1.0", "bun-1.1", "go-1.23", - "static-1" + "static-1", + "flutter-3.24" ], "x-enum-name": null, "x-enum-keys": [] @@ -9019,7 +9020,8 @@ "bun-1.0", "bun-1.1", "go-1.23", - "static-1" + "static-1", + "flutter-3.24" ], "x-enum-name": null, "x-enum-keys": [] @@ -12527,7 +12529,7 @@ }, "x-appwrite": { "method": "listMessages", - "weight": 390, + "weight": 384, "cookies": false, "type": "", "deprecated": false, @@ -12606,7 +12608,7 @@ }, "x-appwrite": { "method": "createEmail", - "weight": 387, + "weight": 381, "cookies": false, "type": "", "deprecated": false, @@ -12753,7 +12755,7 @@ }, "x-appwrite": { "method": "updateEmail", - "weight": 394, + "weight": 388, "cookies": false, "type": "", "deprecated": false, @@ -12902,7 +12904,7 @@ }, "x-appwrite": { "method": "createPush", - "weight": 389, + "weight": 383, "cookies": false, "type": "", "deprecated": false, @@ -13060,7 +13062,7 @@ }, "x-appwrite": { "method": "updatePush", - "weight": 396, + "weight": 390, "cookies": false, "type": "", "deprecated": false, @@ -13220,7 +13222,7 @@ }, "x-appwrite": { "method": "createSms", - "weight": 388, + "weight": 382, "cookies": false, "type": "", "deprecated": false, @@ -13332,7 +13334,7 @@ }, "x-appwrite": { "method": "updateSms", - "weight": 395, + "weight": 389, "cookies": false, "type": "", "deprecated": false, @@ -13447,7 +13449,7 @@ }, "x-appwrite": { "method": "getMessage", - "weight": 393, + "weight": 387, "cookies": false, "type": "", "deprecated": false, @@ -13503,7 +13505,7 @@ }, "x-appwrite": { "method": "delete", - "weight": 397, + "weight": 391, "cookies": false, "type": "", "deprecated": false, @@ -13568,7 +13570,7 @@ }, "x-appwrite": { "method": "listMessageLogs", - "weight": 391, + "weight": 385, "cookies": false, "type": "", "deprecated": false, @@ -13646,7 +13648,7 @@ }, "x-appwrite": { "method": "listTargets", - "weight": 392, + "weight": 386, "cookies": false, "type": "", "deprecated": false, @@ -13724,7 +13726,7 @@ }, "x-appwrite": { "method": "listProviders", - "weight": 362, + "weight": 356, "cookies": false, "type": "", "deprecated": false, @@ -13803,7 +13805,7 @@ }, "x-appwrite": { "method": "createApnsProvider", - "weight": 361, + "weight": 355, "cookies": false, "type": "", "deprecated": false, @@ -13911,7 +13913,7 @@ }, "x-appwrite": { "method": "updateApnsProvider", - "weight": 374, + "weight": 368, "cookies": false, "type": "", "deprecated": false, @@ -14022,7 +14024,7 @@ }, "x-appwrite": { "method": "createFcmProvider", - "weight": 360, + "weight": 354, "cookies": false, "type": "", "deprecated": false, @@ -14110,7 +14112,7 @@ }, "x-appwrite": { "method": "updateFcmProvider", - "weight": 373, + "weight": 367, "cookies": false, "type": "", "deprecated": false, @@ -14201,7 +14203,7 @@ }, "x-appwrite": { "method": "createMailgunProvider", - "weight": 352, + "weight": 346, "cookies": false, "type": "", "deprecated": false, @@ -14319,7 +14321,7 @@ }, "x-appwrite": { "method": "updateMailgunProvider", - "weight": 365, + "weight": 359, "cookies": false, "type": "", "deprecated": false, @@ -14440,7 +14442,7 @@ }, "x-appwrite": { "method": "createMsg91Provider", - "weight": 355, + "weight": 349, "cookies": false, "type": "", "deprecated": false, @@ -14538,7 +14540,7 @@ }, "x-appwrite": { "method": "updateMsg91Provider", - "weight": 368, + "weight": 362, "cookies": false, "type": "", "deprecated": false, @@ -14639,7 +14641,7 @@ }, "x-appwrite": { "method": "createSendgridProvider", - "weight": 353, + "weight": 347, "cookies": false, "type": "", "deprecated": false, @@ -14747,7 +14749,7 @@ }, "x-appwrite": { "method": "updateSendgridProvider", - "weight": 366, + "weight": 360, "cookies": false, "type": "", "deprecated": false, @@ -14858,7 +14860,7 @@ }, "x-appwrite": { "method": "createSmtpProvider", - "weight": 354, + "weight": 348, "cookies": false, "type": "", "deprecated": false, @@ -15004,7 +15006,7 @@ }, "x-appwrite": { "method": "updateSmtpProvider", - "weight": 367, + "weight": 361, "cookies": false, "type": "", "deprecated": false, @@ -15152,7 +15154,7 @@ }, "x-appwrite": { "method": "createTelesignProvider", - "weight": 356, + "weight": 350, "cookies": false, "type": "", "deprecated": false, @@ -15250,7 +15252,7 @@ }, "x-appwrite": { "method": "updateTelesignProvider", - "weight": 369, + "weight": 363, "cookies": false, "type": "", "deprecated": false, @@ -15351,7 +15353,7 @@ }, "x-appwrite": { "method": "createTextmagicProvider", - "weight": 357, + "weight": 351, "cookies": false, "type": "", "deprecated": false, @@ -15449,7 +15451,7 @@ }, "x-appwrite": { "method": "updateTextmagicProvider", - "weight": 370, + "weight": 364, "cookies": false, "type": "", "deprecated": false, @@ -15550,7 +15552,7 @@ }, "x-appwrite": { "method": "createTwilioProvider", - "weight": 358, + "weight": 352, "cookies": false, "type": "", "deprecated": false, @@ -15648,7 +15650,7 @@ }, "x-appwrite": { "method": "updateTwilioProvider", - "weight": 371, + "weight": 365, "cookies": false, "type": "", "deprecated": false, @@ -15749,7 +15751,7 @@ }, "x-appwrite": { "method": "createVonageProvider", - "weight": 359, + "weight": 353, "cookies": false, "type": "", "deprecated": false, @@ -15847,7 +15849,7 @@ }, "x-appwrite": { "method": "updateVonageProvider", - "weight": 372, + "weight": 366, "cookies": false, "type": "", "deprecated": false, @@ -15948,7 +15950,7 @@ }, "x-appwrite": { "method": "getProvider", - "weight": 364, + "weight": 358, "cookies": false, "type": "", "deprecated": false, @@ -16004,7 +16006,7 @@ }, "x-appwrite": { "method": "deleteProvider", - "weight": 375, + "weight": 369, "cookies": false, "type": "", "deprecated": false, @@ -16069,7 +16071,7 @@ }, "x-appwrite": { "method": "listProviderLogs", - "weight": 363, + "weight": 357, "cookies": false, "type": "", "deprecated": false, @@ -16147,7 +16149,7 @@ }, "x-appwrite": { "method": "listSubscriberLogs", - "weight": 384, + "weight": 378, "cookies": false, "type": "", "deprecated": false, @@ -16225,7 +16227,7 @@ }, "x-appwrite": { "method": "listTopics", - "weight": 377, + "weight": 371, "cookies": false, "type": "", "deprecated": false, @@ -16302,7 +16304,7 @@ }, "x-appwrite": { "method": "createTopic", - "weight": 376, + "weight": 370, "cookies": false, "type": "", "deprecated": false, @@ -16388,7 +16390,7 @@ }, "x-appwrite": { "method": "getTopic", - "weight": 379, + "weight": 373, "cookies": false, "type": "", "deprecated": false, @@ -16451,7 +16453,7 @@ }, "x-appwrite": { "method": "updateTopic", - "weight": 380, + "weight": 374, "cookies": false, "type": "", "deprecated": false, @@ -16531,7 +16533,7 @@ }, "x-appwrite": { "method": "deleteTopic", - "weight": 381, + "weight": 375, "cookies": false, "type": "", "deprecated": false, @@ -16596,7 +16598,7 @@ }, "x-appwrite": { "method": "listTopicLogs", - "weight": 378, + "weight": 372, "cookies": false, "type": "", "deprecated": false, @@ -16674,7 +16676,7 @@ }, "x-appwrite": { "method": "listSubscribers", - "weight": 383, + "weight": 377, "cookies": false, "type": "", "deprecated": false, @@ -16761,7 +16763,7 @@ }, "x-appwrite": { "method": "createSubscriber", - "weight": 382, + "weight": 376, "cookies": false, "type": "", "deprecated": false, @@ -16855,7 +16857,7 @@ }, "x-appwrite": { "method": "getSubscriber", - "weight": 385, + "weight": 379, "cookies": false, "type": "", "deprecated": false, @@ -16921,7 +16923,7 @@ }, "x-appwrite": { "method": "deleteSubscriber", - "weight": 386, + "weight": 380, "cookies": false, "type": "", "deprecated": false, @@ -25746,7 +25748,7 @@ }, "schedule": { "type": "string", - "description": "Function execution schedult in CRON format.", + "description": "Function execution schedule in CRON format.", "x-example": "5 4 * * *" }, "timeout": { diff --git a/app/config/specs/open-api3-latest-console.json b/app/config/specs/open-api3-latest-console.json index 6878909f64..5b0e5376f5 100644 --- a/app/config/specs/open-api3-latest-console.json +++ b/app/config/specs/open-api3-latest-console.json @@ -36709,7 +36709,8 @@ "resourceId": { "type": "string", "description": "Resource ID.", - "x-example": "5e5ea5c16897e" + "x-example": "5e5ea5c16897e", + "nullable": true }, "name": { "type": "string", @@ -36721,10 +36722,16 @@ "description": "The value of this metric at the timestamp.", "x-example": 1, "format": "int32" + }, + "estimate": { + "type": "number", + "description": "The estimated value of this metric at the end of the period.", + "x-example": 1, + "format": "double", + "nullable": true } }, "required": [ - "resourceId", "name", "value" ] @@ -37540,6 +37547,26 @@ "$ref": "#\/components\/schemas\/metricBreakdown" }, "x-example": [] + }, + "authPhoneTotal": { + "type": "integer", + "description": "Total aggregated number of phone auth.", + "x-example": 0, + "format": "int32" + }, + "authPhoneEstimate": { + "type": "number", + "description": "Estimated total aggregated cost of phone auth.", + "x-example": 0, + "format": "double" + }, + "authPhoneCountryBreakdown": { + "type": "array", + "description": "Aggregated breakdown in totals of phone auth by country.", + "items": { + "$ref": "#\/components\/schemas\/metricBreakdown" + }, + "x-example": [] } }, "required": [ @@ -37564,7 +37591,10 @@ "databasesStorageBreakdown", "executionsMbSecondsBreakdown", "buildsMbSecondsBreakdown", - "functionsStorageBreakdown" + "functionsStorageBreakdown", + "authPhoneTotal", + "authPhoneEstimate", + "authPhoneCountryBreakdown" ] }, "headers": { diff --git a/app/config/specs/swagger2-1.6.x-client.json b/app/config/specs/swagger2-1.6.x-client.json index b1b9ce8dca..77025ec042 100644 --- a/app/config/specs/swagger2-1.6.x-client.json +++ b/app/config/specs/swagger2-1.6.x-client.json @@ -1,7 +1,7 @@ { "swagger": "2.0", "info": { - "version": "1.6.0", + "version": "1.6.1", "title": "Appwrite", "description": "Appwrite backend as a service cuts up to 70% of the time and costs required for building a modern application. We abstract and simplify common development tasks behind a REST APIs, to help you develop your app in a fast and secure way. For full API documentation and tutorials go to [https:\/\/appwrite.io\/docs](https:\/\/appwrite.io\/docs)", "termsOfService": "https:\/\/appwrite.io\/policy\/terms", @@ -2922,7 +2922,7 @@ "tags": [ "account" ], - "description": "Sends the user an email with a secret key for creating a session. If the provided user ID has not been registered, a new user will be created. When the user clicks the link in the email, the user is redirected back to the URL you provided with the secret key and userId values attached to the URL query string. Use the query string parameters to submit a request to the [POST \/v1\/account\/sessions\/token](https:\/\/appwrite.io\/docs\/references\/cloud\/client-web\/account#createSession) endpoint to complete the login process. The link sent to the user's email address is valid for 1 hour. If you are on a mobile device you can leave the URL parameter empty, so that the login completion will be handled by your Appwrite instance by default.\n\nA user is limited to 10 active sessions at a time by default. [Learn more about session limits](https:\/\/appwrite.io\/docs\/authentication-security#limits).\n", + "description": "Sends the user an email with a secret key for creating a session. If the provided user ID has not been registered, a new user will be created. When the user clicks the link in the email, the user is redirected back to the URL you provided with the secret key and userId values attached to the URL query string. Use the query string parameters to submit a request to the [POST \/v1\/account\/sessions\/token](https:\/\/appwrite.io\/docs\/references\/cloud\/client-web\/account#createSession) endpoint to complete the login process. The link sent to the user's email address is valid for 1 hour.\n\nA user is limited to 10 active sessions at a time by default. [Learn more about session limits](https:\/\/appwrite.io\/docs\/authentication-security#limits).\n", "responses": { "201": { "description": "Token", @@ -5981,7 +5981,7 @@ }, "x-appwrite": { "method": "createSubscriber", - "weight": 382, + "weight": 376, "cookies": false, "type": "", "deprecated": false, @@ -6070,7 +6070,7 @@ }, "x-appwrite": { "method": "deleteSubscriber", - "weight": 386, + "weight": 380, "cookies": false, "type": "", "deprecated": false, diff --git a/app/config/specs/swagger2-1.6.x-console.json b/app/config/specs/swagger2-1.6.x-console.json index 9ce9d5f60d..d5c82bba49 100644 --- a/app/config/specs/swagger2-1.6.x-console.json +++ b/app/config/specs/swagger2-1.6.x-console.json @@ -1,7 +1,7 @@ { "swagger": "2.0", "info": { - "version": "1.6.0", + "version": "1.6.1", "title": "Appwrite", "description": "Appwrite backend as a service cuts up to 70% of the time and costs required for building a modern application. We abstract and simplify common development tasks behind a REST APIs, to help you develop your app in a fast and secure way. For full API documentation and tutorials go to [https:\/\/appwrite.io\/docs](https:\/\/appwrite.io\/docs)", "termsOfService": "https:\/\/appwrite.io\/policy\/terms", @@ -2949,7 +2949,7 @@ "tags": [ "account" ], - "description": "Sends the user an email with a secret key for creating a session. If the provided user ID has not been registered, a new user will be created. When the user clicks the link in the email, the user is redirected back to the URL you provided with the secret key and userId values attached to the URL query string. Use the query string parameters to submit a request to the [POST \/v1\/account\/sessions\/token](https:\/\/appwrite.io\/docs\/references\/cloud\/client-web\/account#createSession) endpoint to complete the login process. The link sent to the user's email address is valid for 1 hour. If you are on a mobile device you can leave the URL parameter empty, so that the login completion will be handled by your Appwrite instance by default.\n\nA user is limited to 10 active sessions at a time by default. [Learn more about session limits](https:\/\/appwrite.io\/docs\/authentication-security#limits).\n", + "description": "Sends the user an email with a secret key for creating a session. If the provided user ID has not been registered, a new user will be created. When the user clicks the link in the email, the user is redirected back to the URL you provided with the secret key and userId values attached to the URL query string. Use the query string parameters to submit a request to the [POST \/v1\/account\/sessions\/token](https:\/\/appwrite.io\/docs\/references\/cloud\/client-web\/account#createSession) endpoint to complete the login process. The link sent to the user's email address is valid for 1 hour.\n\nA user is limited to 10 active sessions at a time by default. [Learn more about session limits](https:\/\/appwrite.io\/docs\/authentication-security#limits).\n", "responses": { "201": { "description": "Token", @@ -9610,7 +9610,8 @@ "bun-1.0", "bun-1.1", "go-1.23", - "static-1" + "static-1", + "flutter-3.24" ], "x-enum-name": null, "x-enum-keys": [] @@ -10286,7 +10287,8 @@ "bun-1.0", "bun-1.1", "go-1.23", - "static-1" + "static-1", + "flutter-3.24" ], "x-enum-name": null, "x-enum-keys": [] @@ -13890,7 +13892,7 @@ }, "x-appwrite": { "method": "listMessages", - "weight": 390, + "weight": 384, "cookies": false, "type": "", "deprecated": false, @@ -13967,7 +13969,7 @@ }, "x-appwrite": { "method": "createEmail", - "weight": 387, + "weight": 381, "cookies": false, "type": "", "deprecated": false, @@ -14127,7 +14129,7 @@ }, "x-appwrite": { "method": "updateEmail", - "weight": 394, + "weight": 388, "cookies": false, "type": "", "deprecated": false, @@ -14284,7 +14286,7 @@ }, "x-appwrite": { "method": "createPush", - "weight": 389, + "weight": 383, "cookies": false, "type": "", "deprecated": false, @@ -14459,7 +14461,7 @@ }, "x-appwrite": { "method": "updatePush", - "weight": 396, + "weight": 390, "cookies": false, "type": "", "deprecated": false, @@ -14631,7 +14633,7 @@ }, "x-appwrite": { "method": "createSms", - "weight": 388, + "weight": 382, "cookies": false, "type": "", "deprecated": false, @@ -14751,7 +14753,7 @@ }, "x-appwrite": { "method": "updateSms", - "weight": 395, + "weight": 389, "cookies": false, "type": "", "deprecated": false, @@ -14869,7 +14871,7 @@ }, "x-appwrite": { "method": "getMessage", - "weight": 393, + "weight": 387, "cookies": false, "type": "", "deprecated": false, @@ -14928,7 +14930,7 @@ }, "x-appwrite": { "method": "delete", - "weight": 397, + "weight": 391, "cookies": false, "type": "", "deprecated": false, @@ -14992,7 +14994,7 @@ }, "x-appwrite": { "method": "listMessageLogs", - "weight": 391, + "weight": 385, "cookies": false, "type": "", "deprecated": false, @@ -15068,7 +15070,7 @@ }, "x-appwrite": { "method": "listTargets", - "weight": 392, + "weight": 386, "cookies": false, "type": "", "deprecated": false, @@ -15144,7 +15146,7 @@ }, "x-appwrite": { "method": "listProviders", - "weight": 362, + "weight": 356, "cookies": false, "type": "", "deprecated": false, @@ -15221,7 +15223,7 @@ }, "x-appwrite": { "method": "createApnsProvider", - "weight": 361, + "weight": 355, "cookies": false, "type": "", "deprecated": false, @@ -15338,7 +15340,7 @@ }, "x-appwrite": { "method": "updateApnsProvider", - "weight": 374, + "weight": 368, "cookies": false, "type": "", "deprecated": false, @@ -15453,7 +15455,7 @@ }, "x-appwrite": { "method": "createFcmProvider", - "weight": 360, + "weight": 354, "cookies": false, "type": "", "deprecated": false, @@ -15546,7 +15548,7 @@ }, "x-appwrite": { "method": "updateFcmProvider", - "weight": 373, + "weight": 367, "cookies": false, "type": "", "deprecated": false, @@ -15637,7 +15639,7 @@ }, "x-appwrite": { "method": "createMailgunProvider", - "weight": 352, + "weight": 346, "cookies": false, "type": "", "deprecated": false, @@ -15766,7 +15768,7 @@ }, "x-appwrite": { "method": "updateMailgunProvider", - "weight": 365, + "weight": 359, "cookies": false, "type": "", "deprecated": false, @@ -15893,7 +15895,7 @@ }, "x-appwrite": { "method": "createMsg91Provider", - "weight": 355, + "weight": 349, "cookies": false, "type": "", "deprecated": false, @@ -15998,7 +16000,7 @@ }, "x-appwrite": { "method": "updateMsg91Provider", - "weight": 368, + "weight": 362, "cookies": false, "type": "", "deprecated": false, @@ -16101,7 +16103,7 @@ }, "x-appwrite": { "method": "createSendgridProvider", - "weight": 353, + "weight": 347, "cookies": false, "type": "", "deprecated": false, @@ -16218,7 +16220,7 @@ }, "x-appwrite": { "method": "updateSendgridProvider", - "weight": 366, + "weight": 360, "cookies": false, "type": "", "deprecated": false, @@ -16333,7 +16335,7 @@ }, "x-appwrite": { "method": "createSmtpProvider", - "weight": 354, + "weight": 348, "cookies": false, "type": "", "deprecated": false, @@ -16494,7 +16496,7 @@ }, "x-appwrite": { "method": "updateSmtpProvider", - "weight": 367, + "weight": 361, "cookies": false, "type": "", "deprecated": false, @@ -16652,7 +16654,7 @@ }, "x-appwrite": { "method": "createTelesignProvider", - "weight": 356, + "weight": 350, "cookies": false, "type": "", "deprecated": false, @@ -16757,7 +16759,7 @@ }, "x-appwrite": { "method": "updateTelesignProvider", - "weight": 369, + "weight": 363, "cookies": false, "type": "", "deprecated": false, @@ -16860,7 +16862,7 @@ }, "x-appwrite": { "method": "createTextmagicProvider", - "weight": 357, + "weight": 351, "cookies": false, "type": "", "deprecated": false, @@ -16965,7 +16967,7 @@ }, "x-appwrite": { "method": "updateTextmagicProvider", - "weight": 370, + "weight": 364, "cookies": false, "type": "", "deprecated": false, @@ -17068,7 +17070,7 @@ }, "x-appwrite": { "method": "createTwilioProvider", - "weight": 358, + "weight": 352, "cookies": false, "type": "", "deprecated": false, @@ -17173,7 +17175,7 @@ }, "x-appwrite": { "method": "updateTwilioProvider", - "weight": 371, + "weight": 365, "cookies": false, "type": "", "deprecated": false, @@ -17276,7 +17278,7 @@ }, "x-appwrite": { "method": "createVonageProvider", - "weight": 359, + "weight": 353, "cookies": false, "type": "", "deprecated": false, @@ -17381,7 +17383,7 @@ }, "x-appwrite": { "method": "updateVonageProvider", - "weight": 372, + "weight": 366, "cookies": false, "type": "", "deprecated": false, @@ -17484,7 +17486,7 @@ }, "x-appwrite": { "method": "getProvider", - "weight": 364, + "weight": 358, "cookies": false, "type": "", "deprecated": false, @@ -17543,7 +17545,7 @@ }, "x-appwrite": { "method": "deleteProvider", - "weight": 375, + "weight": 369, "cookies": false, "type": "", "deprecated": false, @@ -17607,7 +17609,7 @@ }, "x-appwrite": { "method": "listProviderLogs", - "weight": 363, + "weight": 357, "cookies": false, "type": "", "deprecated": false, @@ -17683,7 +17685,7 @@ }, "x-appwrite": { "method": "listSubscriberLogs", - "weight": 384, + "weight": 378, "cookies": false, "type": "", "deprecated": false, @@ -17759,7 +17761,7 @@ }, "x-appwrite": { "method": "listTopics", - "weight": 377, + "weight": 371, "cookies": false, "type": "", "deprecated": false, @@ -17834,7 +17836,7 @@ }, "x-appwrite": { "method": "createTopic", - "weight": 376, + "weight": 370, "cookies": false, "type": "", "deprecated": false, @@ -17926,7 +17928,7 @@ }, "x-appwrite": { "method": "getTopic", - "weight": 379, + "weight": 373, "cookies": false, "type": "", "deprecated": false, @@ -17988,7 +17990,7 @@ }, "x-appwrite": { "method": "updateTopic", - "weight": 380, + "weight": 374, "cookies": false, "type": "", "deprecated": false, @@ -18071,7 +18073,7 @@ }, "x-appwrite": { "method": "deleteTopic", - "weight": 381, + "weight": 375, "cookies": false, "type": "", "deprecated": false, @@ -18135,7 +18137,7 @@ }, "x-appwrite": { "method": "listTopicLogs", - "weight": 378, + "weight": 372, "cookies": false, "type": "", "deprecated": false, @@ -18211,7 +18213,7 @@ }, "x-appwrite": { "method": "listSubscribers", - "weight": 383, + "weight": 377, "cookies": false, "type": "", "deprecated": false, @@ -18294,7 +18296,7 @@ }, "x-appwrite": { "method": "createSubscriber", - "weight": 382, + "weight": 376, "cookies": false, "type": "", "deprecated": false, @@ -18386,7 +18388,7 @@ }, "x-appwrite": { "method": "getSubscriber", - "weight": 385, + "weight": 379, "cookies": false, "type": "", "deprecated": false, @@ -18453,7 +18455,7 @@ }, "x-appwrite": { "method": "deleteSubscriber", - "weight": 386, + "weight": 380, "cookies": false, "type": "", "deprecated": false, @@ -18528,7 +18530,7 @@ }, "x-appwrite": { "method": "list", - "weight": 339, + "weight": 338, "cookies": false, "type": "", "deprecated": false, @@ -18699,7 +18701,7 @@ }, "x-appwrite": { "method": "getAppwriteReport", - "weight": 341, + "weight": 340, "cookies": false, "type": "", "deprecated": false, @@ -18767,7 +18769,7 @@ }, "\/migrations\/firebase": { "post": { - "summary": "Migrate Firebase data (Service Account)", + "summary": "Migrate Firebase data", "operationId": "migrationsCreateFirebaseMigration", "consumes": [ "application\/json" @@ -18789,7 +18791,7 @@ }, "x-appwrite": { "method": "createFirebaseMigration", - "weight": 336, + "weight": 335, "cookies": false, "type": "", "deprecated": false, @@ -18847,192 +18849,6 @@ ] } }, - "\/migrations\/firebase\/deauthorize": { - "get": { - "summary": "Revoke Appwrite's authorization to access Firebase projects", - "operationId": "migrationsDeleteFirebaseAuth", - "consumes": [ - "application\/json" - ], - "produces": [ - "application\/json" - ], - "tags": [ - "migrations" - ], - "description": "", - "responses": { - "200": { - "description": "File", - "schema": { - "type": "file" - } - } - }, - "x-appwrite": { - "method": "deleteFirebaseAuth", - "weight": 347, - "cookies": false, - "type": "", - "deprecated": false, - "demo": "migrations\/delete-firebase-auth.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master", - "rate-limit": 0, - "rate-time": 3600, - "rate-key": "url:{url},ip:{ip}", - "scope": "migrations.write", - "platforms": [ - "console" - ], - "packaging": false, - "offline-model": "", - "offline-key": "", - "offline-response-key": "$id", - "auth": { - "Project": [] - } - }, - "security": [ - { - "Project": [] - } - ] - } - }, - "\/migrations\/firebase\/oauth": { - "post": { - "summary": "Migrate Firebase data (OAuth)", - "operationId": "migrationsCreateFirebaseOAuthMigration", - "consumes": [ - "application\/json" - ], - "produces": [ - "application\/json" - ], - "tags": [ - "migrations" - ], - "description": "", - "responses": { - "202": { - "description": "Migration", - "schema": { - "$ref": "#\/definitions\/migration" - } - } - }, - "x-appwrite": { - "method": "createFirebaseOAuthMigration", - "weight": 335, - "cookies": false, - "type": "", - "deprecated": false, - "demo": "migrations\/create-firebase-o-auth-migration.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/migrations\/migration-firebase.md", - "rate-limit": 0, - "rate-time": 3600, - "rate-key": "url:{url},ip:{ip}", - "scope": "migrations.write", - "platforms": [ - "console" - ], - "packaging": false, - "offline-model": "", - "offline-key": "", - "offline-response-key": "$id", - "auth": { - "Project": [] - } - }, - "security": [ - { - "Project": [] - } - ], - "parameters": [ - { - "name": "payload", - "in": "body", - "schema": { - "type": "object", - "properties": { - "resources": { - "type": "array", - "description": "List of resources to migrate", - "default": null, - "x-example": null, - "items": { - "type": "string" - } - }, - "projectId": { - "type": "string", - "description": "Project ID of the Firebase Project", - "default": null, - "x-example": "" - } - }, - "required": [ - "resources", - "projectId" - ] - } - } - ] - } - }, - "\/migrations\/firebase\/projects": { - "get": { - "summary": "List Firebase projects", - "operationId": "migrationsListFirebaseProjects", - "consumes": [ - "application\/json" - ], - "produces": [ - "application\/json" - ], - "tags": [ - "migrations" - ], - "description": "", - "responses": { - "200": { - "description": "Migrations Firebase Projects List", - "schema": { - "$ref": "#\/definitions\/firebaseProjectList" - } - } - }, - "x-appwrite": { - "method": "listFirebaseProjects", - "weight": 346, - "cookies": false, - "type": "", - "deprecated": false, - "demo": "migrations\/list-firebase-projects.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master", - "rate-limit": 0, - "rate-time": 3600, - "rate-key": "url:{url},ip:{ip}", - "scope": "migrations.read", - "platforms": [ - "console" - ], - "packaging": false, - "offline-model": "", - "offline-key": "", - "offline-response-key": "$id", - "auth": { - "Project": [] - } - }, - "security": [ - { - "Project": [] - } - ] - } - }, "\/migrations\/firebase\/report": { "get": { "summary": "Generate a report on Firebase data", @@ -19057,7 +18873,7 @@ }, "x-appwrite": { "method": "getFirebaseReport", - "weight": 342, + "weight": 341, "cookies": false, "type": "", "deprecated": false, @@ -19106,79 +18922,6 @@ ] } }, - "\/migrations\/firebase\/report\/oauth": { - "get": { - "summary": "Generate a report on Firebase data using OAuth", - "operationId": "migrationsGetFirebaseReportOAuth", - "consumes": [ - "application\/json" - ], - "produces": [ - "application\/json" - ], - "tags": [ - "migrations" - ], - "description": "", - "responses": { - "200": { - "description": "Migration Report", - "schema": { - "$ref": "#\/definitions\/migrationReport" - } - } - }, - "x-appwrite": { - "method": "getFirebaseReportOAuth", - "weight": 343, - "cookies": false, - "type": "", - "deprecated": false, - "demo": "migrations\/get-firebase-report-o-auth.md", - "edit": "https:\/\/github.com\/appwrite\/appwrite\/edit\/master\/docs\/references\/migrations\/migration-firebase-report.md", - "rate-limit": 0, - "rate-time": 3600, - "rate-key": "url:{url},ip:{ip}", - "scope": "migrations.write", - "platforms": [ - "console" - ], - "packaging": false, - "offline-model": "", - "offline-key": "", - "offline-response-key": "$id", - "auth": { - "Project": [] - } - }, - "security": [ - { - "Project": [] - } - ], - "parameters": [ - { - "name": "resources", - "description": "List of resources to migrate", - "required": true, - "type": "array", - "collectionFormat": "multi", - "items": { - "type": "string" - }, - "in": "query" - }, - { - "name": "projectId", - "description": "Project ID", - "required": true, - "type": "string", - "x-example": "", - "in": "query" - } - ] - } - }, "\/migrations\/nhost": { "post": { "summary": "Migrate NHost data", @@ -19203,7 +18946,7 @@ }, "x-appwrite": { "method": "createNHostMigration", - "weight": 338, + "weight": 337, "cookies": false, "type": "", "deprecated": false, @@ -19326,7 +19069,7 @@ }, "x-appwrite": { "method": "getNHostReport", - "weight": 349, + "weight": 343, "cookies": false, "type": "", "deprecated": false, @@ -19448,7 +19191,7 @@ }, "x-appwrite": { "method": "createSupabaseMigration", - "weight": 337, + "weight": 336, "cookies": false, "type": "", "deprecated": false, @@ -19564,7 +19307,7 @@ }, "x-appwrite": { "method": "getSupabaseReport", - "weight": 348, + "weight": 342, "cookies": false, "type": "", "deprecated": false, @@ -19679,7 +19422,7 @@ }, "x-appwrite": { "method": "get", - "weight": 340, + "weight": 339, "cookies": false, "type": "", "deprecated": false, @@ -19739,7 +19482,7 @@ }, "x-appwrite": { "method": "retry", - "weight": 350, + "weight": 344, "cookies": false, "type": "", "deprecated": false, @@ -19794,7 +19537,7 @@ }, "x-appwrite": { "method": "delete", - "weight": 351, + "weight": 345, "cookies": false, "type": "", "deprecated": false, @@ -33250,31 +32993,6 @@ "migrations" ] }, - "firebaseProjectList": { - "description": "Migrations Firebase Projects List", - "type": "object", - "properties": { - "total": { - "type": "integer", - "description": "Total number of projects documents that matched your query.", - "x-example": 5, - "format": "int32" - }, - "projects": { - "type": "array", - "description": "List of projects.", - "items": { - "type": "object", - "$ref": "#\/definitions\/firebaseProject" - }, - "x-example": "" - } - }, - "required": [ - "total", - "projects" - ] - }, "specificationList": { "description": "Specifications List", "type": "object", @@ -35618,7 +35336,7 @@ }, "schedule": { "type": "string", - "description": "Function execution schedult in CRON format.", + "description": "Function execution schedule in CRON format.", "x-example": "5 4 * * *" }, "timeout": { @@ -37494,7 +37212,8 @@ "resourceId": { "type": "string", "description": "Resource ID.", - "x-example": "5e5ea5c16897e" + "x-example": "5e5ea5c16897e", + "x-nullable": true }, "name": { "type": "string", @@ -37506,10 +37225,16 @@ "description": "The value of this metric at the timestamp.", "x-example": 1, "format": "int32" + }, + "estimate": { + "type": "number", + "description": "The estimated value of this metric at the end of the period.", + "x-example": 1, + "format": "double", + "x-nullable": true } }, "required": [ - "resourceId", "name", "value" ] @@ -38369,6 +38094,27 @@ "$ref": "#\/definitions\/metricBreakdown" }, "x-example": [] + }, + "authPhoneTotal": { + "type": "integer", + "description": "Total aggregated number of phone auth.", + "x-example": 0, + "format": "int32" + }, + "authPhoneEstimate": { + "type": "number", + "description": "Estimated total aggregated cost of phone auth.", + "x-example": 0, + "format": "double" + }, + "authPhoneCountryBreakdown": { + "type": "array", + "description": "Aggregated breakdown in totals of phone auth by country.", + "items": { + "type": "object", + "$ref": "#\/definitions\/metricBreakdown" + }, + "x-example": [] } }, "required": [ @@ -38393,7 +38139,10 @@ "databasesStorageBreakdown", "executionsMbSecondsBreakdown", "buildsMbSecondsBreakdown", - "functionsStorageBreakdown" + "functionsStorageBreakdown", + "authPhoneTotal", + "authPhoneEstimate", + "authPhoneCountryBreakdown" ] }, "headers": { @@ -39274,26 +39023,6 @@ "size", "version" ] - }, - "firebaseProject": { - "description": "MigrationFirebaseProject", - "type": "object", - "properties": { - "projectId": { - "type": "string", - "description": "Project ID.", - "x-example": "my-project" - }, - "displayName": { - "type": "string", - "description": "Project display name.", - "x-example": "My Project" - } - }, - "required": [ - "projectId", - "displayName" - ] } }, "externalDocs": { diff --git a/app/config/specs/swagger2-1.6.x-server.json b/app/config/specs/swagger2-1.6.x-server.json index 2c8e80c65e..f5b1f24a22 100644 --- a/app/config/specs/swagger2-1.6.x-server.json +++ b/app/config/specs/swagger2-1.6.x-server.json @@ -1,7 +1,7 @@ { "swagger": "2.0", "info": { - "version": "1.6.0", + "version": "1.6.1", "title": "Appwrite", "description": "Appwrite backend as a service cuts up to 70% of the time and costs required for building a modern application. We abstract and simplify common development tasks behind a REST APIs, to help you develop your app in a fast and secure way. For full API documentation and tutorials go to [https:\/\/appwrite.io\/docs](https:\/\/appwrite.io\/docs)", "termsOfService": "https:\/\/appwrite.io\/policy\/terms", @@ -2604,7 +2604,7 @@ "tags": [ "account" ], - "description": "Sends the user an email with a secret key for creating a session. If the provided user ID has not been registered, a new user will be created. When the user clicks the link in the email, the user is redirected back to the URL you provided with the secret key and userId values attached to the URL query string. Use the query string parameters to submit a request to the [POST \/v1\/account\/sessions\/token](https:\/\/appwrite.io\/docs\/references\/cloud\/client-web\/account#createSession) endpoint to complete the login process. The link sent to the user's email address is valid for 1 hour. If you are on a mobile device you can leave the URL parameter empty, so that the login completion will be handled by your Appwrite instance by default.\n\nA user is limited to 10 active sessions at a time by default. [Learn more about session limits](https:\/\/appwrite.io\/docs\/authentication-security#limits).\n", + "description": "Sends the user an email with a secret key for creating a session. If the provided user ID has not been registered, a new user will be created. When the user clicks the link in the email, the user is redirected back to the URL you provided with the secret key and userId values attached to the URL query string. Use the query string parameters to submit a request to the [POST \/v1\/account\/sessions\/token](https:\/\/appwrite.io\/docs\/references\/cloud\/client-web\/account#createSession) endpoint to complete the login process. The link sent to the user's email address is valid for 1 hour.\n\nA user is limited to 10 active sessions at a time by default. [Learn more about session limits](https:\/\/appwrite.io\/docs\/authentication-security#limits).\n", "responses": { "201": { "description": "Token", @@ -8720,7 +8720,8 @@ "bun-1.0", "bun-1.1", "go-1.23", - "static-1" + "static-1", + "flutter-3.24" ], "x-enum-name": null, "x-enum-keys": [] @@ -9166,7 +9167,8 @@ "bun-1.0", "bun-1.1", "go-1.23", - "static-1" + "static-1", + "flutter-3.24" ], "x-enum-name": null, "x-enum-keys": [] @@ -12752,7 +12754,7 @@ }, "x-appwrite": { "method": "listMessages", - "weight": 390, + "weight": 384, "cookies": false, "type": "", "deprecated": false, @@ -12830,7 +12832,7 @@ }, "x-appwrite": { "method": "createEmail", - "weight": 387, + "weight": 381, "cookies": false, "type": "", "deprecated": false, @@ -12991,7 +12993,7 @@ }, "x-appwrite": { "method": "updateEmail", - "weight": 394, + "weight": 388, "cookies": false, "type": "", "deprecated": false, @@ -13149,7 +13151,7 @@ }, "x-appwrite": { "method": "createPush", - "weight": 389, + "weight": 383, "cookies": false, "type": "", "deprecated": false, @@ -13325,7 +13327,7 @@ }, "x-appwrite": { "method": "updatePush", - "weight": 396, + "weight": 390, "cookies": false, "type": "", "deprecated": false, @@ -13498,7 +13500,7 @@ }, "x-appwrite": { "method": "createSms", - "weight": 388, + "weight": 382, "cookies": false, "type": "", "deprecated": false, @@ -13619,7 +13621,7 @@ }, "x-appwrite": { "method": "updateSms", - "weight": 395, + "weight": 389, "cookies": false, "type": "", "deprecated": false, @@ -13738,7 +13740,7 @@ }, "x-appwrite": { "method": "getMessage", - "weight": 393, + "weight": 387, "cookies": false, "type": "", "deprecated": false, @@ -13798,7 +13800,7 @@ }, "x-appwrite": { "method": "delete", - "weight": 397, + "weight": 391, "cookies": false, "type": "", "deprecated": false, @@ -13863,7 +13865,7 @@ }, "x-appwrite": { "method": "listMessageLogs", - "weight": 391, + "weight": 385, "cookies": false, "type": "", "deprecated": false, @@ -13940,7 +13942,7 @@ }, "x-appwrite": { "method": "listTargets", - "weight": 392, + "weight": 386, "cookies": false, "type": "", "deprecated": false, @@ -14017,7 +14019,7 @@ }, "x-appwrite": { "method": "listProviders", - "weight": 362, + "weight": 356, "cookies": false, "type": "", "deprecated": false, @@ -14095,7 +14097,7 @@ }, "x-appwrite": { "method": "createApnsProvider", - "weight": 361, + "weight": 355, "cookies": false, "type": "", "deprecated": false, @@ -14213,7 +14215,7 @@ }, "x-appwrite": { "method": "updateApnsProvider", - "weight": 374, + "weight": 368, "cookies": false, "type": "", "deprecated": false, @@ -14329,7 +14331,7 @@ }, "x-appwrite": { "method": "createFcmProvider", - "weight": 360, + "weight": 354, "cookies": false, "type": "", "deprecated": false, @@ -14423,7 +14425,7 @@ }, "x-appwrite": { "method": "updateFcmProvider", - "weight": 373, + "weight": 367, "cookies": false, "type": "", "deprecated": false, @@ -14515,7 +14517,7 @@ }, "x-appwrite": { "method": "createMailgunProvider", - "weight": 352, + "weight": 346, "cookies": false, "type": "", "deprecated": false, @@ -14645,7 +14647,7 @@ }, "x-appwrite": { "method": "updateMailgunProvider", - "weight": 365, + "weight": 359, "cookies": false, "type": "", "deprecated": false, @@ -14773,7 +14775,7 @@ }, "x-appwrite": { "method": "createMsg91Provider", - "weight": 355, + "weight": 349, "cookies": false, "type": "", "deprecated": false, @@ -14879,7 +14881,7 @@ }, "x-appwrite": { "method": "updateMsg91Provider", - "weight": 368, + "weight": 362, "cookies": false, "type": "", "deprecated": false, @@ -14983,7 +14985,7 @@ }, "x-appwrite": { "method": "createSendgridProvider", - "weight": 353, + "weight": 347, "cookies": false, "type": "", "deprecated": false, @@ -15101,7 +15103,7 @@ }, "x-appwrite": { "method": "updateSendgridProvider", - "weight": 366, + "weight": 360, "cookies": false, "type": "", "deprecated": false, @@ -15217,7 +15219,7 @@ }, "x-appwrite": { "method": "createSmtpProvider", - "weight": 354, + "weight": 348, "cookies": false, "type": "", "deprecated": false, @@ -15379,7 +15381,7 @@ }, "x-appwrite": { "method": "updateSmtpProvider", - "weight": 367, + "weight": 361, "cookies": false, "type": "", "deprecated": false, @@ -15538,7 +15540,7 @@ }, "x-appwrite": { "method": "createTelesignProvider", - "weight": 356, + "weight": 350, "cookies": false, "type": "", "deprecated": false, @@ -15644,7 +15646,7 @@ }, "x-appwrite": { "method": "updateTelesignProvider", - "weight": 369, + "weight": 363, "cookies": false, "type": "", "deprecated": false, @@ -15748,7 +15750,7 @@ }, "x-appwrite": { "method": "createTextmagicProvider", - "weight": 357, + "weight": 351, "cookies": false, "type": "", "deprecated": false, @@ -15854,7 +15856,7 @@ }, "x-appwrite": { "method": "updateTextmagicProvider", - "weight": 370, + "weight": 364, "cookies": false, "type": "", "deprecated": false, @@ -15958,7 +15960,7 @@ }, "x-appwrite": { "method": "createTwilioProvider", - "weight": 358, + "weight": 352, "cookies": false, "type": "", "deprecated": false, @@ -16064,7 +16066,7 @@ }, "x-appwrite": { "method": "updateTwilioProvider", - "weight": 371, + "weight": 365, "cookies": false, "type": "", "deprecated": false, @@ -16168,7 +16170,7 @@ }, "x-appwrite": { "method": "createVonageProvider", - "weight": 359, + "weight": 353, "cookies": false, "type": "", "deprecated": false, @@ -16274,7 +16276,7 @@ }, "x-appwrite": { "method": "updateVonageProvider", - "weight": 372, + "weight": 366, "cookies": false, "type": "", "deprecated": false, @@ -16378,7 +16380,7 @@ }, "x-appwrite": { "method": "getProvider", - "weight": 364, + "weight": 358, "cookies": false, "type": "", "deprecated": false, @@ -16438,7 +16440,7 @@ }, "x-appwrite": { "method": "deleteProvider", - "weight": 375, + "weight": 369, "cookies": false, "type": "", "deprecated": false, @@ -16503,7 +16505,7 @@ }, "x-appwrite": { "method": "listProviderLogs", - "weight": 363, + "weight": 357, "cookies": false, "type": "", "deprecated": false, @@ -16580,7 +16582,7 @@ }, "x-appwrite": { "method": "listSubscriberLogs", - "weight": 384, + "weight": 378, "cookies": false, "type": "", "deprecated": false, @@ -16657,7 +16659,7 @@ }, "x-appwrite": { "method": "listTopics", - "weight": 377, + "weight": 371, "cookies": false, "type": "", "deprecated": false, @@ -16733,7 +16735,7 @@ }, "x-appwrite": { "method": "createTopic", - "weight": 376, + "weight": 370, "cookies": false, "type": "", "deprecated": false, @@ -16826,7 +16828,7 @@ }, "x-appwrite": { "method": "getTopic", - "weight": 379, + "weight": 373, "cookies": false, "type": "", "deprecated": false, @@ -16889,7 +16891,7 @@ }, "x-appwrite": { "method": "updateTopic", - "weight": 380, + "weight": 374, "cookies": false, "type": "", "deprecated": false, @@ -16973,7 +16975,7 @@ }, "x-appwrite": { "method": "deleteTopic", - "weight": 381, + "weight": 375, "cookies": false, "type": "", "deprecated": false, @@ -17038,7 +17040,7 @@ }, "x-appwrite": { "method": "listTopicLogs", - "weight": 378, + "weight": 372, "cookies": false, "type": "", "deprecated": false, @@ -17115,7 +17117,7 @@ }, "x-appwrite": { "method": "listSubscribers", - "weight": 383, + "weight": 377, "cookies": false, "type": "", "deprecated": false, @@ -17199,7 +17201,7 @@ }, "x-appwrite": { "method": "createSubscriber", - "weight": 382, + "weight": 376, "cookies": false, "type": "", "deprecated": false, @@ -17293,7 +17295,7 @@ }, "x-appwrite": { "method": "getSubscriber", - "weight": 385, + "weight": 379, "cookies": false, "type": "", "deprecated": false, @@ -17361,7 +17363,7 @@ }, "x-appwrite": { "method": "deleteSubscriber", - "weight": 386, + "weight": 380, "cookies": false, "type": "", "deprecated": false, @@ -26233,7 +26235,7 @@ }, "schedule": { "type": "string", - "description": "Function execution schedult in CRON format.", + "description": "Function execution schedule in CRON format.", "x-example": "5 4 * * *" }, "timeout": { diff --git a/app/config/specs/swagger2-latest-console.json b/app/config/specs/swagger2-latest-console.json index 0ef937050c..d5c82bba49 100644 --- a/app/config/specs/swagger2-latest-console.json +++ b/app/config/specs/swagger2-latest-console.json @@ -37212,7 +37212,8 @@ "resourceId": { "type": "string", "description": "Resource ID.", - "x-example": "5e5ea5c16897e" + "x-example": "5e5ea5c16897e", + "x-nullable": true }, "name": { "type": "string", @@ -37224,10 +37225,16 @@ "description": "The value of this metric at the timestamp.", "x-example": 1, "format": "int32" + }, + "estimate": { + "type": "number", + "description": "The estimated value of this metric at the end of the period.", + "x-example": 1, + "format": "double", + "x-nullable": true } }, "required": [ - "resourceId", "name", "value" ] @@ -38087,6 +38094,27 @@ "$ref": "#\/definitions\/metricBreakdown" }, "x-example": [] + }, + "authPhoneTotal": { + "type": "integer", + "description": "Total aggregated number of phone auth.", + "x-example": 0, + "format": "int32" + }, + "authPhoneEstimate": { + "type": "number", + "description": "Estimated total aggregated cost of phone auth.", + "x-example": 0, + "format": "double" + }, + "authPhoneCountryBreakdown": { + "type": "array", + "description": "Aggregated breakdown in totals of phone auth by country.", + "items": { + "type": "object", + "$ref": "#\/definitions\/metricBreakdown" + }, + "x-example": [] } }, "required": [ @@ -38111,7 +38139,10 @@ "databasesStorageBreakdown", "executionsMbSecondsBreakdown", "buildsMbSecondsBreakdown", - "functionsStorageBreakdown" + "functionsStorageBreakdown", + "authPhoneTotal", + "authPhoneEstimate", + "authPhoneCountryBreakdown" ] }, "headers": { From cc7cef59b344e094c047a95131a06910d6b7c917 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Thu, 9 Jan 2025 19:03:25 +0000 Subject: [PATCH 60/74] isset over empty --- app/controllers/api/project.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/project.php b/app/controllers/api/project.php index d12e35d9d7..7e704762b8 100644 --- a/app/controllers/api/project.php +++ b/app/controllers/api/project.php @@ -285,14 +285,14 @@ App::get('/v1/project/usage') $value = $metric->getAttribute('value', 0); - if (!empty($smsRates[$countryCode])) { + if (isset($smsRates[$countryCode])) { $authPhoneEstimate += $value * $smsRates[$countryCode]; } $authPhoneCountryBreakdown[] = [ 'name' => $countryCode, 'value' => $value, - 'estimate' => !empty($smsRates[$countryCode]) + 'estimate' => isset($smsRates[$countryCode]) ? $value * $smsRates[$countryCode] : 0.0, ]; From a85abdab4af4735f2215e1d55eed2cb0cc7494d6 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Thu, 9 Jan 2025 20:11:02 +0000 Subject: [PATCH 61/74] fix: usage tests --- tests/e2e/General/UsageTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index c50a15fd3f..7263a2eee1 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -143,7 +143,7 @@ class UsageTest extends Scope ); $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals(22, count($response['body'])); + $this->assertEquals(25, count($response['body'])); $this->validateDates($response['body']['network']); $this->validateDates($response['body']['requests']); $this->validateDates($response['body']['users']); From ec7f76f9fb640421dcfc6becdd0672e66ae531e3 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Thu, 9 Jan 2025 21:17:53 +0000 Subject: [PATCH 62/74] fix: stats tests --- tests/e2e/General/UsageTest.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/e2e/General/UsageTest.php b/tests/e2e/General/UsageTest.php index 7263a2eee1..f6b2ca4882 100644 --- a/tests/e2e/General/UsageTest.php +++ b/tests/e2e/General/UsageTest.php @@ -324,7 +324,7 @@ class UsageTest extends Scope ] ); - $this->assertEquals(22, count($response['body'])); + $this->assertEquals(25, count($response['body'])); $this->assertEquals(1, count($response['body']['requests'])); $this->assertEquals($requestsTotal, $response['body']['requests'][array_key_last($response['body']['requests'])]['value']); $this->validateDates($response['body']['requests']); @@ -545,7 +545,7 @@ class UsageTest extends Scope ] ); - $this->assertEquals(22, count($response['body'])); + $this->assertEquals(25, count($response['body'])); $this->assertEquals(1, count($response['body']['requests'])); $this->assertEquals(1, count($response['body']['network'])); $this->assertEquals($requestsTotal, $response['body']['requests'][array_key_last($response['body']['requests'])]['value']); From d86984f6cf5ab936ad26075a4c7e7fa43745d83e Mon Sep 17 00:00:00 2001 From: ChiragAgg5k Date: Fri, 10 Jan 2025 08:42:10 +0530 Subject: [PATCH 63/74] chore: used config in auth --- app/controllers/api/account.php | 4 ++-- app/controllers/shared/api/auth.php | 17 +++++++++-------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index ff96acfc57..6935029450 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -278,7 +278,7 @@ App::post('/v1/account') ->desc('Create account') ->groups(['api', 'account', 'auth']) ->label('scope', 'sessions.write') - ->label('auth.type', 'emailPassword') + ->label('auth.type', 'email-password') ->label('audits.event', 'user.create') ->label('audits.resource', 'user/{response.$id}') ->label('audits.userId', '{response.$id}') @@ -799,7 +799,7 @@ App::post('/v1/account/sessions/email') ->groups(['api', 'account', 'auth', 'session']) ->label('event', 'users.[userId].sessions.[sessionId].create') ->label('scope', 'sessions.write') - ->label('auth.type', 'emailPassword') + ->label('auth.type', 'email-password') ->label('audits.event', 'session.create') ->label('audits.resource', 'user/{response.userId}') ->label('audits.userId', '{response.userId}') diff --git a/app/controllers/shared/api/auth.php b/app/controllers/shared/api/auth.php index 53aacabe21..ecabc641ec 100644 --- a/app/controllers/shared/api/auth.php +++ b/app/controllers/shared/api/auth.php @@ -5,6 +5,7 @@ use Appwrite\Extend\Exception; use Appwrite\Utopia\Request; use MaxMind\Db\Reader; use Utopia\App; +use Utopia\Config\Config; use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Validator\Authorization; @@ -57,44 +58,44 @@ App::init() $auths = $project->getAttribute('auths', []); switch ($route->getLabel('auth.type', '')) { - case 'emailPassword': - if (($auths['emailPassword'] ?? true) === false) { + case 'email-password': + if (($auths[Config::getParam('auth')['email-password']['key']] ?? true) === false) { throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Email / Password authentication is disabled for this project'); } break; case 'magic-url': - if (($auths['usersAuthMagicURL'] ?? true) === false) { + if (($auths[Config::getParam('auth')['magic-url']['key']] ?? true) === false) { throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Magic URL authentication is disabled for this project'); } break; case 'anonymous': - if (($auths['anonymous'] ?? true) === false) { + if (($auths[Config::getParam('auth')['anonymous']['key']] ?? true) === false) { throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Anonymous authentication is disabled for this project'); } break; case 'phone': - if (($auths['phone'] ?? true) === false) { + if (($auths[Config::getParam('auth')['phone']['key']] ?? true) === false) { throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Phone authentication is disabled for this project'); } break; case 'invites': - if (($auths['invites'] ?? true) === false) { + if (($auths[Config::getParam('auth')['invites']['key']] ?? true) === false) { throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Invites authentication is disabled for this project'); } break; case 'jwt': - if (($auths['JWT'] ?? true) === false) { + if (($auths[Config::getParam('auth')['jwt']['key']] ?? true) === false) { throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'JWT authentication is disabled for this project'); } break; case 'email-otp': - if (($auths['emailOTP'] ?? true) === false) { + if (($auths[Config::getParam('auth')['email-otp']['key']] ?? true) === false) { throw new Exception(Exception::USER_AUTH_METHOD_UNSUPPORTED, 'Email OTP authentication is disabled for this project'); } break; From b88d9c51fb374c1cc4faa35f178cda3dcd27fd20 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 10 Jan 2025 20:19:34 +1300 Subject: [PATCH 64/74] Revert "Fix dead connections" --- app/controllers/general.php | 14 +---------- app/http.php | 11 -------- app/init.php | 31 ++++++----------------- app/worker.php | 50 ++++++++++++++----------------------- composer.json | 2 +- composer.lock | 38 ++++++++++++++-------------- 6 files changed, 48 insertions(+), 98 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 90cb3293e6..db9df71341 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -21,7 +21,6 @@ use Appwrite\Utopia\Response\Filters\V18 as ResponseV18; use Appwrite\Utopia\View; use Executor\Executor; use MaxMind\Db\Reader; -use Swoole\Database\DetectsLostConnections; use Swoole\Http\Request as SwooleRequest; use Utopia\App; use Utopia\CLI\Console; @@ -29,7 +28,6 @@ use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; -use Utopia\Database\Exception as DatabaseException; use Utopia\Database\Helpers\ID; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; @@ -40,7 +38,6 @@ use Utopia\Logger\Adapter\Sentry; use Utopia\Logger\Log; use Utopia\Logger\Log\User; use Utopia\Logger\Logger; -use Utopia\Pools\Connection; use Utopia\System\System; use Utopia\Validator\Hostname; use Utopia\Validator\Text; @@ -749,16 +746,7 @@ App::error() ->inject('logger') ->inject('log') ->inject('queueForUsage') - ->inject('connectionForProject') - ->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, Log $log, Usage $queueForUsage, Connection $connectionForProject) { - if ( - ($error instanceof PDOException || $error instanceof DatabaseException) - && DetectsLostConnections::causedByLostConnection($error) - ) { - // Mark connection as unhealthy so it will be recycled on next reclaim. - $connectionForProject->setHealthy(false); - } - + ->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, Log $log, Usage $queueForUsage) { $version = System::getEnv('_APP_VERSION', 'UNKNOWN'); $route = $utopia->getRoute(); $class = \get_class($error); diff --git a/app/http.php b/app/http.php index b988c7d8ac..3a7562ffd1 100644 --- a/app/http.php +++ b/app/http.php @@ -5,7 +5,6 @@ require_once __DIR__ . '/../vendor/autoload.php'; use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; use Swoole\Constant; -use Swoole\Database\DetectsLostConnections; use Swoole\Http\Request as SwooleRequest; use Swoole\Http\Response as SwooleResponse; use Swoole\Http\Server; @@ -18,7 +17,6 @@ use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; -use Utopia\Database\Exception as DatabaseException; use Utopia\Database\Exception\Duplicate; use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Permission; @@ -348,15 +346,6 @@ $http->on(Constant::EVENT_REQUEST, function (SwooleRequest $swooleRequest, Swool $app->run($request, $response); } catch (\Throwable $th) { - if ( - ($th instanceof PDOException || $th instanceof DatabaseException) - && DetectsLostConnections::causedByLostConnection($th) - ) { - // Mark connection as unhealthy so it will be recycled on next reclaim. - $connectionForProject = $app->getResource('connectionForProject'); - $connectionForProject->setHealthy(false); - } - $version = System::getEnv('_APP_VERSION', 'UNKNOWN'); $logger = $app->getResource("logger"); diff --git a/app/init.php b/app/init.php index 51d9c38239..b4e60d7924 100644 --- a/app/init.php +++ b/app/init.php @@ -74,7 +74,6 @@ use Utopia\Logger\Adapter\Raygun; use Utopia\Logger\Adapter\Sentry; use Utopia\Logger\Log; use Utopia\Logger\Logger; -use Utopia\Pools\Connection as PoolConnection; use Utopia\Pools\Group; use Utopia\Pools\Pool; use Utopia\Queue; @@ -1413,26 +1412,7 @@ App::setResource('console', function () { ]); }, []); -App::setResource('connectionForProject', function (Group $pools, Document $project) { - if ($project->isEmpty() || $project->getId() === 'console') { - return $pools - ->get('console') - ->pop(); - } - - try { - $dsn = new DSN($project->getAttribute('database')); - } catch (\InvalidArgumentException) { - // TODO: Temporary until all projects are using shared tables - $dsn = new DSN('mysql://' . $project->getAttribute('database')); - } - - return $pools - ->get($dsn->getHost()) - ->pop(); -}, ['pools', 'project']); - -App::setResource('dbForProject', function (Group $pools, PoolConnection $connectionForProject, Database $dbForPlatform, Cache $cache, Document $project) { +App::setResource('dbForProject', function (Group $pools, Database $dbForPlatform, Cache $cache, Document $project) { if ($project->isEmpty() || $project->getId() === 'console') { return $dbForPlatform; } @@ -1444,7 +1424,12 @@ App::setResource('dbForProject', function (Group $pools, PoolConnection $connect $dsn = new DSN('mysql://' . $project->getAttribute('database')); } - $database = new Database($connectionForProject->getResource(), $cache); + $dbAdapter = $pools + ->get($dsn->getHost()) + ->pop() + ->getResource(); + + $database = new Database($dbAdapter, $cache); $database ->setMetadata('host', \gethostname()) @@ -1467,7 +1452,7 @@ App::setResource('dbForProject', function (Group $pools, PoolConnection $connect } return $database; -}, ['pools', 'connectionForProject', 'dbForPlatform', 'cache', 'project']); +}, ['pools', 'dbForPlatform', 'cache', 'project']); App::setResource('dbForPlatform', function (Group $pools, Cache $cache) { $dbAdapter = $pools diff --git a/app/worker.php b/app/worker.php index 62882b32b1..6eb1363e9b 100644 --- a/app/worker.php +++ b/app/worker.php @@ -16,7 +16,6 @@ use Appwrite\Event\Migration; use Appwrite\Event\Usage; use Appwrite\Event\UsageDump; use Appwrite\Platform\Appwrite; -use Swoole\Database\DetectsLostConnections; use Swoole\Runtime; use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis; use Utopia\Cache\Adapter\Sharding; @@ -26,13 +25,11 @@ use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; -use Utopia\Database\Exception as DatabaseException; use Utopia\Database\Validator\Authorization; use Utopia\DSN\DSN; use Utopia\Logger\Log; use Utopia\Logger\Logger; use Utopia\Platform\Service; -use Utopia\Pools\Connection as PoolConnection; use Utopia\Pools\Group; use Utopia\Queue\Connection; use Utopia\Queue\Message; @@ -69,13 +66,13 @@ Server::setResource('project', function (Message $message, Database $dbForPlatfo return $dbForPlatform->getDocument('projects', $project->getId()); }, ['message', 'dbForPlatform']); -Server::setResource('connectionForProject', function (Group $pools, Document $project) { +Server::setResource('dbForProject', function (Cache $cache, Registry $register, Message $message, Document $project, Database $dbForPlatform) { if ($project->isEmpty() || $project->getId() === 'console') { - return $pools - ->get('console') - ->pop(); + return $dbForPlatform; } + $pools = $register->get('pools'); + try { $dsn = new DSN($project->getAttribute('database')); } catch (\InvalidArgumentException) { @@ -83,17 +80,12 @@ Server::setResource('connectionForProject', function (Group $pools, Document $pr $dsn = new DSN('mysql://' . $project->getAttribute('database')); } - return $pools + $adapter = $pools ->get($dsn->getHost()) - ->pop(); -}, ['pools', 'project']); + ->pop() + ->getResource(); -Server::setResource('dbForProject', function (PoolConnection $connectionForProject, Cache $cache, Registry $register, Message $message, Document $project, Database $dbForPlatform) { - if ($project->isEmpty() || $project->getId() === 'console') { - return $dbForPlatform; - } - - $database = new Database($connectionForProject->getResource(), $cache); + $database = new Database($adapter, $cache); try { $dsn = new DSN($project->getAttribute('database')); @@ -117,12 +109,12 @@ Server::setResource('dbForProject', function (PoolConnection $connectionForProje } return $database; -}, ['connectionForProject', 'cache', 'register', 'message', 'project', 'dbForPlatform']); +}, ['cache', 'register', 'message', 'project', 'dbForPlatform']); -Server::setResource('getProjectDB', function (Group $pools, PoolConnection $connectionForProject, Database $dbForPlatform, $cache) { +Server::setResource('getProjectDB', function (Group $pools, Database $dbForPlatform, $cache) { $databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools - return function (Document $project) use ($pools, $connectionForProject, $dbForPlatform, $cache, &$databases): Database { + return function (Document $project) use ($pools, $dbForPlatform, $cache, &$databases): Database { if ($project->isEmpty() || $project->getId() === 'console') { return $dbForPlatform; } @@ -154,7 +146,12 @@ Server::setResource('getProjectDB', function (Group $pools, PoolConnection $conn return $database; } - $database = new Database($connectionForProject->getResource(), $cache); + $dbAdapter = $pools + ->get($dsn->getHost()) + ->pop() + ->getResource(); + + $database = new Database($dbAdapter, $cache); $databases[$dsn->getHost()] = $database; @@ -174,7 +171,7 @@ Server::setResource('getProjectDB', function (Group $pools, PoolConnection $conn return $database; }; -}, ['pools', 'connectionForProject', 'dbForPlatform', 'cache']); +}, ['pools', 'dbForPlatform', 'cache']); Server::setResource('abuseRetention', function () { return time() - (int) System::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', 86400); @@ -413,16 +410,7 @@ $worker ->inject('log') ->inject('pools') ->inject('project') - ->inject('connectionForProject') - ->action(function (Throwable $error, ?Logger $logger, Log $log, Group $pools, Document $project, PoolConnection $connectionForProject) use ($queueName) { - if ( - ($error instanceof PDOException || $error instanceof DatabaseException) - && DetectsLostConnections::causedByLostConnection($error) - ) { - // Mark connection as unhealthy, it will be recycled on next reclaim. - $connectionForProject->setHealthy(false); - } - + ->action(function (Throwable $error, ?Logger $logger, Log $log, Group $pools, Document $project) use ($queueName) { $pools->reclaim(); $version = System::getEnv('_APP_VERSION', 'UNKNOWN'); diff --git a/composer.json b/composer.json index 2612403c1a..e44dd0bffc 100644 --- a/composer.json +++ b/composer.json @@ -63,7 +63,7 @@ "utopia-php/migration": "0.6.*", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.7.*", - "utopia-php/pools": "0.6.*", + "utopia-php/pools": "0.5.*", "utopia-php/preloader": "0.2.*", "utopia-php/queue": "0.7.*", "utopia-php/registry": "0.5.*", diff --git a/composer.lock b/composer.lock index 045ef31ffa..b906af3a81 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "7b5b5926b452186543903eb539f59c2d", + "content-hash": "6b136b5490c0d5d331eac0d70bb3e198", "packages": [ { "name": "adhocore/jwt", @@ -3929,16 +3929,16 @@ }, { "name": "utopia-php/migration", - "version": "0.6.14", + "version": "0.6.13", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "59a19f09ded0ccab4c8cca35b1242c01e2b9cfd2" + "reference": "68d9b0a9477755afcda607e7e8109785cae17a13" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/59a19f09ded0ccab4c8cca35b1242c01e2b9cfd2", - "reference": "59a19f09ded0ccab4c8cca35b1242c01e2b9cfd2", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/68d9b0a9477755afcda607e7e8109785cae17a13", + "reference": "68d9b0a9477755afcda607e7e8109785cae17a13", "shasum": "" }, "require": { @@ -3979,9 +3979,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/0.6.14" + "source": "https://github.com/utopia-php/migration/tree/0.6.13" }, - "time": "2025-01-08T01:07:25+00:00" + "time": "2024-11-26T13:57:53+00:00" }, { "name": "utopia-php/mongo", @@ -4145,25 +4145,25 @@ }, { "name": "utopia-php/pools", - "version": "0.6.0", + "version": "0.5.0", "source": { "type": "git", "url": "https://github.com/utopia-php/pools.git", - "reference": "59414ab7b57728edfde6d5ccc5a2583b7967ac18" + "reference": "6f716a213a08db95eda1b5dddfa90983c1834817" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/pools/zipball/59414ab7b57728edfde6d5ccc5a2583b7967ac18", - "reference": "59414ab7b57728edfde6d5ccc5a2583b7967ac18", + "url": "https://api.github.com/repos/utopia-php/pools/zipball/6f716a213a08db95eda1b5dddfa90983c1834817", + "reference": "6f716a213a08db95eda1b5dddfa90983c1834817", "shasum": "" }, "require": { - "php": ">=8.3" + "php": ">=8.0" }, "require-dev": { - "laravel/pint": "1.*", - "phpstan/phpstan": "1.*", - "phpunit/phpunit": "11.*" + "laravel/pint": "1.2.*", + "phpstan/phpstan": "1.8.*", + "phpunit/phpunit": "^9.3" }, "type": "library", "autoload": { @@ -4190,9 +4190,9 @@ ], "support": { "issues": "https://github.com/utopia-php/pools/issues", - "source": "https://github.com/utopia-php/pools/tree/0.6.0" + "source": "https://github.com/utopia-php/pools/tree/0.5.0" }, - "time": "2025-01-08T07:58:42+00:00" + "time": "2024-04-19T11:11:54+00:00" }, { "name": "utopia-php/preloader", @@ -8556,7 +8556,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": {}, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { @@ -8580,5 +8580,5 @@ "platform-overrides": { "php": "8.3" }, - "plugin-api-version": "2.6.0" + "plugin-api-version": "2.2.0" } From b76260ecce1f8e9f14c9f07a037052c9cfdf34fc Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 10 Jan 2025 20:28:50 +1300 Subject: [PATCH 65/74] Update database --- composer.json | 4 +-- composer.lock | 84 +++++++++++++++++++++++++-------------------------- 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/composer.json b/composer.json index 2612403c1a..90edca6485 100644 --- a/composer.json +++ b/composer.json @@ -45,13 +45,13 @@ "ext-sockets": "*", "appwrite/php-runtimes": "0.16.*", "appwrite/php-clamav": "2.0.*", - "utopia-php/abuse": "0.46.*", + "utopia-php/abuse": "0.43.*", "utopia-php/analytics": "0.10.*", "utopia-php/audit": "0.43.*", "utopia-php/cache": "0.11.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "0.53.200", + "utopia-php/database": "0.53.31", "utopia-php/domains": "0.5.*", "utopia-php/dsn": "0.2.1", "utopia-php/framework": "0.33.*", diff --git a/composer.lock b/composer.lock index 045ef31ffa..b8e02a8b65 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "7b5b5926b452186543903eb539f59c2d", + "content-hash": "0cd8b051f3b6e11f9c537ccfe6b166d1", "packages": [ { "name": "adhocore/jwt", @@ -709,16 +709,16 @@ }, { "name": "google/protobuf", - "version": "v4.29.2", + "version": "v4.29.3", "source": { "type": "git", "url": "https://github.com/protocolbuffers/protobuf-php.git", - "reference": "79aa5014efeeec3d137df5cdb0ae2fc163953945" + "reference": "ab5077c2cfdd1f415f42d11fdbdf903ba8e3d9b7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/79aa5014efeeec3d137df5cdb0ae2fc163953945", - "reference": "79aa5014efeeec3d137df5cdb0ae2fc163953945", + "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/ab5077c2cfdd1f415f42d11fdbdf903ba8e3d9b7", + "reference": "ab5077c2cfdd1f415f42d11fdbdf903ba8e3d9b7", "shasum": "" }, "require": { @@ -747,9 +747,9 @@ "proto" ], "support": { - "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.29.2" + "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.29.3" }, - "time": "2024-12-18T14:11:12+00:00" + "time": "2025-01-08T21:00:13+00:00" }, { "name": "jean85/pretty-package-versions", @@ -1237,16 +1237,16 @@ }, { "name": "open-telemetry/api", - "version": "1.1.2", + "version": "1.2.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/api.git", - "reference": "04c85a1e41a3d59fa9bdc801a5de1df6624b95ed" + "reference": "351a30baa79699de3de3a814c8ccc7b52ccdfb1d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/04c85a1e41a3d59fa9bdc801a5de1df6624b95ed", - "reference": "04c85a1e41a3d59fa9bdc801a5de1df6624b95ed", + "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/351a30baa79699de3de3a814c8ccc7b52ccdfb1d", + "reference": "351a30baa79699de3de3a814c8ccc7b52ccdfb1d", "shasum": "" }, "require": { @@ -1303,7 +1303,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2024-11-16T04:32:30+00:00" + "time": "2025-01-08T23:50:34+00:00" }, { "name": "open-telemetry/context", @@ -1366,16 +1366,16 @@ }, { "name": "open-telemetry/exporter-otlp", - "version": "1.1.0", + "version": "1.2.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/exporter-otlp.git", - "reference": "9b6de12204f25f8ab9540b46d6e7b5151897ce18" + "reference": "243d9657c44a06f740cf384f486afe954c2b725f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/exporter-otlp/zipball/9b6de12204f25f8ab9540b46d6e7b5151897ce18", - "reference": "9b6de12204f25f8ab9540b46d6e7b5151897ce18", + "url": "https://api.github.com/repos/opentelemetry-php/exporter-otlp/zipball/243d9657c44a06f740cf384f486afe954c2b725f", + "reference": "243d9657c44a06f740cf384f486afe954c2b725f", "shasum": "" }, "require": { @@ -1426,7 +1426,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2024-04-30T18:28:30+00:00" + "time": "2025-01-08T23:50:03+00:00" }, { "name": "open-telemetry/gen-otlp-protobuf", @@ -1493,16 +1493,16 @@ }, { "name": "open-telemetry/sdk", - "version": "1.1.2", + "version": "1.2.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/sdk.git", - "reference": "fb0ff8d8279a3776bd604791e2531dd0cc147e8b" + "reference": "9a1c3b866239dbff291e5cc555bb7793eab08127" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/fb0ff8d8279a3776bd604791e2531dd0cc147e8b", - "reference": "fb0ff8d8279a3776bd604791e2531dd0cc147e8b", + "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/9a1c3b866239dbff291e5cc555bb7793eab08127", + "reference": "9a1c3b866239dbff291e5cc555bb7793eab08127", "shasum": "" }, "require": { @@ -1579,7 +1579,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2024-10-18T21:01:35+00:00" + "time": "2025-01-08T23:50:34+00:00" }, { "name": "open-telemetry/sem-conv", @@ -3136,16 +3136,16 @@ }, { "name": "utopia-php/abuse", - "version": "0.46.0", + "version": "0.43.3", "source": { "type": "git", "url": "https://github.com/utopia-php/abuse.git", - "reference": "bef426a9a28b3e71b08216bcbe310455f588c11b" + "reference": "c8a403bd79694c9790e3023e0a84c6e350ae0f2a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/abuse/zipball/bef426a9a28b3e71b08216bcbe310455f588c11b", - "reference": "bef426a9a28b3e71b08216bcbe310455f588c11b", + "url": "https://api.github.com/repos/utopia-php/abuse/zipball/c8a403bd79694c9790e3023e0a84c6e350ae0f2a", + "reference": "c8a403bd79694c9790e3023e0a84c6e350ae0f2a", "shasum": "" }, "require": { @@ -3153,7 +3153,7 @@ "ext-pdo": "*", "ext-redis": "*", "php": ">=8.0", - "utopia-php/database": "0.53.200" + "utopia-php/database": "0.53.31" }, "require-dev": { "laravel/pint": "1.5.*", @@ -3181,9 +3181,9 @@ ], "support": { "issues": "https://github.com/utopia-php/abuse/issues", - "source": "https://github.com/utopia-php/abuse/tree/0.46.0" + "source": "https://github.com/utopia-php/abuse/tree/0.43.3" }, - "time": "2024-12-27T17:46:08+00:00" + "time": "2025-01-10T07:26:06+00:00" }, { "name": "utopia-php/analytics", @@ -3233,21 +3233,21 @@ }, { "name": "utopia-php/audit", - "version": "0.43.1", + "version": "0.43.3", "source": { "type": "git", "url": "https://github.com/utopia-php/audit.git", - "reference": "04a47dd1f5f92e2d50e971a06bcc9e759325d277" + "reference": "1fef1d5163e114d3bd357cf4bc0cb8d0482faf53" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/audit/zipball/04a47dd1f5f92e2d50e971a06bcc9e759325d277", - "reference": "04a47dd1f5f92e2d50e971a06bcc9e759325d277", + "url": "https://api.github.com/repos/utopia-php/audit/zipball/1fef1d5163e114d3bd357cf4bc0cb8d0482faf53", + "reference": "1fef1d5163e114d3bd357cf4bc0cb8d0482faf53", "shasum": "" }, "require": { "php": ">=8.0", - "utopia-php/database": "0.53.*" + "utopia-php/database": "0.53.31" }, "require-dev": { "laravel/pint": "1.5.*", @@ -3274,9 +3274,9 @@ ], "support": { "issues": "https://github.com/utopia-php/audit/issues", - "source": "https://github.com/utopia-php/audit/tree/0.43.1" + "source": "https://github.com/utopia-php/audit/tree/0.43.3" }, - "time": "2024-10-23T04:27:59+00:00" + "time": "2025-01-10T07:23:31+00:00" }, { "name": "utopia-php/cache", @@ -3476,16 +3476,16 @@ }, { "name": "utopia-php/database", - "version": "0.53.200", + "version": "0.53.31", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "570c63a3760d0e1404679ddfacd9484af40bd9fc" + "reference": "874f2fa096c85a224fa1cdabea74760a7d7aac0c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/570c63a3760d0e1404679ddfacd9484af40bd9fc", - "reference": "570c63a3760d0e1404679ddfacd9484af40bd9fc", + "url": "https://api.github.com/repos/utopia-php/database/zipball/874f2fa096c85a224fa1cdabea74760a7d7aac0c", + "reference": "874f2fa096c85a224fa1cdabea74760a7d7aac0c", "shasum": "" }, "require": { @@ -3526,9 +3526,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.53.200" + "source": "https://github.com/utopia-php/database/tree/0.53.31" }, - "time": "2024-12-01T07:59:15+00:00" + "time": "2025-01-10T06:51:16+00:00" }, { "name": "utopia-php/domains", From a45e0a59dd968ea566555e97873d539c2ebbf338 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 10 Jan 2025 22:05:40 +1300 Subject: [PATCH 66/74] Update database --- app/init.php | 2 +- app/realtime.php | 2 +- app/worker.php | 2 +- composer.json | 2 +- composer.lock | 42 +++++++++++++++++++++--------------------- 5 files changed, 25 insertions(+), 25 deletions(-) diff --git a/app/init.php b/app/init.php index 51d9c38239..1da4771d1d 100644 --- a/app/init.php +++ b/app/init.php @@ -48,7 +48,7 @@ use Appwrite\Utopia\Request; use MaxMind\Db\Reader; use PHPMailer\PHPMailer\PHPMailer; use Swoole\Database\PDOProxy; -use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis; +use Utopia\Abuse\Adapters\Redis\TimeLimit as TimeLimitRedis; use Utopia\App; use Utopia\Cache\Adapter\Redis as RedisCache; use Utopia\Cache\Adapter\Sharding; diff --git a/app/realtime.php b/app/realtime.php index 86f9c85fdd..3c543ee166 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -13,7 +13,7 @@ use Swoole\Runtime; use Swoole\Table; use Swoole\Timer; use Utopia\Abuse\Abuse; -use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis; +use Utopia\Abuse\Adapters\Redis\TimeLimit as TimeLimitRedis; use Utopia\App; use Utopia\Cache\Adapter\Sharding; use Utopia\Cache\Cache; diff --git a/app/worker.php b/app/worker.php index 62882b32b1..50249ed605 100644 --- a/app/worker.php +++ b/app/worker.php @@ -18,7 +18,7 @@ use Appwrite\Event\UsageDump; use Appwrite\Platform\Appwrite; use Swoole\Database\DetectsLostConnections; use Swoole\Runtime; -use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis; +use Utopia\Abuse\Adapters\Redis\TimeLimit as TimeLimitRedis; use Utopia\Cache\Adapter\Sharding; use Utopia\Cache\Cache; use Utopia\CLI\Console; diff --git a/composer.json b/composer.json index 90edca6485..e21c4abe84 100644 --- a/composer.json +++ b/composer.json @@ -51,7 +51,7 @@ "utopia-php/cache": "0.11.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "0.53.31", + "utopia-php/database": "0.53.32", "utopia-php/domains": "0.5.*", "utopia-php/dsn": "0.2.1", "utopia-php/framework": "0.33.*", diff --git a/composer.lock b/composer.lock index b8e02a8b65..6dec3249da 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "0cd8b051f3b6e11f9c537ccfe6b166d1", + "content-hash": "6f80de1fcc781dfbf379e6898a4f57ab", "packages": [ { "name": "adhocore/jwt", @@ -3136,16 +3136,16 @@ }, { "name": "utopia-php/abuse", - "version": "0.43.3", + "version": "0.43.4", "source": { "type": "git", "url": "https://github.com/utopia-php/abuse.git", - "reference": "c8a403bd79694c9790e3023e0a84c6e350ae0f2a" + "reference": "2f5d9dcf96f41a1a1e1a2ede00b49531ac8cacd9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/abuse/zipball/c8a403bd79694c9790e3023e0a84c6e350ae0f2a", - "reference": "c8a403bd79694c9790e3023e0a84c6e350ae0f2a", + "url": "https://api.github.com/repos/utopia-php/abuse/zipball/2f5d9dcf96f41a1a1e1a2ede00b49531ac8cacd9", + "reference": "2f5d9dcf96f41a1a1e1a2ede00b49531ac8cacd9", "shasum": "" }, "require": { @@ -3153,7 +3153,7 @@ "ext-pdo": "*", "ext-redis": "*", "php": ">=8.0", - "utopia-php/database": "0.53.31" + "utopia-php/database": "0.53.32" }, "require-dev": { "laravel/pint": "1.5.*", @@ -3181,9 +3181,9 @@ ], "support": { "issues": "https://github.com/utopia-php/abuse/issues", - "source": "https://github.com/utopia-php/abuse/tree/0.43.3" + "source": "https://github.com/utopia-php/abuse/tree/0.43.4" }, - "time": "2025-01-10T07:26:06+00:00" + "time": "2025-01-10T08:58:29+00:00" }, { "name": "utopia-php/analytics", @@ -3233,21 +3233,21 @@ }, { "name": "utopia-php/audit", - "version": "0.43.3", + "version": "0.43.4", "source": { "type": "git", "url": "https://github.com/utopia-php/audit.git", - "reference": "1fef1d5163e114d3bd357cf4bc0cb8d0482faf53" + "reference": "d9fe89b67c41215e94ae7bf46cd7e3dd73867dd7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/audit/zipball/1fef1d5163e114d3bd357cf4bc0cb8d0482faf53", - "reference": "1fef1d5163e114d3bd357cf4bc0cb8d0482faf53", + "url": "https://api.github.com/repos/utopia-php/audit/zipball/d9fe89b67c41215e94ae7bf46cd7e3dd73867dd7", + "reference": "d9fe89b67c41215e94ae7bf46cd7e3dd73867dd7", "shasum": "" }, "require": { "php": ">=8.0", - "utopia-php/database": "0.53.31" + "utopia-php/database": "0.53.32" }, "require-dev": { "laravel/pint": "1.5.*", @@ -3274,9 +3274,9 @@ ], "support": { "issues": "https://github.com/utopia-php/audit/issues", - "source": "https://github.com/utopia-php/audit/tree/0.43.3" + "source": "https://github.com/utopia-php/audit/tree/0.43.4" }, - "time": "2025-01-10T07:23:31+00:00" + "time": "2025-01-10T08:57:54+00:00" }, { "name": "utopia-php/cache", @@ -3476,16 +3476,16 @@ }, { "name": "utopia-php/database", - "version": "0.53.31", + "version": "0.53.32", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "874f2fa096c85a224fa1cdabea74760a7d7aac0c" + "reference": "981a1241139b42dccd531511130b79137740b205" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/874f2fa096c85a224fa1cdabea74760a7d7aac0c", - "reference": "874f2fa096c85a224fa1cdabea74760a7d7aac0c", + "url": "https://api.github.com/repos/utopia-php/database/zipball/981a1241139b42dccd531511130b79137740b205", + "reference": "981a1241139b42dccd531511130b79137740b205", "shasum": "" }, "require": { @@ -3526,9 +3526,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.53.31" + "source": "https://github.com/utopia-php/database/tree/0.53.32" }, - "time": "2025-01-10T06:51:16+00:00" + "time": "2025-01-10T08:53:47+00:00" }, { "name": "utopia-php/domains", From 428efdede02d39cf1313ec4892246fb46d0521d9 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 10 Jan 2025 22:32:25 +1300 Subject: [PATCH 67/74] Update lock --- composer.lock | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/composer.lock b/composer.lock index fe3331bd4a..13c76ab59a 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "6f80de1fcc781dfbf379e6898a4f57ab", + "content-hash": "630e10509c6effad82583cd67ad49974", "packages": [ { "name": "adhocore/jwt", @@ -3929,16 +3929,16 @@ }, { "name": "utopia-php/migration", - "version": "0.6.13", + "version": "0.6.14", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "68d9b0a9477755afcda607e7e8109785cae17a13" + "reference": "59a19f09ded0ccab4c8cca35b1242c01e2b9cfd2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/68d9b0a9477755afcda607e7e8109785cae17a13", - "reference": "68d9b0a9477755afcda607e7e8109785cae17a13", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/59a19f09ded0ccab4c8cca35b1242c01e2b9cfd2", + "reference": "59a19f09ded0ccab4c8cca35b1242c01e2b9cfd2", "shasum": "" }, "require": { @@ -3979,9 +3979,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/0.6.13" + "source": "https://github.com/utopia-php/migration/tree/0.6.14" }, - "time": "2024-11-26T13:57:53+00:00" + "time": "2025-01-08T01:07:25+00:00" }, { "name": "utopia-php/mongo", @@ -8556,7 +8556,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": {}, "prefer-stable": false, "prefer-lowest": false, "platform": { @@ -8580,5 +8580,5 @@ "platform-overrides": { "php": "8.3" }, - "plugin-api-version": "2.2.0" + "plugin-api-version": "2.6.0" } From 1695e90399531634bd95802fea5a019a76d03f7b Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Fri, 10 Jan 2025 17:33:08 +0000 Subject: [PATCH 68/74] fix: metric --- app/controllers/api/project.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/project.php b/app/controllers/api/project.php index 7e704762b8..5a1bf063f2 100644 --- a/app/controllers/api/project.php +++ b/app/controllers/api/project.php @@ -268,7 +268,7 @@ App::get('/v1/project/usage') // This estimate is only for paid SMS usage $authPhoneMetrics = Authorization::skip(fn () => $dbForProject->find('stats', [ - Query::startsWith('metric', METRIC_AUTH_METHOD_PHONE_COUNTRY_CODE . '.'), + Query::startsWith('metric', METRIC_AUTH_METHOD_PHONE . '.'), Query::equal('period', ['1d']), Query::greaterThanEqual('time', $firstDay), Query::lessThan('time', $lastDay), From f48c843bea10e77fd5704b47b85d4488f1b12b27 Mon Sep 17 00:00:00 2001 From: Steven Nguyen <1477010+stnguyen90@users.noreply.github.com> Date: Sat, 11 Jan 2025 17:56:36 +0000 Subject: [PATCH 69/74] fix(users): ensure user can delete session The session document created by users.createSession() was missing delete permissions for the user so when the user tried to delete it, they got a 401 error. This PR ensure the permissions are added just like if the document was created from the Account API so that the user has access to delete the document. --- app/controllers/api/users.php | 6 ++++++ tests/e2e/Services/Users/UsersBase.php | 8 ++++++++ 2 files changed, 14 insertions(+) diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index bdb24572eb..9fe7f433c9 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -1814,6 +1814,12 @@ App::post('/v1/users/:userId/sessions') $detector->getDevice() )); + $session->setAttribute('$permissions', [ + Permission::read(Role::user($user->getId())), + Permission::update(Role::user($user->getId())), + Permission::delete(Role::user($user->getId())), + ]); + $countryName = $locale->getText('countries.' . strtolower($session->getAttribute('countryCode')), $locale->getText('locale.country.unknown')); $session = $dbForProject->createDocument('sessions', $session); diff --git a/tests/e2e/Services/Users/UsersBase.php b/tests/e2e/Services/Users/UsersBase.php index bbf9a5e2df..04e0eb5bc3 100644 --- a/tests/e2e/Services/Users/UsersBase.php +++ b/tests/e2e/Services/Users/UsersBase.php @@ -318,6 +318,14 @@ trait UsersBase ]); $this->assertEquals(200, $response['headers']['status-code']); + + $response = $this->client->call(Client::METHOD_DELETE, '/account/sessions/current', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-session' => $session['secret'] + ]); + + $this->assertEquals(204, $response['headers']['status-code']); } From 00a5b8fbb9a952e4f3313fe34e903c434993f3b2 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 13 Jan 2025 14:58:03 +1300 Subject: [PATCH 70/74] Revert "Update database" This reverts commit a45e0a59dd968ea566555e97873d539c2ebbf338. # Conflicts: # composer.lock --- app/init.php | 2 +- app/realtime.php | 2 +- app/worker.php | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/init.php b/app/init.php index c8ee09da8d..0a241813b5 100644 --- a/app/init.php +++ b/app/init.php @@ -48,7 +48,7 @@ use Appwrite\Utopia\Request; use MaxMind\Db\Reader; use PHPMailer\PHPMailer\PHPMailer; use Swoole\Database\PDOProxy; -use Utopia\Abuse\Adapters\Redis\TimeLimit as TimeLimitRedis; +use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis; use Utopia\App; use Utopia\Cache\Adapter\Redis as RedisCache; use Utopia\Cache\Adapter\Sharding; diff --git a/app/realtime.php b/app/realtime.php index 3c543ee166..86f9c85fdd 100644 --- a/app/realtime.php +++ b/app/realtime.php @@ -13,7 +13,7 @@ use Swoole\Runtime; use Swoole\Table; use Swoole\Timer; use Utopia\Abuse\Abuse; -use Utopia\Abuse\Adapters\Redis\TimeLimit as TimeLimitRedis; +use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis; use Utopia\App; use Utopia\Cache\Adapter\Sharding; use Utopia\Cache\Cache; diff --git a/app/worker.php b/app/worker.php index b45eec6d4f..6eb1363e9b 100644 --- a/app/worker.php +++ b/app/worker.php @@ -17,7 +17,7 @@ use Appwrite\Event\Usage; use Appwrite\Event\UsageDump; use Appwrite\Platform\Appwrite; use Swoole\Runtime; -use Utopia\Abuse\Adapters\Redis\TimeLimit as TimeLimitRedis; +use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis; use Utopia\Cache\Adapter\Sharding; use Utopia\Cache\Cache; use Utopia\CLI\Console; From 10e036958ae3df7a845f91122b31f2bc5530ea60 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 13 Jan 2025 15:28:30 +1300 Subject: [PATCH 71/74] Update libraries --- composer.json | 4 ++-- composer.lock | 26 +++++++++++++------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/composer.json b/composer.json index f66968e348..882be5b94e 100644 --- a/composer.json +++ b/composer.json @@ -45,9 +45,9 @@ "ext-sockets": "*", "appwrite/php-runtimes": "0.16.*", "appwrite/php-clamav": "2.0.*", - "utopia-php/abuse": "0.43.*", + "utopia-php/abuse": "0.46.*", "utopia-php/analytics": "0.10.*", - "utopia-php/audit": "0.43.*", + "utopia-php/audit": "0.46.*", "utopia-php/cache": "0.11.*", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", diff --git a/composer.lock b/composer.lock index 13c76ab59a..b77915f0a4 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "630e10509c6effad82583cd67ad49974", + "content-hash": "c88950f1d3119d0764a469e1d804ce71", "packages": [ { "name": "adhocore/jwt", @@ -3136,16 +3136,16 @@ }, { "name": "utopia-php/abuse", - "version": "0.43.4", + "version": "0.46.2", "source": { "type": "git", "url": "https://github.com/utopia-php/abuse.git", - "reference": "2f5d9dcf96f41a1a1e1a2ede00b49531ac8cacd9" + "reference": "59749df988430b28953fb5cabfad88b0ff5b539b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/abuse/zipball/2f5d9dcf96f41a1a1e1a2ede00b49531ac8cacd9", - "reference": "2f5d9dcf96f41a1a1e1a2ede00b49531ac8cacd9", + "url": "https://api.github.com/repos/utopia-php/abuse/zipball/59749df988430b28953fb5cabfad88b0ff5b539b", + "reference": "59749df988430b28953fb5cabfad88b0ff5b539b", "shasum": "" }, "require": { @@ -3181,9 +3181,9 @@ ], "support": { "issues": "https://github.com/utopia-php/abuse/issues", - "source": "https://github.com/utopia-php/abuse/tree/0.43.4" + "source": "https://github.com/utopia-php/abuse/tree/0.46.2" }, - "time": "2025-01-10T08:58:29+00:00" + "time": "2025-01-13T02:09:43+00:00" }, { "name": "utopia-php/analytics", @@ -3233,16 +3233,16 @@ }, { "name": "utopia-php/audit", - "version": "0.43.4", + "version": "0.46.1", "source": { "type": "git", "url": "https://github.com/utopia-php/audit.git", - "reference": "d9fe89b67c41215e94ae7bf46cd7e3dd73867dd7" + "reference": "21255fa1ce66433140a43d380b2859c7f0a0bb37" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/audit/zipball/d9fe89b67c41215e94ae7bf46cd7e3dd73867dd7", - "reference": "d9fe89b67c41215e94ae7bf46cd7e3dd73867dd7", + "url": "https://api.github.com/repos/utopia-php/audit/zipball/21255fa1ce66433140a43d380b2859c7f0a0bb37", + "reference": "21255fa1ce66433140a43d380b2859c7f0a0bb37", "shasum": "" }, "require": { @@ -3274,9 +3274,9 @@ ], "support": { "issues": "https://github.com/utopia-php/audit/issues", - "source": "https://github.com/utopia-php/audit/tree/0.43.4" + "source": "https://github.com/utopia-php/audit/tree/0.46.1" }, - "time": "2025-01-10T08:57:54+00:00" + "time": "2025-01-13T02:19:56+00:00" }, { "name": "utopia-php/cache", From cde1d6e83e0c696b326cb310109b67bf3e316ae7 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 8 Jan 2025 15:00:15 +0530 Subject: [PATCH 72/74] Merge pull request #9190 from appwrite/fix-dead-connections Fix dead connections # Conflicts: # composer.lock --- app/controllers/general.php | 14 ++++++++++- app/http.php | 11 ++++++++ app/init.php | 31 +++++++++++++++++------ app/worker.php | 50 +++++++++++++++++++++++-------------- composer.json | 2 +- composer.lock | 22 ++++++++-------- 6 files changed, 90 insertions(+), 40 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index db9df71341..90cb3293e6 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -21,6 +21,7 @@ use Appwrite\Utopia\Response\Filters\V18 as ResponseV18; use Appwrite\Utopia\View; use Executor\Executor; use MaxMind\Db\Reader; +use Swoole\Database\DetectsLostConnections; use Swoole\Http\Request as SwooleRequest; use Utopia\App; use Utopia\CLI\Console; @@ -28,6 +29,7 @@ use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; +use Utopia\Database\Exception as DatabaseException; use Utopia\Database\Helpers\ID; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; @@ -38,6 +40,7 @@ use Utopia\Logger\Adapter\Sentry; use Utopia\Logger\Log; use Utopia\Logger\Log\User; use Utopia\Logger\Logger; +use Utopia\Pools\Connection; use Utopia\System\System; use Utopia\Validator\Hostname; use Utopia\Validator\Text; @@ -746,7 +749,16 @@ App::error() ->inject('logger') ->inject('log') ->inject('queueForUsage') - ->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, Log $log, Usage $queueForUsage) { + ->inject('connectionForProject') + ->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, Log $log, Usage $queueForUsage, Connection $connectionForProject) { + if ( + ($error instanceof PDOException || $error instanceof DatabaseException) + && DetectsLostConnections::causedByLostConnection($error) + ) { + // Mark connection as unhealthy so it will be recycled on next reclaim. + $connectionForProject->setHealthy(false); + } + $version = System::getEnv('_APP_VERSION', 'UNKNOWN'); $route = $utopia->getRoute(); $class = \get_class($error); diff --git a/app/http.php b/app/http.php index 3a7562ffd1..b988c7d8ac 100644 --- a/app/http.php +++ b/app/http.php @@ -5,6 +5,7 @@ require_once __DIR__ . '/../vendor/autoload.php'; use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; use Swoole\Constant; +use Swoole\Database\DetectsLostConnections; use Swoole\Http\Request as SwooleRequest; use Swoole\Http\Response as SwooleResponse; use Swoole\Http\Server; @@ -17,6 +18,7 @@ use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; +use Utopia\Database\Exception as DatabaseException; use Utopia\Database\Exception\Duplicate; use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Permission; @@ -346,6 +348,15 @@ $http->on(Constant::EVENT_REQUEST, function (SwooleRequest $swooleRequest, Swool $app->run($request, $response); } catch (\Throwable $th) { + if ( + ($th instanceof PDOException || $th instanceof DatabaseException) + && DetectsLostConnections::causedByLostConnection($th) + ) { + // Mark connection as unhealthy so it will be recycled on next reclaim. + $connectionForProject = $app->getResource('connectionForProject'); + $connectionForProject->setHealthy(false); + } + $version = System::getEnv('_APP_VERSION', 'UNKNOWN'); $logger = $app->getResource("logger"); diff --git a/app/init.php b/app/init.php index 0a241813b5..acaf3daaca 100644 --- a/app/init.php +++ b/app/init.php @@ -74,6 +74,7 @@ use Utopia\Logger\Adapter\Raygun; use Utopia\Logger\Adapter\Sentry; use Utopia\Logger\Log; use Utopia\Logger\Logger; +use Utopia\Pools\Connection as PoolConnection; use Utopia\Pools\Group; use Utopia\Pools\Pool; use Utopia\Queue; @@ -1412,7 +1413,26 @@ App::setResource('console', function () { ]); }, []); -App::setResource('dbForProject', function (Group $pools, Database $dbForPlatform, Cache $cache, Document $project) { +App::setResource('connectionForProject', function (Group $pools, Document $project) { + if ($project->isEmpty() || $project->getId() === 'console') { + return $pools + ->get('console') + ->pop(); + } + + try { + $dsn = new DSN($project->getAttribute('database')); + } catch (\InvalidArgumentException) { + // TODO: Temporary until all projects are using shared tables + $dsn = new DSN('mysql://' . $project->getAttribute('database')); + } + + return $pools + ->get($dsn->getHost()) + ->pop(); +}, ['pools', 'project']); + +App::setResource('dbForProject', function (Group $pools, PoolConnection $connectionForProject, Database $dbForPlatform, Cache $cache, Document $project) { if ($project->isEmpty() || $project->getId() === 'console') { return $dbForPlatform; } @@ -1424,12 +1444,7 @@ App::setResource('dbForProject', function (Group $pools, Database $dbForPlatform $dsn = new DSN('mysql://' . $project->getAttribute('database')); } - $dbAdapter = $pools - ->get($dsn->getHost()) - ->pop() - ->getResource(); - - $database = new Database($dbAdapter, $cache); + $database = new Database($connectionForProject->getResource(), $cache); $database ->setMetadata('host', \gethostname()) @@ -1452,7 +1467,7 @@ App::setResource('dbForProject', function (Group $pools, Database $dbForPlatform } return $database; -}, ['pools', 'dbForPlatform', 'cache', 'project']); +}, ['pools', 'connectionForProject', 'dbForPlatform', 'cache', 'project']); App::setResource('dbForPlatform', function (Group $pools, Cache $cache) { $dbAdapter = $pools diff --git a/app/worker.php b/app/worker.php index 6eb1363e9b..62882b32b1 100644 --- a/app/worker.php +++ b/app/worker.php @@ -16,6 +16,7 @@ use Appwrite\Event\Migration; use Appwrite\Event\Usage; use Appwrite\Event\UsageDump; use Appwrite\Platform\Appwrite; +use Swoole\Database\DetectsLostConnections; use Swoole\Runtime; use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis; use Utopia\Cache\Adapter\Sharding; @@ -25,11 +26,13 @@ use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; +use Utopia\Database\Exception as DatabaseException; use Utopia\Database\Validator\Authorization; use Utopia\DSN\DSN; use Utopia\Logger\Log; use Utopia\Logger\Logger; use Utopia\Platform\Service; +use Utopia\Pools\Connection as PoolConnection; use Utopia\Pools\Group; use Utopia\Queue\Connection; use Utopia\Queue\Message; @@ -66,13 +69,13 @@ Server::setResource('project', function (Message $message, Database $dbForPlatfo return $dbForPlatform->getDocument('projects', $project->getId()); }, ['message', 'dbForPlatform']); -Server::setResource('dbForProject', function (Cache $cache, Registry $register, Message $message, Document $project, Database $dbForPlatform) { +Server::setResource('connectionForProject', function (Group $pools, Document $project) { if ($project->isEmpty() || $project->getId() === 'console') { - return $dbForPlatform; + return $pools + ->get('console') + ->pop(); } - $pools = $register->get('pools'); - try { $dsn = new DSN($project->getAttribute('database')); } catch (\InvalidArgumentException) { @@ -80,12 +83,17 @@ Server::setResource('dbForProject', function (Cache $cache, Registry $register, $dsn = new DSN('mysql://' . $project->getAttribute('database')); } - $adapter = $pools + return $pools ->get($dsn->getHost()) - ->pop() - ->getResource(); + ->pop(); +}, ['pools', 'project']); - $database = new Database($adapter, $cache); +Server::setResource('dbForProject', function (PoolConnection $connectionForProject, Cache $cache, Registry $register, Message $message, Document $project, Database $dbForPlatform) { + if ($project->isEmpty() || $project->getId() === 'console') { + return $dbForPlatform; + } + + $database = new Database($connectionForProject->getResource(), $cache); try { $dsn = new DSN($project->getAttribute('database')); @@ -109,12 +117,12 @@ Server::setResource('dbForProject', function (Cache $cache, Registry $register, } return $database; -}, ['cache', 'register', 'message', 'project', 'dbForPlatform']); +}, ['connectionForProject', 'cache', 'register', 'message', 'project', 'dbForPlatform']); -Server::setResource('getProjectDB', function (Group $pools, Database $dbForPlatform, $cache) { +Server::setResource('getProjectDB', function (Group $pools, PoolConnection $connectionForProject, Database $dbForPlatform, $cache) { $databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools - return function (Document $project) use ($pools, $dbForPlatform, $cache, &$databases): Database { + return function (Document $project) use ($pools, $connectionForProject, $dbForPlatform, $cache, &$databases): Database { if ($project->isEmpty() || $project->getId() === 'console') { return $dbForPlatform; } @@ -146,12 +154,7 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForPlatf return $database; } - $dbAdapter = $pools - ->get($dsn->getHost()) - ->pop() - ->getResource(); - - $database = new Database($dbAdapter, $cache); + $database = new Database($connectionForProject->getResource(), $cache); $databases[$dsn->getHost()] = $database; @@ -171,7 +174,7 @@ Server::setResource('getProjectDB', function (Group $pools, Database $dbForPlatf return $database; }; -}, ['pools', 'dbForPlatform', 'cache']); +}, ['pools', 'connectionForProject', 'dbForPlatform', 'cache']); Server::setResource('abuseRetention', function () { return time() - (int) System::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', 86400); @@ -410,7 +413,16 @@ $worker ->inject('log') ->inject('pools') ->inject('project') - ->action(function (Throwable $error, ?Logger $logger, Log $log, Group $pools, Document $project) use ($queueName) { + ->inject('connectionForProject') + ->action(function (Throwable $error, ?Logger $logger, Log $log, Group $pools, Document $project, PoolConnection $connectionForProject) use ($queueName) { + if ( + ($error instanceof PDOException || $error instanceof DatabaseException) + && DetectsLostConnections::causedByLostConnection($error) + ) { + // Mark connection as unhealthy, it will be recycled on next reclaim. + $connectionForProject->setHealthy(false); + } + $pools->reclaim(); $version = System::getEnv('_APP_VERSION', 'UNKNOWN'); diff --git a/composer.json b/composer.json index 882be5b94e..bc51d6b359 100644 --- a/composer.json +++ b/composer.json @@ -63,7 +63,7 @@ "utopia-php/migration": "0.6.*", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.7.*", - "utopia-php/pools": "0.5.*", + "utopia-php/pools": "0.6.*", "utopia-php/preloader": "0.2.*", "utopia-php/queue": "0.7.*", "utopia-php/registry": "0.5.*", diff --git a/composer.lock b/composer.lock index b77915f0a4..c42c52d403 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "c88950f1d3119d0764a469e1d804ce71", + "content-hash": "a84318e49e4ac82fac110ef65a2847ea", "packages": [ { "name": "adhocore/jwt", @@ -4145,25 +4145,25 @@ }, { "name": "utopia-php/pools", - "version": "0.5.0", + "version": "0.6.0", "source": { "type": "git", "url": "https://github.com/utopia-php/pools.git", - "reference": "6f716a213a08db95eda1b5dddfa90983c1834817" + "reference": "59414ab7b57728edfde6d5ccc5a2583b7967ac18" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/pools/zipball/6f716a213a08db95eda1b5dddfa90983c1834817", - "reference": "6f716a213a08db95eda1b5dddfa90983c1834817", + "url": "https://api.github.com/repos/utopia-php/pools/zipball/59414ab7b57728edfde6d5ccc5a2583b7967ac18", + "reference": "59414ab7b57728edfde6d5ccc5a2583b7967ac18", "shasum": "" }, "require": { - "php": ">=8.0" + "php": ">=8.3" }, "require-dev": { - "laravel/pint": "1.2.*", - "phpstan/phpstan": "1.8.*", - "phpunit/phpunit": "^9.3" + "laravel/pint": "1.*", + "phpstan/phpstan": "1.*", + "phpunit/phpunit": "11.*" }, "type": "library", "autoload": { @@ -4190,9 +4190,9 @@ ], "support": { "issues": "https://github.com/utopia-php/pools/issues", - "source": "https://github.com/utopia-php/pools/tree/0.5.0" + "source": "https://github.com/utopia-php/pools/tree/0.6.0" }, - "time": "2024-04-19T11:11:54+00:00" + "time": "2025-01-08T07:58:42+00:00" }, { "name": "utopia-php/preloader", From d997b558dc750dd7038033839e823cb406ac2544 Mon Sep 17 00:00:00 2001 From: Darshan Date: Mon, 13 Jan 2025 11:20:17 +0530 Subject: [PATCH 73/74] update: there are no multiple connections for logs. As confirmed by Damodar, Logs DB is similar to Console DB. so no multiple connections. --- app/init.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/init.php b/app/init.php index af83368082..5a4731c1e6 100644 --- a/app/init.php +++ b/app/init.php @@ -872,7 +872,7 @@ $register->set('pools', function () { 'logs' => [ 'type' => 'database', 'dsns' => System::getEnv('_APP_CONNECTIONS_DB_LOGS', $fallbackForDB), - 'multiple' => true, + 'multiple' => false, 'schemes' => ['mariadb', 'mysql'], ], 'queue' => [ From 1cd7c397decc498b0e8aba18fdda9f5c1c7abf77 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 13 Jan 2025 19:24:58 +1300 Subject: [PATCH 74/74] Revert "Fix dead connections" --- app/controllers/general.php | 14 +---------- app/http.php | 11 -------- app/init.php | 31 ++++++----------------- app/worker.php | 50 ++++++++++++++----------------------- composer.json | 2 +- composer.lock | 22 ++++++++-------- 6 files changed, 40 insertions(+), 90 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 90cb3293e6..db9df71341 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -21,7 +21,6 @@ use Appwrite\Utopia\Response\Filters\V18 as ResponseV18; use Appwrite\Utopia\View; use Executor\Executor; use MaxMind\Db\Reader; -use Swoole\Database\DetectsLostConnections; use Swoole\Http\Request as SwooleRequest; use Utopia\App; use Utopia\CLI\Console; @@ -29,7 +28,6 @@ use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; -use Utopia\Database\Exception as DatabaseException; use Utopia\Database\Helpers\ID; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; @@ -40,7 +38,6 @@ use Utopia\Logger\Adapter\Sentry; use Utopia\Logger\Log; use Utopia\Logger\Log\User; use Utopia\Logger\Logger; -use Utopia\Pools\Connection; use Utopia\System\System; use Utopia\Validator\Hostname; use Utopia\Validator\Text; @@ -749,16 +746,7 @@ App::error() ->inject('logger') ->inject('log') ->inject('queueForUsage') - ->inject('connectionForProject') - ->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, Log $log, Usage $queueForUsage, Connection $connectionForProject) { - if ( - ($error instanceof PDOException || $error instanceof DatabaseException) - && DetectsLostConnections::causedByLostConnection($error) - ) { - // Mark connection as unhealthy so it will be recycled on next reclaim. - $connectionForProject->setHealthy(false); - } - + ->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, Log $log, Usage $queueForUsage) { $version = System::getEnv('_APP_VERSION', 'UNKNOWN'); $route = $utopia->getRoute(); $class = \get_class($error); diff --git a/app/http.php b/app/http.php index b988c7d8ac..3a7562ffd1 100644 --- a/app/http.php +++ b/app/http.php @@ -5,7 +5,6 @@ require_once __DIR__ . '/../vendor/autoload.php'; use Appwrite\Utopia\Request; use Appwrite\Utopia\Response; use Swoole\Constant; -use Swoole\Database\DetectsLostConnections; use Swoole\Http\Request as SwooleRequest; use Swoole\Http\Response as SwooleResponse; use Swoole\Http\Server; @@ -18,7 +17,6 @@ use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; -use Utopia\Database\Exception as DatabaseException; use Utopia\Database\Exception\Duplicate; use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Permission; @@ -348,15 +346,6 @@ $http->on(Constant::EVENT_REQUEST, function (SwooleRequest $swooleRequest, Swool $app->run($request, $response); } catch (\Throwable $th) { - if ( - ($th instanceof PDOException || $th instanceof DatabaseException) - && DetectsLostConnections::causedByLostConnection($th) - ) { - // Mark connection as unhealthy so it will be recycled on next reclaim. - $connectionForProject = $app->getResource('connectionForProject'); - $connectionForProject->setHealthy(false); - } - $version = System::getEnv('_APP_VERSION', 'UNKNOWN'); $logger = $app->getResource("logger"); diff --git a/app/init.php b/app/init.php index acaf3daaca..0a241813b5 100644 --- a/app/init.php +++ b/app/init.php @@ -74,7 +74,6 @@ use Utopia\Logger\Adapter\Raygun; use Utopia\Logger\Adapter\Sentry; use Utopia\Logger\Log; use Utopia\Logger\Logger; -use Utopia\Pools\Connection as PoolConnection; use Utopia\Pools\Group; use Utopia\Pools\Pool; use Utopia\Queue; @@ -1413,26 +1412,7 @@ App::setResource('console', function () { ]); }, []); -App::setResource('connectionForProject', function (Group $pools, Document $project) { - if ($project->isEmpty() || $project->getId() === 'console') { - return $pools - ->get('console') - ->pop(); - } - - try { - $dsn = new DSN($project->getAttribute('database')); - } catch (\InvalidArgumentException) { - // TODO: Temporary until all projects are using shared tables - $dsn = new DSN('mysql://' . $project->getAttribute('database')); - } - - return $pools - ->get($dsn->getHost()) - ->pop(); -}, ['pools', 'project']); - -App::setResource('dbForProject', function (Group $pools, PoolConnection $connectionForProject, Database $dbForPlatform, Cache $cache, Document $project) { +App::setResource('dbForProject', function (Group $pools, Database $dbForPlatform, Cache $cache, Document $project) { if ($project->isEmpty() || $project->getId() === 'console') { return $dbForPlatform; } @@ -1444,7 +1424,12 @@ App::setResource('dbForProject', function (Group $pools, PoolConnection $connect $dsn = new DSN('mysql://' . $project->getAttribute('database')); } - $database = new Database($connectionForProject->getResource(), $cache); + $dbAdapter = $pools + ->get($dsn->getHost()) + ->pop() + ->getResource(); + + $database = new Database($dbAdapter, $cache); $database ->setMetadata('host', \gethostname()) @@ -1467,7 +1452,7 @@ App::setResource('dbForProject', function (Group $pools, PoolConnection $connect } return $database; -}, ['pools', 'connectionForProject', 'dbForPlatform', 'cache', 'project']); +}, ['pools', 'dbForPlatform', 'cache', 'project']); App::setResource('dbForPlatform', function (Group $pools, Cache $cache) { $dbAdapter = $pools diff --git a/app/worker.php b/app/worker.php index 62882b32b1..6eb1363e9b 100644 --- a/app/worker.php +++ b/app/worker.php @@ -16,7 +16,6 @@ use Appwrite\Event\Migration; use Appwrite\Event\Usage; use Appwrite\Event\UsageDump; use Appwrite\Platform\Appwrite; -use Swoole\Database\DetectsLostConnections; use Swoole\Runtime; use Utopia\Abuse\Adapters\TimeLimit\Redis as TimeLimitRedis; use Utopia\Cache\Adapter\Sharding; @@ -26,13 +25,11 @@ use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; -use Utopia\Database\Exception as DatabaseException; use Utopia\Database\Validator\Authorization; use Utopia\DSN\DSN; use Utopia\Logger\Log; use Utopia\Logger\Logger; use Utopia\Platform\Service; -use Utopia\Pools\Connection as PoolConnection; use Utopia\Pools\Group; use Utopia\Queue\Connection; use Utopia\Queue\Message; @@ -69,13 +66,13 @@ Server::setResource('project', function (Message $message, Database $dbForPlatfo return $dbForPlatform->getDocument('projects', $project->getId()); }, ['message', 'dbForPlatform']); -Server::setResource('connectionForProject', function (Group $pools, Document $project) { +Server::setResource('dbForProject', function (Cache $cache, Registry $register, Message $message, Document $project, Database $dbForPlatform) { if ($project->isEmpty() || $project->getId() === 'console') { - return $pools - ->get('console') - ->pop(); + return $dbForPlatform; } + $pools = $register->get('pools'); + try { $dsn = new DSN($project->getAttribute('database')); } catch (\InvalidArgumentException) { @@ -83,17 +80,12 @@ Server::setResource('connectionForProject', function (Group $pools, Document $pr $dsn = new DSN('mysql://' . $project->getAttribute('database')); } - return $pools + $adapter = $pools ->get($dsn->getHost()) - ->pop(); -}, ['pools', 'project']); + ->pop() + ->getResource(); -Server::setResource('dbForProject', function (PoolConnection $connectionForProject, Cache $cache, Registry $register, Message $message, Document $project, Database $dbForPlatform) { - if ($project->isEmpty() || $project->getId() === 'console') { - return $dbForPlatform; - } - - $database = new Database($connectionForProject->getResource(), $cache); + $database = new Database($adapter, $cache); try { $dsn = new DSN($project->getAttribute('database')); @@ -117,12 +109,12 @@ Server::setResource('dbForProject', function (PoolConnection $connectionForProje } return $database; -}, ['connectionForProject', 'cache', 'register', 'message', 'project', 'dbForPlatform']); +}, ['cache', 'register', 'message', 'project', 'dbForPlatform']); -Server::setResource('getProjectDB', function (Group $pools, PoolConnection $connectionForProject, Database $dbForPlatform, $cache) { +Server::setResource('getProjectDB', function (Group $pools, Database $dbForPlatform, $cache) { $databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools - return function (Document $project) use ($pools, $connectionForProject, $dbForPlatform, $cache, &$databases): Database { + return function (Document $project) use ($pools, $dbForPlatform, $cache, &$databases): Database { if ($project->isEmpty() || $project->getId() === 'console') { return $dbForPlatform; } @@ -154,7 +146,12 @@ Server::setResource('getProjectDB', function (Group $pools, PoolConnection $conn return $database; } - $database = new Database($connectionForProject->getResource(), $cache); + $dbAdapter = $pools + ->get($dsn->getHost()) + ->pop() + ->getResource(); + + $database = new Database($dbAdapter, $cache); $databases[$dsn->getHost()] = $database; @@ -174,7 +171,7 @@ Server::setResource('getProjectDB', function (Group $pools, PoolConnection $conn return $database; }; -}, ['pools', 'connectionForProject', 'dbForPlatform', 'cache']); +}, ['pools', 'dbForPlatform', 'cache']); Server::setResource('abuseRetention', function () { return time() - (int) System::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', 86400); @@ -413,16 +410,7 @@ $worker ->inject('log') ->inject('pools') ->inject('project') - ->inject('connectionForProject') - ->action(function (Throwable $error, ?Logger $logger, Log $log, Group $pools, Document $project, PoolConnection $connectionForProject) use ($queueName) { - if ( - ($error instanceof PDOException || $error instanceof DatabaseException) - && DetectsLostConnections::causedByLostConnection($error) - ) { - // Mark connection as unhealthy, it will be recycled on next reclaim. - $connectionForProject->setHealthy(false); - } - + ->action(function (Throwable $error, ?Logger $logger, Log $log, Group $pools, Document $project) use ($queueName) { $pools->reclaim(); $version = System::getEnv('_APP_VERSION', 'UNKNOWN'); diff --git a/composer.json b/composer.json index bc51d6b359..882be5b94e 100644 --- a/composer.json +++ b/composer.json @@ -63,7 +63,7 @@ "utopia-php/migration": "0.6.*", "utopia-php/orchestration": "0.9.*", "utopia-php/platform": "0.7.*", - "utopia-php/pools": "0.6.*", + "utopia-php/pools": "0.5.*", "utopia-php/preloader": "0.2.*", "utopia-php/queue": "0.7.*", "utopia-php/registry": "0.5.*", diff --git a/composer.lock b/composer.lock index c42c52d403..b77915f0a4 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "a84318e49e4ac82fac110ef65a2847ea", + "content-hash": "c88950f1d3119d0764a469e1d804ce71", "packages": [ { "name": "adhocore/jwt", @@ -4145,25 +4145,25 @@ }, { "name": "utopia-php/pools", - "version": "0.6.0", + "version": "0.5.0", "source": { "type": "git", "url": "https://github.com/utopia-php/pools.git", - "reference": "59414ab7b57728edfde6d5ccc5a2583b7967ac18" + "reference": "6f716a213a08db95eda1b5dddfa90983c1834817" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/pools/zipball/59414ab7b57728edfde6d5ccc5a2583b7967ac18", - "reference": "59414ab7b57728edfde6d5ccc5a2583b7967ac18", + "url": "https://api.github.com/repos/utopia-php/pools/zipball/6f716a213a08db95eda1b5dddfa90983c1834817", + "reference": "6f716a213a08db95eda1b5dddfa90983c1834817", "shasum": "" }, "require": { - "php": ">=8.3" + "php": ">=8.0" }, "require-dev": { - "laravel/pint": "1.*", - "phpstan/phpstan": "1.*", - "phpunit/phpunit": "11.*" + "laravel/pint": "1.2.*", + "phpstan/phpstan": "1.8.*", + "phpunit/phpunit": "^9.3" }, "type": "library", "autoload": { @@ -4190,9 +4190,9 @@ ], "support": { "issues": "https://github.com/utopia-php/pools/issues", - "source": "https://github.com/utopia-php/pools/tree/0.6.0" + "source": "https://github.com/utopia-php/pools/tree/0.5.0" }, - "time": "2025-01-08T07:58:42+00:00" + "time": "2024-04-19T11:11:54+00:00" }, { "name": "utopia-php/preloader",