Merge branch '1.4.x' into usage-auto-skip-fix

This commit is contained in:
Christy Jacob 2023-11-15 10:27:44 -08:00 committed by GitHub
commit e36dc817fb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 587 additions and 154 deletions

View file

@ -100,6 +100,7 @@ RUN chmod +x /usr/local/bin/doctor && \
RUN chmod +x /usr/local/bin/hamster && \
chmod +x /usr/local/bin/volume-sync && \
chmod +x /usr/local/bin/patch-delete-schedule-updated-at-attribute && \
chmod +x /usr/local/bin/patch-recreate-repositories-documents && \
chmod +x /usr/local/bin/patch-delete-project-collections && \
chmod +x /usr/local/bin/delete-orphaned-projects && \
chmod +x /usr/local/bin/clear-card-cache && \

View file

@ -240,6 +240,16 @@ return [
'description' => 'OAuth2 provider returned some error.',
'code' => 424,
],
Exception::USER_EMAIL_ALREADY_VERIFIED => [
'name' => Exception::USER_EMAIL_ALREADY_VERIFIED,
'description' => 'User email is already verified',
'code' => 409,
],
Exception::USER_PHONE_ALREADY_VERIFIED => [
'name' => Exception::USER_PHONE_ALREADY_VERIFIED,
'description' => 'User phone is already verified',
'code' => 409
],
/** Teams */
Exception::TEAM_NOT_FOUND => [

View file

@ -2662,6 +2662,10 @@ App::post('/v1/account/verification')
throw new Exception(Exception::GENERAL_SMTP_DISABLED, 'SMTP Disabled');
}
if ($user->getAttribute('emailVerification')) {
throw new Exception(Exception::USER_EMAIL_ALREADY_VERIFIED);
}
$roles = Authorization::getRoles();
$isPrivilegedUser = Auth::isPrivilegedUser($roles);
$isAppUser = Auth::isAppUser($roles);
@ -2883,6 +2887,10 @@ App::post('/v1/account/verification/phone')
throw new Exception(Exception::USER_PHONE_NOT_FOUND);
}
if ($user->getAttribute('phoneVerification')) {
throw new Exception(Exception::USER_PHONE_ALREADY_VERIFIED);
}
$roles = Authorization::getRoles();
$isPrivilegedUser = Auth::isPrivilegedUser($roles);
$isAppUser = Auth::isAppUser($roles);

View file

@ -242,12 +242,16 @@ App::post('/v1/functions')
// Git connect logic
if (!empty($providerRepositoryId)) {
$teamId = $project->getAttribute('teamId', '');
$repository = $dbForConsole->createDocument('repositories', new Document([
'$id' => ID::unique(),
'$permissions' => [
Permission::read(Role::any()),
Permission::update(Role::any()),
Permission::delete(Role::any()),
Permission::read(Role::team(ID::custom($teamId))),
Permission::update(Role::team(ID::custom($teamId), 'owner')),
Permission::update(Role::team(ID::custom($teamId), 'developer')),
Permission::delete(Role::team(ID::custom($teamId), 'owner')),
Permission::delete(Role::team(ID::custom($teamId), 'developer')),
],
'installationId' => $installation->getId(),
'installationInternalId' => $installation->getInternalId(),

View file

@ -14,6 +14,7 @@ use Utopia\Registry\Registry;
use Utopia\Storage\Device;
use Utopia\Storage\Device\Local;
use Utopia\Storage\Storage;
use Utopia\Validator\Integer;
use Utopia\Validator\Text;
App::get('/v1/health')
@ -344,11 +345,20 @@ App::get('/v1/health/queue/webhooks')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
->inject('queue')
->inject('response')
->action(function (Connection $queue, Response $response) {
->action(function (int|string $threshold, Connection $queue, Response $response) {
$threshold = \intval($threshold);
$client = new Client(Event::WEBHOOK_QUEUE_NAME, $queue);
$response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE);
$size = $client->getQueueSize();
if ($size >= $threshold) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
}, ['response']);
App::get('/v1/health/queue/logs')
@ -362,11 +372,20 @@ App::get('/v1/health/queue/logs')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
->inject('queue')
->inject('response')
->action(function (Connection $queue, Response $response) {
->action(function (int|string $threshold, Connection $queue, Response $response) {
$threshold = \intval($threshold);
$client = new Client(Event::AUDITS_QUEUE_NAME, $queue);
$response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE);
$size = $client->getQueueSize();
if ($size >= $threshold) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
}, ['response']);
App::get('/v1/health/queue/certificates')
@ -380,11 +399,20 @@ App::get('/v1/health/queue/certificates')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
->inject('queue')
->inject('response')
->action(function (Connection $queue, Response $response) {
->action(function (int|string $threshold, Connection $queue, Response $response) {
$threshold = \intval($threshold);
$client = new Client(Event::CERTIFICATES_QUEUE_NAME, $queue);
$response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE);
$size = $client->getQueueSize();
if ($size >= $threshold) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
}, ['response']);
App::get('/v1/health/queue/builds')
@ -398,11 +426,20 @@ App::get('/v1/health/queue/builds')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
->inject('queue')
->inject('response')
->action(function (Connection $queue, Response $response) {
->action(function (int|string $threshold, Connection $queue, Response $response) {
$threshold = \intval($threshold);
$client = new Client(Event::BUILDS_QUEUE_NAME, $queue);
$response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE);
$size = $client->getQueueSize();
if ($size >= $threshold) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
}, ['response']);
App::get('/v1/health/queue/databases')
@ -417,11 +454,20 @@ App::get('/v1/health/queue/databases')
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
->param('name', 'database_db_main', new Text(256), 'Queue name for which to check the queue size', true)
->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
->inject('queue')
->inject('response')
->action(function (string $name, Connection $queue, Response $response) {
->action(function (string $name, int|string $threshold, Connection $queue, Response $response) {
$threshold = \intval($threshold);
$client = new Client($name, $queue);
$response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE);
$size = $client->getQueueSize();
if ($size >= $threshold) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
}, ['response']);
App::get('/v1/health/queue/deletes')
@ -435,11 +481,20 @@ App::get('/v1/health/queue/deletes')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
->inject('queue')
->inject('response')
->action(function (Connection $queue, Response $response) {
->action(function (int|string $threshold, Connection $queue, Response $response) {
$threshold = \intval($threshold);
$client = new Client(Event::DELETE_QUEUE_NAME, $queue);
$response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE);
$size = $client->getQueueSize();
if ($size >= $threshold) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
}, ['response']);
App::get('/v1/health/queue/mails')
@ -453,11 +508,20 @@ App::get('/v1/health/queue/mails')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
->inject('queue')
->inject('response')
->action(function (Connection $queue, Response $response) {
->action(function (int|string $threshold, Connection $queue, Response $response) {
$threshold = \intval($threshold);
$client = new Client(Event::MAILS_QUEUE_NAME, $queue);
$response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE);
$size = $client->getQueueSize();
if ($size >= $threshold) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
}, ['response']);
App::get('/v1/health/queue/messaging')
@ -471,11 +535,20 @@ App::get('/v1/health/queue/messaging')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
->inject('queue')
->inject('response')
->action(function (Connection $queue, Response $response) {
->action(function (int|string $threshold, Connection $queue, Response $response) {
$threshold = \intval($threshold);
$client = new Client(Event::MESSAGING_QUEUE_NAME, $queue);
$response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE);
$size = $client->getQueueSize();
if ($size >= $threshold) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
}, ['response']);
App::get('/v1/health/queue/migrations')
@ -489,11 +562,20 @@ App::get('/v1/health/queue/migrations')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
->inject('queue')
->inject('response')
->action(function (Connection $queue, Response $response) {
->action(function (int|string $threshold, Connection $queue, Response $response) {
$threshold = \intval($threshold);
$client = new Client(Event::MIGRATIONS_QUEUE_NAME, $queue);
$response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE);
$size = $client->getQueueSize();
if ($size >= $threshold) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
}, ['response']);
App::get('/v1/health/queue/functions')
@ -507,11 +589,20 @@ App::get('/v1/health/queue/functions')
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE)
->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
->inject('queue')
->inject('response')
->action(function (Connection $queue, Response $response) {
->action(function (int|string $threshold, Connection $queue, Response $response) {
$threshold = \intval($threshold);
$client = new Client(Event::FUNCTIONS_QUEUE_NAME, $queue);
$response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE);
$size = $client->getQueueSize();
if ($size >= $threshold) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
}, ['response']);
App::get('/v1/health/storage/local')

View file

@ -857,10 +857,10 @@ App::post('/v1/vcs/github/events')
$github->initializeVariables($providerInstallationId, $privateKey, $githubAppId);
//find functionId from functions table
$repositories = $dbForConsole->find('repositories', [
$repositories = Authorization::skip(fn () => $dbForConsole->find('repositories', [
Query::equal('providerRepositoryId', [$providerRepositoryId]),
Query::limit(100),
]);
]));
// create new deployment only on push and not when branch is created
if (!$providerBranchCreated) {
@ -877,13 +877,13 @@ App::post('/v1/vcs/github/events')
]);
foreach ($installations as $installation) {
$repositories = $dbForConsole->find('repositories', [
$repositories = Authorization::skip(fn () => $dbForConsole->find('repositories', [
Query::equal('installationInternalId', [$installation->getInternalId()]),
Query::limit(1000)
]);
]));
foreach ($repositories as $repository) {
$dbForConsole->deleteDocument('repositories', $repository->getId());
Authorization::skip(fn () => $dbForConsole->deleteDocument('repositories', $repository->getId()));
}
$dbForConsole->deleteDocument('installations', $installation->getId());
@ -915,10 +915,10 @@ App::post('/v1/vcs/github/events')
$providerCommitAuthor = $commitDetails["commitAuthor"] ?? '';
$providerCommitMessage = $commitDetails["commitMessage"] ?? '';
$repositories = $dbForConsole->find('repositories', [
$repositories = Authorization::skip(fn () => $dbForConsole->find('repositories', [
Query::equal('providerRepositoryId', [$providerRepositoryId]),
Query::orderDesc('$createdAt')
]);
]));
$createGitDeployments($github, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthor, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, $providerPullRequestId, $external, $dbForConsole, $queueForBuilds, $getProjectDB, $request);
} elseif ($parsedPayload["action"] == "closed") {
@ -929,10 +929,10 @@ App::post('/v1/vcs/github/events')
$external = $parsedPayload["external"] ?? true;
if ($external) {
$repositories = $dbForConsole->find('repositories', [
$repositories = Authorization::skip(fn () => $dbForConsole->find('repositories', [
Query::equal('providerRepositoryId', [$providerRepositoryId]),
Query::orderDesc('$createdAt')
]);
]));
foreach ($repositories as $repository) {
$providerPullRequestIds = $repository->getAttribute('providerPullRequestIds', []);
@ -1046,8 +1046,8 @@ App::delete('/v1/vcs/installations/:installationId')
->inject('response')
->inject('project')
->inject('dbForConsole')
->inject('deletes')
->action(function (string $installationId, Response $response, Document $project, Database $dbForConsole, Delete $deletes) {
->inject('queueForDeletes')
->action(function (string $installationId, Response $response, Document $project, Database $dbForConsole, Delete $queueForDeletes) {
$installation = $dbForConsole->getDocument('installations', $installationId);
if ($installation->isEmpty()) {
@ -1058,7 +1058,7 @@ App::delete('/v1/vcs/installations/:installationId')
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove installation from DB');
}
$deletes
$queueForDeletes
->setType(DELETE_TYPE_DOCUMENT)
->setDocument($installation);
@ -1092,9 +1092,9 @@ App::patch('/v1/vcs/github/installations/:installationId/repositories/:repositor
throw new Exception(Exception::INSTALLATION_NOT_FOUND);
}
$repository = $dbForConsole->getDocument('repositories', $repositoryId, [
$repository = Authorization::skip(fn () => $dbForConsole->getDocument('repositories', $repositoryId, [
Query::equal('projectInternalId', [$project->getInternalId()])
]);
]));
if ($repository->isEmpty()) {
throw new Exception(Exception::REPOSITORY_NOT_FOUND);
@ -1109,7 +1109,7 @@ App::patch('/v1/vcs/github/installations/:installationId/repositories/:repositor
// TODO: Delete from array when PR is closed
$repository = $dbForConsole->updateDocument('repositories', $repository->getId(), $repository);
$repository = Authorization::skip(fn () => $dbForConsole->updateDocument('repositories', $repository->getId(), $repository));
$privateKey = App::getEnv('_APP_VCS_GITHUB_PRIVATE_KEY');
$githubAppId = App::getEnv('_APP_VCS_GITHUB_APP_ID');

View file

@ -0,0 +1,3 @@
#!/bin/sh
php /usr/src/code/app/cli.php patch-recreate-repositories-documents $@

View file

@ -84,6 +84,8 @@ class Exception extends \Exception
public const USER_OAUTH2_BAD_REQUEST = 'user_oauth2_bad_request';
public const USER_OAUTH2_UNAUTHORIZED = 'user_oauth2_unauthorized';
public const USER_OAUTH2_PROVIDER_ERROR = 'user_oauth2_provider_error';
public const USER_EMAIL_ALREADY_VERIFIED = 'user_email_alread_verified';
public const USER_PHONE_ALREADY_VERIFIED = 'user_phone_already_verified';
/** Teams */
public const TEAM_NOT_FOUND = 'team_not_found';

View file

@ -19,6 +19,7 @@ use Appwrite\Platform\Tasks\VolumeSync;
use Appwrite\Platform\Tasks\CalcTierStats;
use Appwrite\Platform\Tasks\Upgrade;
use Appwrite\Platform\Tasks\DeleteOrphanedProjects;
use Appwrite\Platform\Tasks\PatchRecreateRepositoriesDocuments;
class Tasks extends Service
{
@ -42,6 +43,7 @@ class Tasks extends Service
->addAction(Specs::getName(), new Specs())
->addAction(CalcTierStats::getName(), new CalcTierStats())
->addAction(DeleteOrphanedProjects::getName(), new DeleteOrphanedProjects())
->addAction(PatchRecreateRepositoriesDocuments::getName(), new PatchRecreateRepositoriesDocuments())
;
}

View file

@ -0,0 +1,169 @@
<?php
namespace Appwrite\Platform\Tasks;
use Utopia\Platform\Action;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Helpers\Permission;
use Utopia\Database\Helpers\Role;
use Utopia\Database\Query;
use Utopia\Validator\Text;
class PatchRecreateRepositoriesDocuments extends Action
{
public static function getName(): string
{
return 'patch-recreate-repositories-documents';
}
public function __construct()
{
$this
->desc('Recreate missing repositories in consoleDB from projectDBs. They can be missing if you used Appwrite 1.4.10 or 1.4.11, and deleted a function.')
->param('after', '', new Text(36), 'After cursor', true)
->param('projectId', '', new Text(36), 'Select project to validate', true)
->inject('dbForConsole')
->inject('getProjectDB')
->callback(fn ($after, $projectId, $dbForConsole, $getProjectDB) => $this->action($after, $projectId, $dbForConsole, $getProjectDB));
}
public function action($after, $projectId, Database $dbForConsole, callable $getProjectDB): void
{
Console::info("Starting the patch");
$startTime = microtime(true);
if (!empty($projectId)) {
try {
$project = $dbForConsole->getDocument('projects', $projectId);
$dbForProject = call_user_func($getProjectDB, $project);
$this->recreateRepositories($dbForConsole, $dbForProject, $project);
} catch (\Throwable $th) {
Console::error("Unexpected error occured with Project ID {$projectId}");
Console::error('[Error] Type: ' . get_class($th));
Console::error('[Error] Message: ' . $th->getMessage());
Console::error('[Error] File: ' . $th->getFile());
Console::error('[Error] Line: ' . $th->getLine());
}
} else {
$queries = [];
if (!empty($after)) {
Console::info("Iterating remaining projects after project with ID {$after}");
$project = $dbForConsole->getDocument('projects', $after);
$queries = [Query::cursorAfter($project)];
} else {
Console::info("Iterating all projects");
}
$this->foreachDocument($dbForConsole, 'projects', $queries, function (Document $project) use ($getProjectDB, $dbForConsole) {
$projectId = $project->getId();
try {
$dbForProject = call_user_func($getProjectDB, $project);
$this->recreateRepositories($dbForConsole, $dbForProject, $project);
} catch (\Throwable $th) {
Console::error("Unexpected error occured with Project ID {$projectId}");
Console::error('[Error] Type: ' . get_class($th));
Console::error('[Error] Message: ' . $th->getMessage());
Console::error('[Error] File: ' . $th->getFile());
Console::error('[Error] Line: ' . $th->getLine());
}
});
}
$endTime = microtime(true);
$timeTaken = $endTime - $startTime;
$hours = (int)($timeTaken / 3600);
$timeTaken -= $hours * 3600;
$minutes = (int)($timeTaken / 60);
$timeTaken -= $minutes * 60;
$seconds = (int)$timeTaken;
$milliseconds = ($timeTaken - $seconds) * 1000;
Console::info("Recreate patch completed in $hours h, $minutes m, $seconds s, $milliseconds mis ( total $timeTaken milliseconds)");
}
protected function foreachDocument(Database $database, string $collection, array $queries = [], callable $callback = null): void
{
$limit = 1000;
$results = [];
$sum = $limit;
$latestDocument = null;
while ($sum === $limit) {
$newQueries = $queries;
if ($latestDocument != null) {
array_unshift($newQueries, Query::cursorAfter($latestDocument));
}
$newQueries[] = Query::limit($limit);
$results = $database->find($collection, $newQueries);
if (empty($results)) {
return;
}
$sum = count($results);
foreach ($results as $document) {
if (is_callable($callback)) {
$callback($document);
}
}
$latestDocument = $results[array_key_last($results)];
}
}
public function recreateRepositories(Database $dbForConsole, Database $dbForProject, Document $project): void
{
$projectId = $project->getId();
Console::log("Running patch for project {$projectId}");
$this->foreachDocument($dbForProject, 'functions', [], function (Document $function) use ($dbForProject, $dbForConsole, $project) {
$isConnected = !empty($function->getAttribute('providerRepositoryId', ''));
if ($isConnected) {
$repository = $dbForConsole->getDocument('repositories', $function->getAttribute('repositoryId', ''));
if ($repository->isEmpty()) {
$projectId = $project->getId();
$functionId = $function->getId();
Console::success("Recreating repositories document for project ID {$projectId}, function ID {$functionId}");
$repository = $dbForConsole->createDocument('repositories', new Document([
'$id' => ID::unique(),
'$permissions' => [
Permission::read(Role::any()),
Permission::update(Role::any()),
Permission::delete(Role::any()),
],
'installationId' => $function->getAttribute('installationId', ''),
'installationInternalId' => $function->getAttribute('installationInternalId', ''),
'projectId' => $project->getId(),
'projectInternalId' => $project->getInternalId(),
'providerRepositoryId' => $function->getAttribute('providerRepositoryId', ''),
'resourceId' => $function->getId(),
'resourceInternalId' => $function->getInternalId(),
'resourceType' => 'function',
'providerPullRequestIds' => []
]));
$function = $dbForProject->updateDocument('functions', $function->getId(), $function
->setAttribute('repositoryId', $repository->getId())
->setAttribute('repositoryInternalId', $repository->getInternalId()));
$this->foreachDocument($dbForProject, 'deployments', [
Query::equal('resourceInternalId', [$function->getInternalId()]),
Query::equal('resourceType', ['functions'])
], function (Document $deployment) use ($dbForProject, $repository) {
$dbForProject->updateDocument('deployments', $deployment->getId(), $deployment
->setAttribute('repositoryId', $repository->getId())
->setAttribute('repositoryInternalId', $repository->getInternalId()));
});
}
}
});
}
}

View file

@ -730,14 +730,15 @@ class Deletes extends Action
*/
Console::info("Deleting VCS repositories and comments linked to function " . $functionId);
$this->deleteByGroup('repositories', [
Query::equal('projectInternalId', [$project->getInternalId()]),
Query::equal('resourceInternalId', [$functionInternalId]),
Query::equal('resourceType', ['function']),
], $dbForConsole, function (Document $document) use ($dbForConsole) {
$providerRepositoryId = $document->getAttribute('providerRepositoryId', '');
$projectId = $document->getAttribute('projectId', '');
$projectInternalId = $document->getAttribute('projectInternalId', '');
$this->deleteByGroup('vcsComments', [
Query::equal('providerRepositoryId', [$providerRepositoryId]),
Query::equal('projectId', [$projectId]),
Query::equal('projectInternalId', [$projectInternalId]),
], $dbForConsole);
});

View file

@ -945,6 +945,32 @@ trait AccountBase
return $data;
}
/**
* @depends testUpdateAccountVerification
*/
public function testCreateAccountVerificationForVerifiedEmail($data): array
{
$email = $data['email'] ?? '';
$name = $data['name'] ?? '';
$session = $data['session'] ?? '';
/**
* Test for SUCCESS
*/
$response = $this->client->call(Client::METHOD_POST, '/account/verification', array_merge([
'origin' => 'http://localhost',
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session,
]), [
'url' => 'http://localhost/verification',
]);
$this->assertEquals(409, $response['headers']['status-code']);
return $data;
}
/**
* @depends testUpdateAccountVerification
*/

View file

@ -1068,4 +1068,27 @@ class AccountCustomClientTest extends Scope
return $data;
}
/**
* @depends testPhoneVerification
*/
#[Retry(count: 1)]
public function testPhoneVerificationForVerifiedPhone(array $data): array
{
$session = $data['session'] ?? '';
/**
* Test for SUCCESS
*/
$response = $this->client->call(Client::METHOD_POST, '/account/verification/phone', array_merge([
'origin' => 'http://localhost',
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session,
]));
$this->assertEquals(409, $response['headers']['status-code']);
return $data;
}
}

View file

@ -138,6 +138,15 @@ class HealthCustomServerTest extends Scope
$this->assertIsInt($response['body']['size']);
$this->assertLessThan(100, $response['body']['size']);
/**
* Test for FAILURE
*/
$response = $this->client->call(Client::METHOD_GET, '/health/queue/webhooks?threshold=0', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(500, $response['headers']['status-code']);
return [];
}
@ -155,6 +164,15 @@ class HealthCustomServerTest extends Scope
$this->assertIsInt($response['body']['size']);
$this->assertLessThan(100, $response['body']['size']);
/**
* Test for FAILURE
*/
$response = $this->client->call(Client::METHOD_GET, '/health/queue/logs?threshold=0', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(500, $response['headers']['status-code']);
return [];
}
@ -172,6 +190,15 @@ class HealthCustomServerTest extends Scope
$this->assertIsInt($response['body']['size']);
$this->assertLessThan(100, $response['body']['size']);
/**
* Test for FAILURE
*/
$response = $this->client->call(Client::METHOD_GET, '/health/queue/certificates?threshold=0', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(500, $response['headers']['status-code']);
return [];
}
@ -189,6 +216,15 @@ class HealthCustomServerTest extends Scope
$this->assertIsInt($response['body']['size']);
$this->assertLessThan(100, $response['body']['size']);
/**
* Test for FAILURE
*/
$response = $this->client->call(Client::METHOD_GET, '/health/queue/functions?threshold=0', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(500, $response['headers']['status-code']);
return [];
}
@ -206,6 +242,15 @@ class HealthCustomServerTest extends Scope
$this->assertIsInt($response['body']['size']);
$this->assertLessThan(100, $response['body']['size']);
/**
* Test for FAILURE
*/
$response = $this->client->call(Client::METHOD_GET, '/health/queue/builds?threshold=0', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(500, $response['headers']['status-code']);
return [];
}
@ -225,6 +270,18 @@ class HealthCustomServerTest extends Scope
$this->assertIsInt($response['body']['size']);
$this->assertLessThan(100, $response['body']['size']);
/**
* Test for FAILURE
*/
$response = $this->client->call(Client::METHOD_GET, '/health/queue/databases', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'name' => 'database_db_main',
'threshold' => '0'
]);
$this->assertEquals(500, $response['headers']['status-code']);
return [];
}
@ -242,6 +299,15 @@ class HealthCustomServerTest extends Scope
$this->assertIsInt($response['body']['size']);
$this->assertLessThan(100, $response['body']['size']);
/**
* Test for FAILURE
*/
$response = $this->client->call(Client::METHOD_GET, '/health/queue/deletes?threshold=0', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(500, $response['headers']['status-code']);
return [];
}
@ -259,6 +325,15 @@ class HealthCustomServerTest extends Scope
$this->assertIsInt($response['body']['size']);
$this->assertLessThan(100, $response['body']['size']);
/**
* Test for FAILURE
*/
$response = $this->client->call(Client::METHOD_GET, '/health/queue/mails?threshold=0', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(500, $response['headers']['status-code']);
return [];
}
@ -276,6 +351,15 @@ class HealthCustomServerTest extends Scope
$this->assertIsInt($response['body']['size']);
$this->assertLessThan(100, $response['body']['size']);
/**
* Test for FAILURE
*/
$response = $this->client->call(Client::METHOD_GET, '/health/queue/messaging?threshold=0', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(500, $response['headers']['status-code']);
return [];
}
@ -293,6 +377,15 @@ class HealthCustomServerTest extends Scope
$this->assertIsInt($response['body']['size']);
$this->assertLessThan(100, $response['body']['size']);
/**
* Test for FAILURE
*/
$response = $this->client->call(Client::METHOD_GET, '/health/queue/migrations?threshold=0', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(500, $response['headers']['status-code']);
return [];
}

View file

@ -636,6 +636,120 @@ class WebhooksCustomClientTest extends Scope
return $data;
}
/**
* @depends testUpdateAccountPrefs
*/
public function testCreateAccountVerification($data): array
{
$id = $data['id'] ?? '';
$email = $data['email'] ?? '';
$session = $data['session'] ?? '';
$verification = $this->client->call(Client::METHOD_POST, '/account/verification', array_merge([
'origin' => 'http://localhost',
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session,
]), [
'url' => 'http://localhost/verification',
]);
$verificationId = $verification['body']['$id'];
$this->assertEquals(201, $verification['headers']['status-code']);
$this->assertIsArray($verification['body']);
$webhook = $this->getLastRequest();
$signatureKey = $this->getProject()['signatureKey'];
$payload = json_encode($webhook['data']);
$url = $webhook['url'];
$signatureExpected = base64_encode(hash_hmac('sha1', $url . $payload, $signatureKey, true));
$this->assertEquals($webhook['method'], 'POST');
$this->assertEquals($webhook['headers']['Content-Type'], 'application/json');
$this->assertEquals($webhook['headers']['User-Agent'], 'Appwrite-Server vdev. Please report abuse at security@appwrite.io');
$this->assertStringContainsString('users.*', $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString('users.*.verification.*', $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString('users.*.verification.*.create', $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString("users.*.verification.{$verificationId}", $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString("users.*.verification.{$verificationId}.create", $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString("users.{$id}", $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString("users.{$id}.verification.*", $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString("users.{$id}.verification.*.create", $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString("users.{$id}.verification.{$verificationId}", $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString("users.{$id}.verification.{$verificationId}.create", $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Signature'], $signatureExpected);
$this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Id'] ?? '', $this->getProject()['webhookId']);
$this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Project-Id'] ?? '', $this->getProject()['$id']);
$this->assertEquals(empty($webhook['headers']['X-Appwrite-Webhook-User-Id'] ?? ''), ('server' === $this->getSide()));
$this->assertNotEmpty($webhook['data']['$id']);
$this->assertNotEmpty($webhook['data']['userId']);
$this->assertNotEmpty($webhook['data']['secret']);
$this->assertEquals(true, (new DatetimeValidator())->isValid($webhook['data']['expire']));
$data['secret'] = $webhook['data']['secret'];
return $data;
}
/**
* @depends testCreateAccountVerification
*/
public function testUpdateAccountVerification($data): array
{
$id = $data['id'] ?? '';
$email = $data['email'] ?? '';
$session = $data['session'] ?? '';
$secret = $data['secret'] ?? '';
$verification = $this->client->call(Client::METHOD_PUT, '/account/verification', array_merge([
'origin' => 'http://localhost',
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session,
]), [
'userId' => $id,
'secret' => $secret,
]);
$verificationId = $verification['body']['$id'];
$this->assertEquals(200, $verification['headers']['status-code']);
$this->assertIsArray($verification['body']);
$webhook = $this->getLastRequest();
$signatureKey = $this->getProject()['signatureKey'];
$payload = json_encode($webhook['data']);
$url = $webhook['url'];
$signatureExpected = base64_encode(hash_hmac('sha1', $url . $payload, $signatureKey, true));
$this->assertEquals($webhook['method'], 'POST');
$this->assertEquals($webhook['headers']['Content-Type'], 'application/json');
$this->assertEquals($webhook['headers']['User-Agent'], 'Appwrite-Server vdev. Please report abuse at security@appwrite.io');
$this->assertStringContainsString('users.*', $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString('users.*.verification.*', $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString('users.*.verification.*.update', $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString("users.*.verification.{$verificationId}", $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString("users.*.verification.{$verificationId}.update", $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString("users.{$id}", $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString("users.{$id}.verification.*", $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString("users.{$id}.verification.*.update", $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString("users.{$id}.verification.{$verificationId}", $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString("users.{$id}.verification.{$verificationId}.update", $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Signature'], $signatureExpected);
$this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Id'] ?? '', $this->getProject()['webhookId']);
$this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Project-Id'] ?? '', $this->getProject()['$id']);
$this->assertEquals(empty($webhook['headers']['X-Appwrite-Webhook-User-Id'] ?? ''), ('server' === $this->getSide()));
$this->assertNotEmpty($webhook['data']['$id']);
$this->assertNotEmpty($webhook['data']['userId']);
$this->assertNotEmpty($webhook['data']['secret']);
$this->assertEquals(true, (new DatetimeValidator())->isValid($webhook['data']['expire']));
$data['secret'] = $webhook['data']['secret'];
return $data;
}
/**
* @depends testUpdateAccountPrefs
*/
@ -751,120 +865,6 @@ class WebhooksCustomClientTest extends Scope
return $data;
}
/**
* @depends testUpdateAccountPrefs
*/
public function testCreateAccountVerification($data): array
{
$id = $data['id'] ?? '';
$email = $data['email'] ?? '';
$session = $data['session'] ?? '';
$verification = $this->client->call(Client::METHOD_POST, '/account/verification', array_merge([
'origin' => 'http://localhost',
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session,
]), [
'url' => 'http://localhost/verification',
]);
$verificationId = $verification['body']['$id'];
$this->assertEquals(201, $verification['headers']['status-code']);
$this->assertIsArray($verification['body']);
$webhook = $this->getLastRequest();
$signatureKey = $this->getProject()['signatureKey'];
$payload = json_encode($webhook['data']);
$url = $webhook['url'];
$signatureExpected = base64_encode(hash_hmac('sha1', $url . $payload, $signatureKey, true));
$this->assertEquals($webhook['method'], 'POST');
$this->assertEquals($webhook['headers']['Content-Type'], 'application/json');
$this->assertEquals($webhook['headers']['User-Agent'], 'Appwrite-Server vdev. Please report abuse at security@appwrite.io');
$this->assertStringContainsString('users.*', $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString('users.*.verification.*', $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString('users.*.verification.*.create', $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString("users.*.verification.{$verificationId}", $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString("users.*.verification.{$verificationId}.create", $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString("users.{$id}", $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString("users.{$id}.verification.*", $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString("users.{$id}.verification.*.create", $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString("users.{$id}.verification.{$verificationId}", $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString("users.{$id}.verification.{$verificationId}.create", $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Signature'], $signatureExpected);
$this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Id'] ?? '', $this->getProject()['webhookId']);
$this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Project-Id'] ?? '', $this->getProject()['$id']);
$this->assertEquals(empty($webhook['headers']['X-Appwrite-Webhook-User-Id'] ?? ''), ('server' === $this->getSide()));
$this->assertNotEmpty($webhook['data']['$id']);
$this->assertNotEmpty($webhook['data']['userId']);
$this->assertNotEmpty($webhook['data']['secret']);
$this->assertEquals(true, (new DatetimeValidator())->isValid($webhook['data']['expire']));
$data['secret'] = $webhook['data']['secret'];
return $data;
}
/**
* @depends testCreateAccountVerification
*/
public function testUpdateAccountVerification($data): array
{
$id = $data['id'] ?? '';
$email = $data['email'] ?? '';
$session = $data['session'] ?? '';
$secret = $data['secret'] ?? '';
$verification = $this->client->call(Client::METHOD_PUT, '/account/verification', array_merge([
'origin' => 'http://localhost',
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session,
]), [
'userId' => $id,
'secret' => $secret,
]);
$verificationId = $verification['body']['$id'];
$this->assertEquals(200, $verification['headers']['status-code']);
$this->assertIsArray($verification['body']);
$webhook = $this->getLastRequest();
$signatureKey = $this->getProject()['signatureKey'];
$payload = json_encode($webhook['data']);
$url = $webhook['url'];
$signatureExpected = base64_encode(hash_hmac('sha1', $url . $payload, $signatureKey, true));
$this->assertEquals($webhook['method'], 'POST');
$this->assertEquals($webhook['headers']['Content-Type'], 'application/json');
$this->assertEquals($webhook['headers']['User-Agent'], 'Appwrite-Server vdev. Please report abuse at security@appwrite.io');
$this->assertStringContainsString('users.*', $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString('users.*.verification.*', $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString('users.*.verification.*.update', $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString("users.*.verification.{$verificationId}", $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString("users.*.verification.{$verificationId}.update", $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString("users.{$id}", $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString("users.{$id}.verification.*", $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString("users.{$id}.verification.*.update", $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString("users.{$id}.verification.{$verificationId}", $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertStringContainsString("users.{$id}.verification.{$verificationId}.update", $webhook['headers']['X-Appwrite-Webhook-Events']);
$this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Signature'], $signatureExpected);
$this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Id'] ?? '', $this->getProject()['webhookId']);
$this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Project-Id'] ?? '', $this->getProject()['$id']);
$this->assertEquals(empty($webhook['headers']['X-Appwrite-Webhook-User-Id'] ?? ''), ('server' === $this->getSide()));
$this->assertNotEmpty($webhook['data']['$id']);
$this->assertNotEmpty($webhook['data']['userId']);
$this->assertNotEmpty($webhook['data']['secret']);
$this->assertEquals(true, (new DatetimeValidator())->isValid($webhook['data']['expire']));
$data['secret'] = $webhook['data']['secret'];
return $data;
}
/**
* @depends testCreateTeamMembership
*/