From f07f246f8db2dd4ae3525ed31a1f09de8b879f53 Mon Sep 17 00:00:00 2001 From: Yatharth Verma Date: Fri, 22 Sep 2023 22:53:41 +0530 Subject: [PATCH 01/34] fix conflicts --- app/config/errors.php | 9 +++++++++ app/console | 2 +- app/controllers/api/account.php | 8 ++++++++ src/Appwrite/Extend/Exception.php | 2 ++ 4 files changed, 20 insertions(+), 1 deletion(-) diff --git a/app/config/errors.php b/app/config/errors.php index 575a4588ba..560741b9d3 100644 --- a/app/config/errors.php +++ b/app/config/errors.php @@ -239,6 +239,15 @@ return [ 'name' => Exception::USER_OAUTH2_PROVIDER_ERROR, 'description' => 'OAuth2 provider returned some error.', 'code' => 424, + Exception::USER_EMAIL_ALREADY_VERIFIED => [ + 'name' => Exception::USER_EMAIL_ALREADY_VERIFIED, + 'description' => 'User email is already verified', + 'code' => 400, + ], + Exception::USER_PHONE_ALREADY_VERIFIED => [ + 'name' => Exception::USER_PHONE_ALREADY_VERIFIED, + 'description' => 'User phone is already verified', + 'code' => 400 ], /** Teams */ diff --git a/app/console b/app/console index 9b4bcb8140..88b6d59051 160000 --- a/app/console +++ b/app/console @@ -1 +1 @@ -Subproject commit 9b4bcb8140484669421685b4ba89fa1c4d331360 +Subproject commit 88b6d59051992ed86183ee83d77bf678d1cb73bf diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 87dcd95f03..575a343391 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -2669,6 +2669,10 @@ App::post('/v1/account/verification') if (empty(App::getEnv('_APP_SMTP_HOST'))) { throw new Exception(Exception::GENERAL_SMTP_DISABLED, 'SMTP Disabled'); } + + if($user->getAttribute('emailVerification') == true){ + throw new Exception(Exception::USER_EMAIL_ALREADY_VERIFIED); + } $roles = Authorization::getRoles(); $isPrivilegedUser = Auth::isPrivilegedUser($roles); @@ -2894,6 +2898,10 @@ App::post('/v1/account/verification/phone') if (empty($user->getAttribute('phone'))) { throw new Exception(Exception::USER_PHONE_NOT_FOUND); } + + if($user->getAttribute('phoneVerification') == true){ + throw new Exception(Exception::USER_PHONE_ALREADY_VERIFIED); + } $roles = Authorization::getRoles(); $isPrivilegedUser = Auth::isPrivilegedUser($roles); diff --git a/src/Appwrite/Extend/Exception.php b/src/Appwrite/Extend/Exception.php index 77dc03e310..5f779ff841 100644 --- a/src/Appwrite/Extend/Exception.php +++ b/src/Appwrite/Extend/Exception.php @@ -84,6 +84,8 @@ class Exception extends \Exception public const USER_OAUTH2_BAD_REQUEST = 'user_oauth2_bad_request'; public const USER_OAUTH2_UNAUTHORIZED = 'user_oauth2_unauthorized'; public const USER_OAUTH2_PROVIDER_ERROR = 'user_oauth2_provider_error'; + public const USER_EMAIL_ALREADY_VERIFIED = 'user_email_alread_verified'; + public const USER_PHONE_ALREADY_VERIFIED = 'user_phone_already_verified'; /** Teams */ public const TEAM_NOT_FOUND = 'team_not_found'; From 6b4799912069ff6fed0442120a2147f4b01383c9 Mon Sep 17 00:00:00 2001 From: Yatharth Verma Date: Fri, 22 Sep 2023 22:56:07 +0530 Subject: [PATCH 02/34] fix conflicts --- app/config/errors.php | 4 ++-- app/controllers/api/account.php | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/config/errors.php b/app/config/errors.php index 560741b9d3..192b03e2b0 100644 --- a/app/config/errors.php +++ b/app/config/errors.php @@ -242,12 +242,12 @@ return [ Exception::USER_EMAIL_ALREADY_VERIFIED => [ 'name' => Exception::USER_EMAIL_ALREADY_VERIFIED, 'description' => 'User email is already verified', - 'code' => 400, + 'code' => 409, ], Exception::USER_PHONE_ALREADY_VERIFIED => [ 'name' => Exception::USER_PHONE_ALREADY_VERIFIED, 'description' => 'User phone is already verified', - 'code' => 400 + 'code' => 409 ], /** Teams */ diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 575a343391..9b21ef7d0e 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -2669,8 +2669,8 @@ App::post('/v1/account/verification') if (empty(App::getEnv('_APP_SMTP_HOST'))) { throw new Exception(Exception::GENERAL_SMTP_DISABLED, 'SMTP Disabled'); } - - if($user->getAttribute('emailVerification') == true){ + + if ($user->getAttribute('emailVerification')) { throw new Exception(Exception::USER_EMAIL_ALREADY_VERIFIED); } @@ -2898,8 +2898,8 @@ App::post('/v1/account/verification/phone') if (empty($user->getAttribute('phone'))) { throw new Exception(Exception::USER_PHONE_NOT_FOUND); } - - if($user->getAttribute('phoneVerification') == true){ + + if ($user->getAttribute('phoneVerification')) { throw new Exception(Exception::USER_PHONE_ALREADY_VERIFIED); } From 56a3b3df9927b8e5fa9823023fe22c76c7c6dee1 Mon Sep 17 00:00:00 2001 From: Yatharth Verma Date: Sun, 7 May 2023 03:16:34 +0000 Subject: [PATCH 03/34] Add test cases --- tests/e2e/Services/Account/AccountBase.php | 53 +++++++++++++++++++ .../Account/AccountCustomClientTest.php | 24 +++++++++ 2 files changed, 77 insertions(+) diff --git a/tests/e2e/Services/Account/AccountBase.php b/tests/e2e/Services/Account/AccountBase.php index e64cf2c4ca..57d6d8de83 100644 --- a/tests/e2e/Services/Account/AccountBase.php +++ b/tests/e2e/Services/Account/AccountBase.php @@ -891,6 +891,7 @@ trait AccountBase return $data; } + /** * @depends testCreateAccountVerification */ @@ -945,6 +946,58 @@ trait AccountBase return $data; } + /** + * @depends testUpdateAccountVerification + */ + public function testCreateAccountVerificationForVerifiedEmail($data): array + { + $email = $data['email'] ?? ''; + $name = $data['name'] ?? ''; + $session = $data['session'] ?? ''; + + /** + * Test for SUCCESS + */ + $response = $this->client->call(Client::METHOD_POST, '/account/verification', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, + + ]), [ + 'url' => 'http://localhost/verification', + ]); + + $this->assertEquals(409, $response['headers']['status-code']); + + /** + * Test for FAILURE + */ + $response = $this->client->call(Client::METHOD_POST, '/account/verification', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, + ]), [ + 'url' => 'localhost/verification', + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + + $response = $this->client->call(Client::METHOD_POST, '/account/verification', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, + ]), [ + 'url' => 'http://remotehost/verification', + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + + return $data; + } + /** * @depends testUpdateAccountVerification */ diff --git a/tests/e2e/Services/Account/AccountCustomClientTest.php b/tests/e2e/Services/Account/AccountCustomClientTest.php index 1441ab7f98..ee8978d73a 100644 --- a/tests/e2e/Services/Account/AccountCustomClientTest.php +++ b/tests/e2e/Services/Account/AccountCustomClientTest.php @@ -1068,4 +1068,28 @@ class AccountCustomClientTest extends Scope return $data; } + + /** + * @depends testPhoneVerification + */ + #[Retry(count: 1)] + public function testPhoneVerificationForVerifiedPhone(array $data): array + { + $session = $data['session'] ?? ''; + + /** + * Test for SUCCESS + */ + $response = $this->client->call(Client::METHOD_POST, '/account/verification/phone', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, + + ])); + + $this->assertEquals(409, $response['headers']['status-code']); + + return \array_merge($data); + } } From 5771ad35ed4bf9c9160b2df5e5a75b9d56ae13c7 Mon Sep 17 00:00:00 2001 From: Yatharth Verma Date: Wed, 21 Jun 2023 15:55:39 +0000 Subject: [PATCH 04/34] fix test cases --- tests/e2e/Services/Account/AccountBase.php | 25 ------------------- .../Account/AccountCustomClientTest.php | 2 +- 2 files changed, 1 insertion(+), 26 deletions(-) diff --git a/tests/e2e/Services/Account/AccountBase.php b/tests/e2e/Services/Account/AccountBase.php index 57d6d8de83..e2f56d54a4 100644 --- a/tests/e2e/Services/Account/AccountBase.php +++ b/tests/e2e/Services/Account/AccountBase.php @@ -970,31 +970,6 @@ trait AccountBase $this->assertEquals(409, $response['headers']['status-code']); - /** - * Test for FAILURE - */ - $response = $this->client->call(Client::METHOD_POST, '/account/verification', array_merge([ - 'origin' => 'http://localhost', - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, - ]), [ - 'url' => 'localhost/verification', - ]); - - $this->assertEquals(400, $response['headers']['status-code']); - - $response = $this->client->call(Client::METHOD_POST, '/account/verification', array_merge([ - 'origin' => 'http://localhost', - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, - ]), [ - 'url' => 'http://remotehost/verification', - ]); - - $this->assertEquals(400, $response['headers']['status-code']); - return $data; } diff --git a/tests/e2e/Services/Account/AccountCustomClientTest.php b/tests/e2e/Services/Account/AccountCustomClientTest.php index ee8978d73a..1719d4e049 100644 --- a/tests/e2e/Services/Account/AccountCustomClientTest.php +++ b/tests/e2e/Services/Account/AccountCustomClientTest.php @@ -1090,6 +1090,6 @@ class AccountCustomClientTest extends Scope $this->assertEquals(409, $response['headers']['status-code']); - return \array_merge($data); + return $data; } } From b420c570bc2e76753eaaf540d345b86eafd3c7ef Mon Sep 17 00:00:00 2001 From: Yatharth Verma Date: Sat, 14 Oct 2023 11:51:27 +0530 Subject: [PATCH 05/34] Reverted to last commit before merging main --- app/console | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/console b/app/console index 88b6d59051..9b4bcb8140 160000 --- a/app/console +++ b/app/console @@ -1 +1 @@ -Subproject commit 88b6d59051992ed86183ee83d77bf678d1cb73bf +Subproject commit 9b4bcb8140484669421685b4ba89fa1c4d331360 From 1489f77499727711195ba0404122efbaafa61b90 Mon Sep 17 00:00:00 2001 From: Yatharth Verma Date: Sat, 14 Oct 2023 12:45:29 +0530 Subject: [PATCH 06/34] fix changes suggested by steven and also fixed some bugs came after merging with 1.4.x --- app/config/errors.php | 1 + tests/e2e/Services/Account/AccountBase.php | 2 - .../Account/AccountCustomClientTest.php | 1 - .../Webhooks/WebhooksCustomClientTest.php | 228 +++++++++--------- 4 files changed, 115 insertions(+), 117 deletions(-) diff --git a/app/config/errors.php b/app/config/errors.php index 192b03e2b0..e287b846b1 100644 --- a/app/config/errors.php +++ b/app/config/errors.php @@ -239,6 +239,7 @@ return [ 'name' => Exception::USER_OAUTH2_PROVIDER_ERROR, 'description' => 'OAuth2 provider returned some error.', 'code' => 424, + ], Exception::USER_EMAIL_ALREADY_VERIFIED => [ 'name' => Exception::USER_EMAIL_ALREADY_VERIFIED, 'description' => 'User email is already verified', diff --git a/tests/e2e/Services/Account/AccountBase.php b/tests/e2e/Services/Account/AccountBase.php index e2f56d54a4..953b89ced7 100644 --- a/tests/e2e/Services/Account/AccountBase.php +++ b/tests/e2e/Services/Account/AccountBase.php @@ -891,7 +891,6 @@ trait AccountBase return $data; } - /** * @depends testCreateAccountVerification */ @@ -963,7 +962,6 @@ trait AccountBase 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, - ]), [ 'url' => 'http://localhost/verification', ]); diff --git a/tests/e2e/Services/Account/AccountCustomClientTest.php b/tests/e2e/Services/Account/AccountCustomClientTest.php index 1719d4e049..04322292d8 100644 --- a/tests/e2e/Services/Account/AccountCustomClientTest.php +++ b/tests/e2e/Services/Account/AccountCustomClientTest.php @@ -1085,7 +1085,6 @@ class AccountCustomClientTest extends Scope 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, - ])); $this->assertEquals(409, $response['headers']['status-code']); diff --git a/tests/e2e/Services/Webhooks/WebhooksCustomClientTest.php b/tests/e2e/Services/Webhooks/WebhooksCustomClientTest.php index 67c2dec36b..21c270d199 100644 --- a/tests/e2e/Services/Webhooks/WebhooksCustomClientTest.php +++ b/tests/e2e/Services/Webhooks/WebhooksCustomClientTest.php @@ -636,6 +636,120 @@ class WebhooksCustomClientTest extends Scope return $data; } + /** + * @depends testUpdateAccountPrefs + */ + public function testCreateAccountVerification($data): array + { + $id = $data['id'] ?? ''; + $email = $data['email'] ?? ''; + $session = $data['session'] ?? ''; + + $verification = $this->client->call(Client::METHOD_POST, '/account/verification', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, + ]), [ + 'url' => 'http://localhost/verification', + ]); + + $verificationId = $verification['body']['$id']; + + $this->assertEquals(201, $verification['headers']['status-code']); + $this->assertIsArray($verification['body']); + + $webhook = $this->getLastRequest(); + $signatureKey = $this->getProject()['signatureKey']; + $payload = json_encode($webhook['data']); + $url = $webhook['url']; + $signatureExpected = base64_encode(hash_hmac('sha1', $url . $payload, $signatureKey, true)); + + $this->assertEquals($webhook['method'], 'POST'); + $this->assertEquals($webhook['headers']['Content-Type'], 'application/json'); + $this->assertEquals($webhook['headers']['User-Agent'], 'Appwrite-Server vdev. Please report abuse at security@appwrite.io'); + $this->assertStringContainsString('users.*', $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString('users.*.verification.*', $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString('users.*.verification.*.create', $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString("users.*.verification.{$verificationId}", $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString("users.*.verification.{$verificationId}.create", $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString("users.{$id}", $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString("users.{$id}.verification.*", $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString("users.{$id}.verification.*.create", $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString("users.{$id}.verification.{$verificationId}", $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString("users.{$id}.verification.{$verificationId}.create", $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Signature'], $signatureExpected); + $this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Id'] ?? '', $this->getProject()['webhookId']); + $this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Project-Id'] ?? '', $this->getProject()['$id']); + $this->assertEquals(empty($webhook['headers']['X-Appwrite-Webhook-User-Id'] ?? ''), ('server' === $this->getSide())); + $this->assertNotEmpty($webhook['data']['$id']); + $this->assertNotEmpty($webhook['data']['userId']); + $this->assertNotEmpty($webhook['data']['secret']); + $this->assertEquals(true, (new DatetimeValidator())->isValid($webhook['data']['expire'])); + + $data['secret'] = $webhook['data']['secret']; + + return $data; + } + + /** + * @depends testCreateAccountVerification + */ + public function testUpdateAccountVerification($data): array + { + $id = $data['id'] ?? ''; + $email = $data['email'] ?? ''; + $session = $data['session'] ?? ''; + $secret = $data['secret'] ?? ''; + + $verification = $this->client->call(Client::METHOD_PUT, '/account/verification', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, + ]), [ + 'userId' => $id, + 'secret' => $secret, + ]); + + $verificationId = $verification['body']['$id']; + + $this->assertEquals(200, $verification['headers']['status-code']); + $this->assertIsArray($verification['body']); + + $webhook = $this->getLastRequest(); + $signatureKey = $this->getProject()['signatureKey']; + $payload = json_encode($webhook['data']); + $url = $webhook['url']; + $signatureExpected = base64_encode(hash_hmac('sha1', $url . $payload, $signatureKey, true)); + + $this->assertEquals($webhook['method'], 'POST'); + $this->assertEquals($webhook['headers']['Content-Type'], 'application/json'); + $this->assertEquals($webhook['headers']['User-Agent'], 'Appwrite-Server vdev. Please report abuse at security@appwrite.io'); + $this->assertStringContainsString('users.*', $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString('users.*.verification.*', $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString('users.*.verification.*.update', $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString("users.*.verification.{$verificationId}", $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString("users.*.verification.{$verificationId}.update", $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString("users.{$id}", $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString("users.{$id}.verification.*", $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString("users.{$id}.verification.*.update", $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString("users.{$id}.verification.{$verificationId}", $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertStringContainsString("users.{$id}.verification.{$verificationId}.update", $webhook['headers']['X-Appwrite-Webhook-Events']); + $this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Signature'], $signatureExpected); + $this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Id'] ?? '', $this->getProject()['webhookId']); + $this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Project-Id'] ?? '', $this->getProject()['$id']); + $this->assertEquals(empty($webhook['headers']['X-Appwrite-Webhook-User-Id'] ?? ''), ('server' === $this->getSide())); + $this->assertNotEmpty($webhook['data']['$id']); + $this->assertNotEmpty($webhook['data']['userId']); + $this->assertNotEmpty($webhook['data']['secret']); + $this->assertEquals(true, (new DatetimeValidator())->isValid($webhook['data']['expire'])); + + $data['secret'] = $webhook['data']['secret']; + + return $data; + } + /** * @depends testUpdateAccountPrefs */ @@ -751,120 +865,6 @@ class WebhooksCustomClientTest extends Scope return $data; } - /** - * @depends testUpdateAccountPrefs - */ - public function testCreateAccountVerification($data): array - { - $id = $data['id'] ?? ''; - $email = $data['email'] ?? ''; - $session = $data['session'] ?? ''; - - $verification = $this->client->call(Client::METHOD_POST, '/account/verification', array_merge([ - 'origin' => 'http://localhost', - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, - ]), [ - 'url' => 'http://localhost/verification', - ]); - - $verificationId = $verification['body']['$id']; - - $this->assertEquals(201, $verification['headers']['status-code']); - $this->assertIsArray($verification['body']); - - $webhook = $this->getLastRequest(); - $signatureKey = $this->getProject()['signatureKey']; - $payload = json_encode($webhook['data']); - $url = $webhook['url']; - $signatureExpected = base64_encode(hash_hmac('sha1', $url . $payload, $signatureKey, true)); - - $this->assertEquals($webhook['method'], 'POST'); - $this->assertEquals($webhook['headers']['Content-Type'], 'application/json'); - $this->assertEquals($webhook['headers']['User-Agent'], 'Appwrite-Server vdev. Please report abuse at security@appwrite.io'); - $this->assertStringContainsString('users.*', $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString('users.*.verification.*', $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString('users.*.verification.*.create', $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString("users.*.verification.{$verificationId}", $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString("users.*.verification.{$verificationId}.create", $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString("users.{$id}", $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString("users.{$id}.verification.*", $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString("users.{$id}.verification.*.create", $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString("users.{$id}.verification.{$verificationId}", $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString("users.{$id}.verification.{$verificationId}.create", $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Signature'], $signatureExpected); - $this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Id'] ?? '', $this->getProject()['webhookId']); - $this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Project-Id'] ?? '', $this->getProject()['$id']); - $this->assertEquals(empty($webhook['headers']['X-Appwrite-Webhook-User-Id'] ?? ''), ('server' === $this->getSide())); - $this->assertNotEmpty($webhook['data']['$id']); - $this->assertNotEmpty($webhook['data']['userId']); - $this->assertNotEmpty($webhook['data']['secret']); - $this->assertEquals(true, (new DatetimeValidator())->isValid($webhook['data']['expire'])); - - $data['secret'] = $webhook['data']['secret']; - - return $data; - } - - /** - * @depends testCreateAccountVerification - */ - public function testUpdateAccountVerification($data): array - { - $id = $data['id'] ?? ''; - $email = $data['email'] ?? ''; - $session = $data['session'] ?? ''; - $secret = $data['secret'] ?? ''; - - $verification = $this->client->call(Client::METHOD_PUT, '/account/verification', array_merge([ - 'origin' => 'http://localhost', - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - 'cookie' => 'a_session_' . $this->getProject()['$id'] . '=' . $session, - ]), [ - 'userId' => $id, - 'secret' => $secret, - ]); - - $verificationId = $verification['body']['$id']; - - $this->assertEquals(200, $verification['headers']['status-code']); - $this->assertIsArray($verification['body']); - - $webhook = $this->getLastRequest(); - $signatureKey = $this->getProject()['signatureKey']; - $payload = json_encode($webhook['data']); - $url = $webhook['url']; - $signatureExpected = base64_encode(hash_hmac('sha1', $url . $payload, $signatureKey, true)); - - $this->assertEquals($webhook['method'], 'POST'); - $this->assertEquals($webhook['headers']['Content-Type'], 'application/json'); - $this->assertEquals($webhook['headers']['User-Agent'], 'Appwrite-Server vdev. Please report abuse at security@appwrite.io'); - $this->assertStringContainsString('users.*', $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString('users.*.verification.*', $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString('users.*.verification.*.update', $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString("users.*.verification.{$verificationId}", $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString("users.*.verification.{$verificationId}.update", $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString("users.{$id}", $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString("users.{$id}.verification.*", $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString("users.{$id}.verification.*.update", $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString("users.{$id}.verification.{$verificationId}", $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertStringContainsString("users.{$id}.verification.{$verificationId}.update", $webhook['headers']['X-Appwrite-Webhook-Events']); - $this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Signature'], $signatureExpected); - $this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Id'] ?? '', $this->getProject()['webhookId']); - $this->assertEquals($webhook['headers']['X-Appwrite-Webhook-Project-Id'] ?? '', $this->getProject()['$id']); - $this->assertEquals(empty($webhook['headers']['X-Appwrite-Webhook-User-Id'] ?? ''), ('server' === $this->getSide())); - $this->assertNotEmpty($webhook['data']['$id']); - $this->assertNotEmpty($webhook['data']['userId']); - $this->assertNotEmpty($webhook['data']['secret']); - $this->assertEquals(true, (new DatetimeValidator())->isValid($webhook['data']['expire'])); - - $data['secret'] = $webhook['data']['secret']; - - return $data; - } - /** * @depends testCreateTeamMembership */ From 0ecdeed3298e50590438f0e8a96dd14c38096020 Mon Sep 17 00:00:00 2001 From: Yatharth Verma Date: Sat, 4 Nov 2023 13:01:05 +0530 Subject: [PATCH 07/34] Checkout to latest tag of console submodule --- app/console | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/console b/app/console index 9b4bcb8140..9810ce8581 160000 --- a/app/console +++ b/app/console @@ -1 +1 @@ -Subproject commit 9b4bcb8140484669421685b4ba89fa1c4d331360 +Subproject commit 9810ce85812ca26c95b7d35196848c92e8ba813d From 0f3c43897621e892e1b1dceed6378cf385112803 Mon Sep 17 00:00:00 2001 From: shimon Date: Wed, 8 Nov 2023 12:05:51 +0200 Subject: [PATCH 08/34] added inf metric --- .../Platform/Tasks/DeleteOrphanedProjects.php | 46 ++++++++++++------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php index 2824f4e286..aeaab4b248 100644 --- a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php +++ b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php @@ -13,6 +13,8 @@ use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Pools\Group; use Utopia\Registry\Registry; +use Utopia\Validator\Boolean; +use Utopia\Validator\Hostname; class DeleteOrphanedProjects extends Action { @@ -26,25 +28,23 @@ class DeleteOrphanedProjects extends Action $this ->desc('Get stats for projects') + ->param('commit', false, new boolean(true), 'Commit project deletion', true) ->inject('pools') ->inject('cache') ->inject('dbForConsole') ->inject('register') - ->callback(function (Group $pools, Cache $cache, Database $dbForConsole, Registry $register) { - $this->action($pools, $cache, $dbForConsole, $register); + ->callback(function (bool $commit, Group $pools, Cache $cache, Database $dbForConsole, Registry $register) { + $this->action($commit, $pools, $cache, $dbForConsole, $register); }); } - public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void + public function action(bool $commit, Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void { Console::title('Delete orphaned projects V1'); Console::success(APP_NAME . ' Delete orphaned projects started'); - /** @var array $collections */ - $collectionsConfig = Config::getParam('collections', [])['projects'] ?? []; - /* Initialise new Utopia app */ $app = new App('UTC'); $console = $app->getResource('console'); @@ -54,7 +54,7 @@ class DeleteOrphanedProjects extends Action $totalProjects = $dbForConsole->count('projects'); Console::success("Found a total of: {$totalProjects} projects"); - $orphans = 0; + $orphans = 1; $count = 0; $limit = 30; $sum = 30; @@ -80,17 +80,22 @@ class DeleteOrphanedProjects extends Action $dbForProject->setDefaultDatabase('appwrite'); $dbForProject->setNamespace('_' . $project->getInternalId()); $collectionsCreated = $dbForProject->count(Database::METADATA); - $message = ' (' . $collectionsCreated . ') collections where found on project (' . $project->getId() . '))'; - if ($collectionsCreated < (count($collectionsConfig) + 2)) { - Console::error($message); + if ($collectionsCreated === 0) { + if ($commit === true) { + Console::info('(' . $orphans . ') deleting project (' . $project->getId() . ')'); + $this->deleteProject($dbForConsole, $project->getId()); + } else { + Console::log('(' . $orphans . ') project (' . $project->getId() . ')'); + } $orphans++; - } else { - Console::log($message); } } catch (\Throwable $th) { - //$dbForConsole->deleteDocument('projects', $project->getId()); - //Console::success('Deleting project (' . $project->getId() . ')'); - Console::error(' (0) collections where found for project (' . $project->getId() . ')'); + if ($commit === true) { + Console::info('(' . $orphans . ') deleting project (' . $project->getId() . ')'); + $this->deleteProject($dbForConsole, $project->getId()); + } else { + Console::log('(' . $orphans . ') project (' . $project->getId() . ')'); + } $orphans++; } finally { $pools @@ -110,6 +115,15 @@ class DeleteOrphanedProjects extends Action $count = $count + $sum; } - Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects found ' . $orphans . ' orphans'); + Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects found ' . $orphans - 1 . ' orphans'); + } + + private function deleteProject(Database $dbForConsole, $projectId): void + { + try { + $dbForConsole->deleteDocument('projects', $projectId); + } catch (\Throwable $th) { + Console::error('Error when trying to delete project (' . $projectId . ') ' . $th->getMessage()); + } } } From a029287e5a1cb958e59dde4f06268c304f1ab270 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 13 Nov 2023 14:07:11 +0100 Subject: [PATCH 09/34] Implement health tresholds --- app/controllers/api/health.php | 131 +++++++++++++++--- .../Health/HealthCustomServerTest.php | 93 +++++++++++++ 2 files changed, 204 insertions(+), 20 deletions(-) diff --git a/app/controllers/api/health.php b/app/controllers/api/health.php index e0e26781cf..5d64e71526 100644 --- a/app/controllers/api/health.php +++ b/app/controllers/api/health.php @@ -14,6 +14,7 @@ use Utopia\Registry\Registry; use Utopia\Storage\Device; use Utopia\Storage\Device\Local; use Utopia\Storage\Storage; +use Utopia\Validator\Integer; use Utopia\Validator\Text; App::get('/v1/health') @@ -344,11 +345,20 @@ App::get('/v1/health/queue/webhooks') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->param('treshold', 5000, new Integer(true), 'Queue size treshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (Connection $queue, Response $response) { + ->action(function (int|string $treshold, Connection $queue, Response $response) { + $treshold = \intval($treshold); + $client = new Client(Event::WEBHOOK_QUEUE_NAME, $queue); - $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE); + $size = $client->getQueueSize(); + + if ($size >= $treshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size treshold hit. Current size is {$size} and treshold is {$treshold}."); + } + + $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/logs') @@ -362,11 +372,20 @@ App::get('/v1/health/queue/logs') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->param('treshold', 5000, new Integer(true), 'Queue size treshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (Connection $queue, Response $response) { + ->action(function (int|string $treshold, Connection $queue, Response $response) { + $treshold = \intval($treshold); + $client = new Client(Event::AUDITS_QUEUE_NAME, $queue); - $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE); + $size = $client->getQueueSize(); + + if ($size >= $treshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size treshold hit. Current size is {$size} and treshold is {$treshold}."); + } + + $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/certificates') @@ -380,11 +399,20 @@ App::get('/v1/health/queue/certificates') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->param('treshold', 5000, new Integer(true), 'Queue size treshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (Connection $queue, Response $response) { + ->action(function (int|string $treshold, Connection $queue, Response $response) { + $treshold = \intval($treshold); + $client = new Client(Event::CERTIFICATES_QUEUE_NAME, $queue); - $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE); + $size = $client->getQueueSize(); + + if ($size >= $treshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size treshold hit. Current size is {$size} and treshold is {$treshold}."); + } + + $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/builds') @@ -398,11 +426,20 @@ App::get('/v1/health/queue/builds') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->param('treshold', 5000, new Integer(true), 'Queue size treshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (Connection $queue, Response $response) { + ->action(function (int|string $treshold, Connection $queue, Response $response) { + $treshold = \intval($treshold); + $client = new Client(Event::BUILDS_QUEUE_NAME, $queue); - $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE); + $size = $client->getQueueSize(); + + if ($size >= $treshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size treshold hit. Current size is {$size} and treshold is {$treshold}."); + } + + $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/databases') @@ -417,11 +454,20 @@ App::get('/v1/health/queue/databases') ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) ->param('name', 'database_db_main', new Text(256), 'Queue name for which to check the queue size', true) + ->param('treshold', 5000, new Integer(true), 'Queue size treshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (string $name, Connection $queue, Response $response) { + ->action(function (string $name, int|string $treshold, Connection $queue, Response $response) { + $treshold = \intval($treshold); + $client = new Client($name, $queue); - $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE); + $size = $client->getQueueSize(); + + if ($size >= $treshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size treshold hit. Current size is {$size} and treshold is {$treshold}."); + } + + $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/deletes') @@ -435,11 +481,20 @@ App::get('/v1/health/queue/deletes') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->param('treshold', 5000, new Integer(true), 'Queue size treshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (Connection $queue, Response $response) { + ->action(function (int|string $treshold, Connection $queue, Response $response) { + $treshold = \intval($treshold); + $client = new Client(Event::DELETE_QUEUE_NAME, $queue); - $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE); + $size = $client->getQueueSize(); + + if ($size >= $treshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size treshold hit. Current size is {$size} and treshold is {$treshold}."); + } + + $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/mails') @@ -453,11 +508,20 @@ App::get('/v1/health/queue/mails') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->param('treshold', 5000, new Integer(true), 'Queue size treshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (Connection $queue, Response $response) { + ->action(function (int|string $treshold, Connection $queue, Response $response) { + $treshold = \intval($treshold); + $client = new Client(Event::MAILS_QUEUE_NAME, $queue); - $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE); + $size = $client->getQueueSize(); + + if ($size >= $treshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size treshold hit. Current size is {$size} and treshold is {$treshold}."); + } + + $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/messaging') @@ -471,11 +535,20 @@ App::get('/v1/health/queue/messaging') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->param('treshold', 5000, new Integer(true), 'Queue size treshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (Connection $queue, Response $response) { + ->action(function (int|string $treshold, Connection $queue, Response $response) { + $treshold = \intval($treshold); + $client = new Client(Event::MESSAGING_QUEUE_NAME, $queue); - $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE); + $size = $client->getQueueSize(); + + if ($size >= $treshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size treshold hit. Current size is {$size} and treshold is {$treshold}."); + } + + $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/migrations') @@ -489,11 +562,20 @@ App::get('/v1/health/queue/migrations') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->param('treshold', 5000, new Integer(true), 'Queue size treshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (Connection $queue, Response $response) { + ->action(function (int|string $treshold, Connection $queue, Response $response) { + $treshold = \intval($treshold); + $client = new Client(Event::MIGRATIONS_QUEUE_NAME, $queue); - $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE); + $size = $client->getQueueSize(); + + if ($size >= $treshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size treshold hit. Current size is {$size} and treshold is {$treshold}."); + } + + $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/queue/functions') @@ -507,11 +589,20 @@ App::get('/v1/health/queue/functions') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) + ->param('treshold', 5000, new Integer(true), 'Queue size treshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (Connection $queue, Response $response) { + ->action(function (int|string $treshold, Connection $queue, Response $response) { + $treshold = \intval($treshold); + $client = new Client(Event::FUNCTIONS_QUEUE_NAME, $queue); - $response->dynamic(new Document([ 'size' => $client->getQueueSize() ]), Response::MODEL_HEALTH_QUEUE); + $size = $client->getQueueSize(); + + if ($size >= $treshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size treshold hit. Current size is {$size} and treshold is {$treshold}."); + } + + $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); }, ['response']); App::get('/v1/health/storage/local') diff --git a/tests/e2e/Services/Health/HealthCustomServerTest.php b/tests/e2e/Services/Health/HealthCustomServerTest.php index eafc961e8b..9a178ef1fa 100644 --- a/tests/e2e/Services/Health/HealthCustomServerTest.php +++ b/tests/e2e/Services/Health/HealthCustomServerTest.php @@ -138,6 +138,15 @@ class HealthCustomServerTest extends Scope $this->assertIsInt($response['body']['size']); $this->assertLessThan(100, $response['body']['size']); + /** + * Test for FAILURE + */ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/webhooks?treshold=0', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + $this->assertEquals(500, $response['headers']['status-code']); + return []; } @@ -155,6 +164,15 @@ class HealthCustomServerTest extends Scope $this->assertIsInt($response['body']['size']); $this->assertLessThan(100, $response['body']['size']); + /** + * Test for FAILURE + */ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/logs?treshold=0', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + $this->assertEquals(500, $response['headers']['status-code']); + return []; } @@ -172,6 +190,15 @@ class HealthCustomServerTest extends Scope $this->assertIsInt($response['body']['size']); $this->assertLessThan(100, $response['body']['size']); + /** + * Test for FAILURE + */ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/certificates?treshold=0', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + $this->assertEquals(500, $response['headers']['status-code']); + return []; } @@ -189,6 +216,15 @@ class HealthCustomServerTest extends Scope $this->assertIsInt($response['body']['size']); $this->assertLessThan(100, $response['body']['size']); + /** + * Test for FAILURE + */ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/functions?treshold=0', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + $this->assertEquals(500, $response['headers']['status-code']); + return []; } @@ -206,6 +242,15 @@ class HealthCustomServerTest extends Scope $this->assertIsInt($response['body']['size']); $this->assertLessThan(100, $response['body']['size']); + /** + * Test for FAILURE + */ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/builds?treshold=0', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + $this->assertEquals(500, $response['headers']['status-code']); + return []; } @@ -225,6 +270,18 @@ class HealthCustomServerTest extends Scope $this->assertIsInt($response['body']['size']); $this->assertLessThan(100, $response['body']['size']); + /** + * Test for FAILURE + */ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/databases', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'name' => 'database_db_main', + 'treshold' => '0' + ]); + $this->assertEquals(500, $response['headers']['status-code']); + return []; } @@ -242,6 +299,15 @@ class HealthCustomServerTest extends Scope $this->assertIsInt($response['body']['size']); $this->assertLessThan(100, $response['body']['size']); + /** + * Test for FAILURE + */ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/deletes?treshold=0', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + $this->assertEquals(500, $response['headers']['status-code']); + return []; } @@ -259,6 +325,15 @@ class HealthCustomServerTest extends Scope $this->assertIsInt($response['body']['size']); $this->assertLessThan(100, $response['body']['size']); + /** + * Test for FAILURE + */ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/mails?treshold=0', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + $this->assertEquals(500, $response['headers']['status-code']); + return []; } @@ -276,6 +351,15 @@ class HealthCustomServerTest extends Scope $this->assertIsInt($response['body']['size']); $this->assertLessThan(100, $response['body']['size']); + /** + * Test for FAILURE + */ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/messaging?treshold=0', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + $this->assertEquals(500, $response['headers']['status-code']); + return []; } @@ -293,6 +377,15 @@ class HealthCustomServerTest extends Scope $this->assertIsInt($response['body']['size']); $this->assertLessThan(100, $response['body']['size']); + /** + * Test for FAILURE + */ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/migrations?treshold=0', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), []); + $this->assertEquals(500, $response['headers']['status-code']); + return []; } From 967854d34af107b4b3c97d6360bf367150f61dc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 13 Nov 2023 14:39:33 +0100 Subject: [PATCH 10/34] Fix grammar --- app/controllers/api/health.php | 100 +++++++++--------- .../Health/HealthCustomServerTest.php | 20 ++-- 2 files changed, 60 insertions(+), 60 deletions(-) diff --git a/app/controllers/api/health.php b/app/controllers/api/health.php index 5d64e71526..90e080d5fa 100644 --- a/app/controllers/api/health.php +++ b/app/controllers/api/health.php @@ -345,17 +345,17 @@ App::get('/v1/health/queue/webhooks') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) - ->param('treshold', 5000, new Integer(true), 'Queue size treshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) + ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (int|string $treshold, Connection $queue, Response $response) { - $treshold = \intval($treshold); + ->action(function (int|string $threshold, Connection $queue, Response $response) { + $threshold = \intval($threshold); $client = new Client(Event::WEBHOOK_QUEUE_NAME, $queue); $size = $client->getQueueSize(); - if ($size >= $treshold) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size treshold hit. Current size is {$size} and treshold is {$treshold}."); + if ($size >= $threshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -372,17 +372,17 @@ App::get('/v1/health/queue/logs') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) - ->param('treshold', 5000, new Integer(true), 'Queue size treshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) + ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (int|string $treshold, Connection $queue, Response $response) { - $treshold = \intval($treshold); + ->action(function (int|string $threshold, Connection $queue, Response $response) { + $threshold = \intval($threshold); $client = new Client(Event::AUDITS_QUEUE_NAME, $queue); $size = $client->getQueueSize(); - if ($size >= $treshold) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size treshold hit. Current size is {$size} and treshold is {$treshold}."); + if ($size >= $threshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -399,17 +399,17 @@ App::get('/v1/health/queue/certificates') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) - ->param('treshold', 5000, new Integer(true), 'Queue size treshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) + ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (int|string $treshold, Connection $queue, Response $response) { - $treshold = \intval($treshold); + ->action(function (int|string $threshold, Connection $queue, Response $response) { + $threshold = \intval($threshold); $client = new Client(Event::CERTIFICATES_QUEUE_NAME, $queue); $size = $client->getQueueSize(); - if ($size >= $treshold) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size treshold hit. Current size is {$size} and treshold is {$treshold}."); + if ($size >= $threshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -426,17 +426,17 @@ App::get('/v1/health/queue/builds') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) - ->param('treshold', 5000, new Integer(true), 'Queue size treshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) + ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (int|string $treshold, Connection $queue, Response $response) { - $treshold = \intval($treshold); + ->action(function (int|string $threshold, Connection $queue, Response $response) { + $threshold = \intval($threshold); $client = new Client(Event::BUILDS_QUEUE_NAME, $queue); $size = $client->getQueueSize(); - if ($size >= $treshold) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size treshold hit. Current size is {$size} and treshold is {$treshold}."); + if ($size >= $threshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -454,17 +454,17 @@ App::get('/v1/health/queue/databases') ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) ->param('name', 'database_db_main', new Text(256), 'Queue name for which to check the queue size', true) - ->param('treshold', 5000, new Integer(true), 'Queue size treshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) + ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (string $name, int|string $treshold, Connection $queue, Response $response) { - $treshold = \intval($treshold); + ->action(function (string $name, int|string $threshold, Connection $queue, Response $response) { + $threshold = \intval($threshold); $client = new Client($name, $queue); $size = $client->getQueueSize(); - if ($size >= $treshold) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size treshold hit. Current size is {$size} and treshold is {$treshold}."); + if ($size >= $threshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -481,17 +481,17 @@ App::get('/v1/health/queue/deletes') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) - ->param('treshold', 5000, new Integer(true), 'Queue size treshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) + ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (int|string $treshold, Connection $queue, Response $response) { - $treshold = \intval($treshold); + ->action(function (int|string $threshold, Connection $queue, Response $response) { + $threshold = \intval($threshold); $client = new Client(Event::DELETE_QUEUE_NAME, $queue); $size = $client->getQueueSize(); - if ($size >= $treshold) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size treshold hit. Current size is {$size} and treshold is {$treshold}."); + if ($size >= $threshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -508,17 +508,17 @@ App::get('/v1/health/queue/mails') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) - ->param('treshold', 5000, new Integer(true), 'Queue size treshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) + ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (int|string $treshold, Connection $queue, Response $response) { - $treshold = \intval($treshold); + ->action(function (int|string $threshold, Connection $queue, Response $response) { + $threshold = \intval($threshold); $client = new Client(Event::MAILS_QUEUE_NAME, $queue); $size = $client->getQueueSize(); - if ($size >= $treshold) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size treshold hit. Current size is {$size} and treshold is {$treshold}."); + if ($size >= $threshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -535,17 +535,17 @@ App::get('/v1/health/queue/messaging') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) - ->param('treshold', 5000, new Integer(true), 'Queue size treshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) + ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (int|string $treshold, Connection $queue, Response $response) { - $treshold = \intval($treshold); + ->action(function (int|string $threshold, Connection $queue, Response $response) { + $threshold = \intval($threshold); $client = new Client(Event::MESSAGING_QUEUE_NAME, $queue); $size = $client->getQueueSize(); - if ($size >= $treshold) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size treshold hit. Current size is {$size} and treshold is {$treshold}."); + if ($size >= $threshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -562,17 +562,17 @@ App::get('/v1/health/queue/migrations') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) - ->param('treshold', 5000, new Integer(true), 'Queue size treshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) + ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (int|string $treshold, Connection $queue, Response $response) { - $treshold = \intval($treshold); + ->action(function (int|string $threshold, Connection $queue, Response $response) { + $threshold = \intval($threshold); $client = new Client(Event::MIGRATIONS_QUEUE_NAME, $queue); $size = $client->getQueueSize(); - if ($size >= $treshold) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size treshold hit. Current size is {$size} and treshold is {$treshold}."); + if ($size >= $threshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); @@ -589,17 +589,17 @@ App::get('/v1/health/queue/functions') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_HEALTH_QUEUE) - ->param('treshold', 5000, new Integer(true), 'Queue size treshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) + ->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true) ->inject('queue') ->inject('response') - ->action(function (int|string $treshold, Connection $queue, Response $response) { - $treshold = \intval($treshold); + ->action(function (int|string $threshold, Connection $queue, Response $response) { + $threshold = \intval($threshold); $client = new Client(Event::FUNCTIONS_QUEUE_NAME, $queue); $size = $client->getQueueSize(); - if ($size >= $treshold) { - throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size treshold hit. Current size is {$size} and treshold is {$treshold}."); + if ($size >= $threshold) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}."); } $response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE); diff --git a/tests/e2e/Services/Health/HealthCustomServerTest.php b/tests/e2e/Services/Health/HealthCustomServerTest.php index 9a178ef1fa..8fa9faadd2 100644 --- a/tests/e2e/Services/Health/HealthCustomServerTest.php +++ b/tests/e2e/Services/Health/HealthCustomServerTest.php @@ -141,7 +141,7 @@ class HealthCustomServerTest extends Scope /** * Test for FAILURE */ - $response = $this->client->call(Client::METHOD_GET, '/health/queue/webhooks?treshold=0', array_merge([ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/webhooks?threshold=0', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), []); @@ -167,7 +167,7 @@ class HealthCustomServerTest extends Scope /** * Test for FAILURE */ - $response = $this->client->call(Client::METHOD_GET, '/health/queue/logs?treshold=0', array_merge([ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/logs?threshold=0', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), []); @@ -193,7 +193,7 @@ class HealthCustomServerTest extends Scope /** * Test for FAILURE */ - $response = $this->client->call(Client::METHOD_GET, '/health/queue/certificates?treshold=0', array_merge([ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/certificates?threshold=0', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), []); @@ -219,7 +219,7 @@ class HealthCustomServerTest extends Scope /** * Test for FAILURE */ - $response = $this->client->call(Client::METHOD_GET, '/health/queue/functions?treshold=0', array_merge([ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/functions?threshold=0', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), []); @@ -245,7 +245,7 @@ class HealthCustomServerTest extends Scope /** * Test for FAILURE */ - $response = $this->client->call(Client::METHOD_GET, '/health/queue/builds?treshold=0', array_merge([ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/builds?threshold=0', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), []); @@ -278,7 +278,7 @@ class HealthCustomServerTest extends Scope 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ 'name' => 'database_db_main', - 'treshold' => '0' + 'threshold' => '0' ]); $this->assertEquals(500, $response['headers']['status-code']); @@ -302,7 +302,7 @@ class HealthCustomServerTest extends Scope /** * Test for FAILURE */ - $response = $this->client->call(Client::METHOD_GET, '/health/queue/deletes?treshold=0', array_merge([ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/deletes?threshold=0', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), []); @@ -328,7 +328,7 @@ class HealthCustomServerTest extends Scope /** * Test for FAILURE */ - $response = $this->client->call(Client::METHOD_GET, '/health/queue/mails?treshold=0', array_merge([ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/mails?threshold=0', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), []); @@ -354,7 +354,7 @@ class HealthCustomServerTest extends Scope /** * Test for FAILURE */ - $response = $this->client->call(Client::METHOD_GET, '/health/queue/messaging?treshold=0', array_merge([ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/messaging?threshold=0', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), []); @@ -380,7 +380,7 @@ class HealthCustomServerTest extends Scope /** * Test for FAILURE */ - $response = $this->client->call(Client::METHOD_GET, '/health/queue/migrations?treshold=0', array_merge([ + $response = $this->client->call(Client::METHOD_GET, '/health/queue/migrations?threshold=0', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), []); From b9007a59564f38b54cf90ba0d88aa62060d62fff Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 13 Nov 2023 15:41:10 +0200 Subject: [PATCH 11/34] wrapping create stats query with Authorization::skip --- src/Appwrite/Usage/Calculators/TimeSeries.php | 40 ++++++++++--------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/src/Appwrite/Usage/Calculators/TimeSeries.php b/src/Appwrite/Usage/Calculators/TimeSeries.php index e0a12b443f..9ead48e93f 100644 --- a/src/Appwrite/Usage/Calculators/TimeSeries.php +++ b/src/Appwrite/Usage/Calculators/TimeSeries.php @@ -8,6 +8,7 @@ use Utopia\Database\Database; use Utopia\Database\Document; use InfluxDB\Database as InfluxDatabase; use DateTime; +use Utopia\Database\Validator\Authorization; use Utopia\Registry\Registry; class TimeSeries extends Calculator @@ -422,14 +423,16 @@ class TimeSeries extends Calculator */ private function createOrUpdateMetric(string $projectId, string $time, string $period, string $metric, int $value, int $type): void { + $id = \md5("{$time}_{$period}_{$metric}"); $project = $this->database->getDocument('projects', $projectId); $database = call_user_func($this->getProjectDB, $project); - try { - $document = $database->getDocument('stats', $id); - if ($document->isEmpty()) { - $database->createDocument('stats', new Document([ + Authorization::skip(function () use ($database, $id, $period, $time, $metric, $value, $type, $projectId) { + try { + $document = $database->getDocument('stats', $id); + if ($document->isEmpty()) { + $database->createDocument('stats', new Document([ '$id' => $id, 'period' => $period, 'time' => $time, @@ -437,21 +440,22 @@ class TimeSeries extends Calculator 'value' => $value, 'type' => $type, 'region' => $this->region, - ])); - } else { - $database->updateDocument( - 'stats', - $document->getId(), - $document->setAttribute('value', $value) - ); + ])); + } else { + $database->updateDocument( + 'stats', + $document->getId(), + $document->setAttribute('value', $value) + ); + } + } catch (\Exception $e) { // if projects are deleted this might fail + if (is_callable($this->errorHandler)) { + call_user_func($this->errorHandler, $e, "sync_project_{$projectId}_metric_{$metric}"); + } else { + throw $e; + } } - } catch (\Exception $e) { // if projects are deleted this might fail - if (is_callable($this->errorHandler)) { - call_user_func($this->errorHandler, $e, "sync_project_{$projectId}_metric_{$metric}"); - } else { - throw $e; - } - } + }); $this->register->get('pools')->reclaim(); } From dd0bf66212450041ab7518a0f958c8fdbb9c8166 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 13 Nov 2023 19:24:55 +0200 Subject: [PATCH 12/34] delete orphaned projects task --- .../Platform/Tasks/DeleteOrphanedProjects.php | 67 +++++++++++-------- 1 file changed, 39 insertions(+), 28 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php index aeaab4b248..2af9c54a6e 100644 --- a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php +++ b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php @@ -2,11 +2,9 @@ namespace Appwrite\Platform\Tasks; -use PHPMailer\PHPMailer\PHPMailer; use Utopia\App; use Utopia\Config\Config; use Utopia\Database\Query; -use Utopia\Database\Validator\Authorization; use Utopia\Platform\Action; use Utopia\Cache\Cache; use Utopia\CLI\Console; @@ -14,7 +12,6 @@ use Utopia\Database\Database; use Utopia\Pools\Group; use Utopia\Registry\Registry; use Utopia\Validator\Boolean; -use Utopia\Validator\Hostname; class DeleteOrphanedProjects extends Action { @@ -27,7 +24,7 @@ class DeleteOrphanedProjects extends Action { $this - ->desc('Get stats for projects') + ->desc('Delete orphaned projects') ->param('commit', false, new boolean(true), 'Commit project deletion', true) ->inject('pools') ->inject('cache') @@ -45,6 +42,9 @@ class DeleteOrphanedProjects extends Action Console::title('Delete orphaned projects V1'); Console::success(APP_NAME . ' Delete orphaned projects started'); + /** @var array $collections */ + $collectionsConfig = Config::getParam('collections', [])['projects'] ?? []; + /* Initialise new Utopia app */ $app = new App('UTC'); $console = $app->getResource('console'); @@ -55,6 +55,7 @@ class DeleteOrphanedProjects extends Action Console::success("Found a total of: {$totalProjects} projects"); $orphans = 1; + $cnt = 0; $count = 0; $limit = 30; $sum = 30; @@ -79,24 +80,43 @@ class DeleteOrphanedProjects extends Action $dbForProject = new Database($adapter, $cache); $dbForProject->setDefaultDatabase('appwrite'); $dbForProject->setNamespace('_' . $project->getInternalId()); - $collectionsCreated = $dbForProject->count(Database::METADATA); - if ($collectionsCreated === 0) { - if ($commit === true) { - Console::info('(' . $orphans . ') deleting project (' . $project->getId() . ')'); - $this->deleteProject($dbForConsole, $project->getId()); - } else { - Console::log('(' . $orphans . ') project (' . $project->getId() . ')'); + $collectionsCreated = 0; + $cnt++; + if ($dbForProject->exists($dbForProject->getDefaultDatabase(), Database::METADATA)) { + $collectionsCreated = $dbForProject->count(Database::METADATA); + } + + $msg = '(' . $cnt . ') ignoring found (' . $collectionsCreated . ') collections on project (' . $project->getInternalId() . ') , database (' . $project['database'] . ')'; + /** + * +2 == audit+abuse + */ + if ($collectionsCreated === (count($collectionsConfig) + 2)) { + Console::log($msg . ' ignoring....'); + continue; + } + + Console::log($msg); + + if ($collectionsCreated > 0) { + $collections = $dbForProject->find(Database::METADATA, []); + foreach ($collections as $collection) { + if ($commit) { + $dbForProject->deleteCollection($collection->getId()); + $dbForConsole->deleteCachedCollection($collection->getId()); + } + Console::info('--Deleting collection (' . $collection->getId() . ') project no (' . $project->getInternalId() . ')'); } + } + if ($commit) { + $dbForConsole->deleteDocument('projects', $project->getId()); + $dbForConsole->deleteCachedDocument('projects', $project->getId()); + } + + Console::info('--Deleting project no (' . $project->getInternalId() . ')'); + $orphans++; - } } catch (\Throwable $th) { - if ($commit === true) { - Console::info('(' . $orphans . ') deleting project (' . $project->getId() . ')'); - $this->deleteProject($dbForConsole, $project->getId()); - } else { - Console::log('(' . $orphans . ') project (' . $project->getId() . ')'); - } - $orphans++; + Console::error('Error: ' . $th->getMessage()); } finally { $pools ->get($db) @@ -117,13 +137,4 @@ class DeleteOrphanedProjects extends Action Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects found ' . $orphans - 1 . ' orphans'); } - - private function deleteProject(Database $dbForConsole, $projectId): void - { - try { - $dbForConsole->deleteDocument('projects', $projectId); - } catch (\Throwable $th) { - Console::error('Error when trying to delete project (' . $projectId . ') ' . $th->getMessage()); - } - } } From 60c0f4c97393d8d665da5ab7d7b697fafa69abbc Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 13 Nov 2023 19:27:20 +0200 Subject: [PATCH 13/34] delete orphaned projects task --- src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php index 2af9c54a6e..2da3c91384 100644 --- a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php +++ b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php @@ -86,9 +86,9 @@ class DeleteOrphanedProjects extends Action $collectionsCreated = $dbForProject->count(Database::METADATA); } - $msg = '(' . $cnt . ') ignoring found (' . $collectionsCreated . ') collections on project (' . $project->getInternalId() . ') , database (' . $project['database'] . ')'; + $msg = '(' . $cnt . ') found (' . $collectionsCreated . ') collections on project (' . $project->getInternalId() . ') , database (' . $project['database'] . ')'; /** - * +2 == audit+abuse + * +2 = audit+abuse */ if ($collectionsCreated === (count($collectionsConfig) + 2)) { Console::log($msg . ' ignoring....'); From 3d9ee8bc521367d8242de2e4035970d44a9050da Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 13 Nov 2023 19:28:48 +0200 Subject: [PATCH 14/34] delete orphaned projects task --- src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php index 2da3c91384..ee9931bde4 100644 --- a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php +++ b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php @@ -25,7 +25,7 @@ class DeleteOrphanedProjects extends Action $this ->desc('Delete orphaned projects') - ->param('commit', false, new boolean(true), 'Commit project deletion', true) + ->param('commit', false, new Boolean(true), 'Commit project deletion', true) ->inject('pools') ->inject('cache') ->inject('dbForConsole') From be85b173e774c0f85ea95bc651b0829648c5859e Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Tue, 14 Nov 2023 16:49:16 +0530 Subject: [PATCH 15/34] Only delete repositories linked to the particular project --- src/Appwrite/Platform/Workers/Deletes.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 6bb8636695..2246816436 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -730,6 +730,7 @@ class Deletes extends Action */ Console::info("Deleting VCS repositories and comments linked to function " . $functionId); $this->deleteByGroup('repositories', [ + Query::equal('projectInternalId', [$projectInternalId]), Query::equal('resourceInternalId', [$functionInternalId]), Query::equal('resourceType', ['function']), ], $dbForConsole, function (Document $document) use ($dbForConsole) { From 78eb5105e7ffc8ea4a204bf25c08e9c7e87c1884 Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Tue, 14 Nov 2023 17:08:09 +0530 Subject: [PATCH 16/34] Update projectInternalId var --- src/Appwrite/Platform/Workers/Deletes.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 2246816436..4b888b9fd6 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -730,7 +730,7 @@ class Deletes extends Action */ Console::info("Deleting VCS repositories and comments linked to function " . $functionId); $this->deleteByGroup('repositories', [ - Query::equal('projectInternalId', [$projectInternalId]), + Query::equal('projectInternalId', [$project->getInternalId()]), Query::equal('resourceInternalId', [$functionInternalId]), Query::equal('resourceType', ['function']), ], $dbForConsole, function (Document $document) use ($dbForConsole) { From 4f2f76db2239510fe833828daefc9e628ddc2a08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 14 Nov 2023 14:11:54 +0100 Subject: [PATCH 17/34] Improve deletion relation with IDs --- src/Appwrite/Platform/Workers/Deletes.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 4b888b9fd6..b95a13a12e 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -735,10 +735,10 @@ class Deletes extends Action Query::equal('resourceType', ['function']), ], $dbForConsole, function (Document $document) use ($dbForConsole) { $providerRepositoryId = $document->getAttribute('providerRepositoryId', ''); - $projectId = $document->getAttribute('projectId', ''); + $projectInternalId = $document->getAttribute('projectInternalId', ''); $this->deleteByGroup('vcsComments', [ Query::equal('providerRepositoryId', [$providerRepositoryId]), - Query::equal('projectId', [$projectId]), + Query::equal('projectInternalId', [$projectInternalId]), ], $dbForConsole); }); From fb6455b783c554536aac864753301b840708e432 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 14 Nov 2023 14:43:33 +0100 Subject: [PATCH 18/34] Add script to patch missing repos documents --- Dockerfile | 1 + bin/patch-recreate-repositories-documents | 3 + .../patchRecreateRepositoriesDocuments.php | 141 ++++++++++++++++++ 3 files changed, 145 insertions(+) create mode 100644 bin/patch-recreate-repositories-documents create mode 100644 src/Appwrite/Platform/Tasks/patchRecreateRepositoriesDocuments.php diff --git a/Dockerfile b/Dockerfile index 33b1434659..059c499bd9 100755 --- a/Dockerfile +++ b/Dockerfile @@ -100,6 +100,7 @@ RUN chmod +x /usr/local/bin/doctor && \ RUN chmod +x /usr/local/bin/hamster && \ chmod +x /usr/local/bin/volume-sync && \ chmod +x /usr/local/bin/patch-delete-schedule-updated-at-attribute && \ + chmod +x /usr/local/bin/patch-recreate-repositories-documents && \ chmod +x /usr/local/bin/patch-delete-project-collections && \ chmod +x /usr/local/bin/delete-orphaned-projects && \ chmod +x /usr/local/bin/clear-card-cache && \ diff --git a/bin/patch-recreate-repositories-documents b/bin/patch-recreate-repositories-documents new file mode 100644 index 0000000000..8c6c4157f4 --- /dev/null +++ b/bin/patch-recreate-repositories-documents @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php patch-recreate-repositories-documents $@ \ No newline at end of file diff --git a/src/Appwrite/Platform/Tasks/patchRecreateRepositoriesDocuments.php b/src/Appwrite/Platform/Tasks/patchRecreateRepositoriesDocuments.php new file mode 100644 index 0000000000..5749380d0a --- /dev/null +++ b/src/Appwrite/Platform/Tasks/patchRecreateRepositoriesDocuments.php @@ -0,0 +1,141 @@ +desc('Recreate missing repositories in consoleDB from projectDBs. They can be missing if you used Appwrite 1.4.10 or 1.4.11, and deleted a function.') + ->param('after', '', new Text(36), 'After cursor', true) + ->param('projectId', '', new Text(36), 'Select project to validate', true) + ->inject('dbForConsole') + ->inject('getProjectDB') + ->callback(fn ($after, $projectId, $dbForConsole, $getProjectDB) => $this->action($after, $projectId, $dbForConsole, $getProjectDB)); + } + + public function action($after, $projectId, Database $dbForConsole, callable $getProjectDB): void + { + Console::info("Starting the patch"); + + $startTime = microtime(true); + + if(!empty($projectId)) { + $project = $dbForConsole->getDocument('projects', $projectId); + $dbForProject = call_user_func($getProjectDB, $project); + $this->recreateRepositories($dbForConsole, $dbForProject, $project); + } else { + $queries = []; + if(!empty($after)) { + Console::info("Iterating remaining projects after project with ID {$after}"); + $project = $dbForConsole->getDocument('projects', $after); + $queries = [Query::cursorAfter($project)]; + } else { + Console::info("Iterating all projects"); + } + $this->foreachDocument($dbForConsole, 'projects', $queries, function(Document $project) use($getProjectDB, $dbForConsole){ + $dbForProject = call_user_func($getProjectDB, $project); + $this->recreateRepositories($dbForConsole, $dbForProject, $project); + }); + } + + $endTime = microtime(true); + $timeTaken = $endTime - $startTime; + + $hours = (int)($timeTaken / 3600); + $timeTaken -= $hours * 3600; + $minutes = (int)($timeTaken / 60); + $timeTaken -= $minutes * 60; + $seconds = (int)$timeTaken; + $milliseconds = ($timeTaken - $seconds) * 1000; + Console::info("Recreate patch completed in $hours h, $minutes m, $seconds s, $milliseconds mis ( total $timeTaken milliseconds)"); + } + + protected function foreachDocument(Database $database, string $collection, array $queries = [], callable $callback = null): void + { + $limit = 1000; + $results = []; + $sum = $limit; + $latestDocument = null; + + while ($sum === $limit) { + $newQueries = $queries; + + if ($latestDocument != null) { + array_unshift($newQueries, Query::cursorAfter($latestDocument)); + } + $newQueries[] = Query::limit($limit); + $results = $database->find($collection, $newQueries); + + if (empty($results)) { + return; + } + + $sum = count($results); + + foreach ($results as $document) { + if (is_callable($callback)) { + $callback($document); + } + } + $latestDocument = $results[array_key_last($results)]; + } + } + + public function recreateRepositories(Database $dbForConsole, Database $dbForProject, Document $project): void + { + $projectId = $project->getId(); + Console::log("Running patch for project {$projectId}"); + + $this->foreachDocument($dbForProject, 'functions', [], function(Document $function) use ($dbForConsole, $project) { + $isConnected = !empty($function->getAttribute('providerRepositoryId', '')); + + if($isConnected) { + $repository = $dbForConsole->getDocument('repositories', $function->getAttribute('repositoryId', '')); + + if($repository->isEmpty()) { + $projectId = $project->getId(); + $functionId = $function->getId(); + Console::success("Recreating repositories document for project ID {$projectId}, function ID {$functionId}"); + + $repository = $dbForConsole->createDocument('repositories', new Document([ + '$id' => ID::unique(), + '$permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + 'installationId' => $function->getAttribute('installationId', ''), + 'installationInternalId' => $function->getAttribute('installationInternalId', ''), + 'projectId' => $project->getId(), + 'projectInternalId' => $project->getInternalId(), + 'providerRepositoryId' => $function->getAttribute('providerRepositoryId', ''), + 'resourceId' => $function->getId(), + 'resourceInternalId' => $function->getInternalId(), + 'resourceType' => 'function', + 'providerPullRequestIds' => [] + ])); + } + } + }); + } +} From c133bccfa93b52cca8dd9e0210f4d2ee8fc2d921 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 14 Nov 2023 15:06:58 +0100 Subject: [PATCH 19/34] Finish recreate repos docs script --- .env | 38 ++++++++++++++++--- src/Appwrite/Platform/Services/Tasks.php | 2 + .../patchRecreateRepositoriesDocuments.php | 32 ++++++++++------ 3 files changed, 55 insertions(+), 17 deletions(-) diff --git a/.env b/.env index ad551e705a..759ddcc051 100644 --- a/.env +++ b/.env @@ -90,12 +90,38 @@ _APP_GRAPHQL_MAX_COMPLEXITY=250 _APP_GRAPHQL_MAX_DEPTH=3 _APP_DOCKER_HUB_USERNAME= _APP_DOCKER_HUB_PASSWORD= -_APP_VCS_GITHUB_APP_NAME= -_APP_VCS_GITHUB_PRIVATE_KEY=disabled -_APP_VCS_GITHUB_APP_ID= -_APP_VCS_GITHUB_CLIENT_ID= -_APP_VCS_GITHUB_CLIENT_SECRET= -_APP_VCS_GITHUB_WEBHOOK_SECRET= +_APP_VCS_GITHUB_APP_NAME=appwrite-generated-on-22-5-2023 +_APP_VCS_GITHUB_PRIVATE_KEY="-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEApUP3k7ratjenwOA1AtnmvZbNIm9Mhu181ryrnb7M1z1/OLUo +jZ1iS8u1K/GQtoqCNI8RR38rBXW1hzfC34qAL3XrJr4caK+AKzV7B4y9e2Zn+VlF +ZZ2rknNPlRLCwll6V60M7suqLONUADs6bvEs8GIR926gwNi8SmomtIj8k8UZFW00 +9L0OM0pvajJKASPynZ7jXfxic5Yu060BQq+IAtOkOaxyBF1xiGQRBuAt2movTI2X +4PwGcSCy7YD6v2DRgHIoYehtdWx0DrbRgCXsmwMin5BR/7ZhBSJE7YVH9vSqWThe +V8VmLKPiZZC5ZW2Vf28kM/FlkCgjfd0tNXs8lQIDAQABAoIBAQCWaFYhODSnE93z +ttnoH2JVd7J4PW0be3ZbhNh3t1d8KPbpKE6hG/SC4QGg3bgDuekoZnCmbkE8NdWh +G4maotVo3FvIJct7JwZxzLmMtHUaoqfMEogLJEUrAxERrkJcWMz0kIUtq2PUeIxR +rZXPtGVe3RJW63MYL+ilnRhexDGDVa5I50jFCTT5GXXHl6tw6tei1rDc2kI7pb3k +BGAxsRGXUK6Di9EmR2Z8xFHvM3LeuSWR2WalR8+abn661P3J+InMqh0s+TBB86kH +6NUxoWKSCXjPfMWj15M0Wwm4CBlzgc6GS106hmHPi3YAQLCV4x0l4qmq+bdsQOnN +VAOnPAohAoGBANidEYsMm5jaTklFnIExBkm5OHcKRuMgplMoEsVE/dglBPCJB3Uf +JUmhMRVazztDGD0ciImypt4j9klR4zqSQv4Aa8OdwW/jcuNPm3qQrdSm86iP2Lbe +V9DNKK0vQ/3srwIYxl9qabOLaQrKAeiPxfUuL91iyDIYtsPPIgfvVGfdAoGBAMNQ +v5wruODDgf0mMm1nA9LNHlkMi8uaVvxFAhjWtmPOH4FHXgzMDGC6syvL+d1XIdtF +tA/j/f/A1zFsYzeZRVBqVmpd8rvRzFTaRrBgLjI/vxmbJ7syr9rT7iZYvFYUv3c+ +mr8m5AIGULiGmMYnSWttIi2prlA17FC5Qp/lq/gZAoGAVrBlePSOwNl9Qy2suLda +AN8zjdB7FiLW7ai3+mLmBD6sf2cXqPPSBGmSLy2sidcMOEjXC+SHi5dw1V8ERUiL +rwOUHTFhXNn1/Kq7Wo3UQ6qdEPSgkm7hThsNEGI+H709POWVXlJEAyrj2wGFSgFg +BAN7/GmwHPxvCGY5BFvvt7ECgYBIWqOA4RmN+h8vfnTz3lOmReJWLrWi6TwMHCxY +s0HB21wEckG/D+AN/Vvef6PCgULDjiDUOiugEPonDvX6ZMcusRXuNXt0ZJYDYREK +ybaTWtYaUEX5rR9EO3pfrkOmx+zd6c09vtR8g4ZntUTnMyqZp0YgEFnI0REIHnk1 +7sk0EQKBgGTzNU9Ir6cEZh4j+Qf5rA38bejkD8aRAYg5ozQcbNRJnrC7QnpmZPeD +X/E9MZ6wj1BVXEn2oNC63n3QB+B8OhrIDAYDbnaCLzVDl/BTuom3uTCYk0beKncz +AurSDpc15RFYjqn0DPBSii/DTaQIz0Rg+seZrOp5Ii2LrSlsnDPf +-----END RSA PRIVATE KEY-----" +_APP_VCS_GITHUB_APP_ID=337303 +_APP_VCS_GITHUB_CLIENT_ID=Iv1.306ee38582d3f948 +_APP_VCS_GITHUB_CLIENT_SECRET=eafc638eaeebe95c0db0fdf59a0a99b9e41832eb +_APP_VCS_GITHUB_WEBHOOK_SECRET=gzbrfuenqodiefbrg39u _APP_MIGRATIONS_FIREBASE_CLIENT_ID= _APP_MIGRATIONS_FIREBASE_CLIENT_SECRET= _APP_ASSISTANT_OPENAI_API_KEY= \ No newline at end of file diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index e725ff5f3e..28d7046dd1 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -19,6 +19,7 @@ use Appwrite\Platform\Tasks\VolumeSync; use Appwrite\Platform\Tasks\CalcTierStats; use Appwrite\Platform\Tasks\Upgrade; use Appwrite\Platform\Tasks\DeleteOrphanedProjects; +use Appwrite\Platform\Tasks\PatchRecreateRepositoriesDocuments; class Tasks extends Service { @@ -42,6 +43,7 @@ class Tasks extends Service ->addAction(Specs::getName(), new Specs()) ->addAction(CalcTierStats::getName(), new CalcTierStats()) ->addAction(DeleteOrphanedProjects::getName(), new DeleteOrphanedProjects()) + ->addAction(PatchRecreateRepositoriesDocuments::getName(), new PatchRecreateRepositoriesDocuments()) ; } diff --git a/src/Appwrite/Platform/Tasks/patchRecreateRepositoriesDocuments.php b/src/Appwrite/Platform/Tasks/patchRecreateRepositoriesDocuments.php index 5749380d0a..4d04802f50 100644 --- a/src/Appwrite/Platform/Tasks/patchRecreateRepositoriesDocuments.php +++ b/src/Appwrite/Platform/Tasks/patchRecreateRepositoriesDocuments.php @@ -3,8 +3,6 @@ namespace Appwrite\Platform\Tasks; use Utopia\Platform\Action; -use Appwrite\Event\Certificate; -use Utopia\App; use Utopia\CLI\Console; use Utopia\Database\Database; use Utopia\Database\Document; @@ -12,10 +10,9 @@ use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Permission; use Utopia\Database\Helpers\Role; use Utopia\Database\Query; -use Utopia\Validator\Hostname; use Utopia\Validator\Text; -class patchRecreateRepositoriesDocuments extends Action +class PatchRecreateRepositoriesDocuments extends Action { public static function getName(): string { @@ -34,25 +31,25 @@ class patchRecreateRepositoriesDocuments extends Action } public function action($after, $projectId, Database $dbForConsole, callable $getProjectDB): void - { + { Console::info("Starting the patch"); $startTime = microtime(true); - if(!empty($projectId)) { + if (!empty($projectId)) { $project = $dbForConsole->getDocument('projects', $projectId); $dbForProject = call_user_func($getProjectDB, $project); $this->recreateRepositories($dbForConsole, $dbForProject, $project); } else { $queries = []; - if(!empty($after)) { + if (!empty($after)) { Console::info("Iterating remaining projects after project with ID {$after}"); $project = $dbForConsole->getDocument('projects', $after); $queries = [Query::cursorAfter($project)]; } else { Console::info("Iterating all projects"); } - $this->foreachDocument($dbForConsole, 'projects', $queries, function(Document $project) use($getProjectDB, $dbForConsole){ + $this->foreachDocument($dbForConsole, 'projects', $queries, function (Document $project) use ($getProjectDB, $dbForConsole) { $dbForProject = call_user_func($getProjectDB, $project); $this->recreateRepositories($dbForConsole, $dbForProject, $project); }); @@ -106,13 +103,13 @@ class patchRecreateRepositoriesDocuments extends Action $projectId = $project->getId(); Console::log("Running patch for project {$projectId}"); - $this->foreachDocument($dbForProject, 'functions', [], function(Document $function) use ($dbForConsole, $project) { + $this->foreachDocument($dbForProject, 'functions', [], function (Document $function) use ($dbForProject, $dbForConsole, $project) { $isConnected = !empty($function->getAttribute('providerRepositoryId', '')); - if($isConnected) { + if ($isConnected) { $repository = $dbForConsole->getDocument('repositories', $function->getAttribute('repositoryId', '')); - if($repository->isEmpty()) { + if ($repository->isEmpty()) { $projectId = $project->getId(); $functionId = $function->getId(); Console::success("Recreating repositories document for project ID {$projectId}, function ID {$functionId}"); @@ -134,6 +131,19 @@ class patchRecreateRepositoriesDocuments extends Action 'resourceType' => 'function', 'providerPullRequestIds' => [] ])); + + $function = $dbForProject->updateDocument('functions', $function->getId(), $function + ->setAttribute('repositoryId', $repository->getId()) + ->setAttribute('repositoryInternalId', $repository->getInternalId())); + + $this->foreachDocument($dbForProject, 'deployments', [ + Query::equal('resourceInternalId', [$function->getInternalId()]), + Query::equal('resourceType', ['functions']) + ], function (Document $deployment) use ($dbForProject, $repository) { + $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment + ->setAttribute('repositoryId', $repository->getId()) + ->setAttribute('repositoryInternalId', $repository->getInternalId())); + }); } } }); From 2572cb43c2007829bb4f91dd58fc0a680cc3f7d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 14 Nov 2023 15:11:17 +0100 Subject: [PATCH 20/34] Code cleanup --- .env | 38 ++++++-------------------------------- 1 file changed, 6 insertions(+), 32 deletions(-) diff --git a/.env b/.env index 759ddcc051..ad551e705a 100644 --- a/.env +++ b/.env @@ -90,38 +90,12 @@ _APP_GRAPHQL_MAX_COMPLEXITY=250 _APP_GRAPHQL_MAX_DEPTH=3 _APP_DOCKER_HUB_USERNAME= _APP_DOCKER_HUB_PASSWORD= -_APP_VCS_GITHUB_APP_NAME=appwrite-generated-on-22-5-2023 -_APP_VCS_GITHUB_PRIVATE_KEY="-----BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEApUP3k7ratjenwOA1AtnmvZbNIm9Mhu181ryrnb7M1z1/OLUo -jZ1iS8u1K/GQtoqCNI8RR38rBXW1hzfC34qAL3XrJr4caK+AKzV7B4y9e2Zn+VlF -ZZ2rknNPlRLCwll6V60M7suqLONUADs6bvEs8GIR926gwNi8SmomtIj8k8UZFW00 -9L0OM0pvajJKASPynZ7jXfxic5Yu060BQq+IAtOkOaxyBF1xiGQRBuAt2movTI2X -4PwGcSCy7YD6v2DRgHIoYehtdWx0DrbRgCXsmwMin5BR/7ZhBSJE7YVH9vSqWThe -V8VmLKPiZZC5ZW2Vf28kM/FlkCgjfd0tNXs8lQIDAQABAoIBAQCWaFYhODSnE93z -ttnoH2JVd7J4PW0be3ZbhNh3t1d8KPbpKE6hG/SC4QGg3bgDuekoZnCmbkE8NdWh -G4maotVo3FvIJct7JwZxzLmMtHUaoqfMEogLJEUrAxERrkJcWMz0kIUtq2PUeIxR -rZXPtGVe3RJW63MYL+ilnRhexDGDVa5I50jFCTT5GXXHl6tw6tei1rDc2kI7pb3k -BGAxsRGXUK6Di9EmR2Z8xFHvM3LeuSWR2WalR8+abn661P3J+InMqh0s+TBB86kH -6NUxoWKSCXjPfMWj15M0Wwm4CBlzgc6GS106hmHPi3YAQLCV4x0l4qmq+bdsQOnN -VAOnPAohAoGBANidEYsMm5jaTklFnIExBkm5OHcKRuMgplMoEsVE/dglBPCJB3Uf -JUmhMRVazztDGD0ciImypt4j9klR4zqSQv4Aa8OdwW/jcuNPm3qQrdSm86iP2Lbe -V9DNKK0vQ/3srwIYxl9qabOLaQrKAeiPxfUuL91iyDIYtsPPIgfvVGfdAoGBAMNQ -v5wruODDgf0mMm1nA9LNHlkMi8uaVvxFAhjWtmPOH4FHXgzMDGC6syvL+d1XIdtF -tA/j/f/A1zFsYzeZRVBqVmpd8rvRzFTaRrBgLjI/vxmbJ7syr9rT7iZYvFYUv3c+ -mr8m5AIGULiGmMYnSWttIi2prlA17FC5Qp/lq/gZAoGAVrBlePSOwNl9Qy2suLda -AN8zjdB7FiLW7ai3+mLmBD6sf2cXqPPSBGmSLy2sidcMOEjXC+SHi5dw1V8ERUiL -rwOUHTFhXNn1/Kq7Wo3UQ6qdEPSgkm7hThsNEGI+H709POWVXlJEAyrj2wGFSgFg -BAN7/GmwHPxvCGY5BFvvt7ECgYBIWqOA4RmN+h8vfnTz3lOmReJWLrWi6TwMHCxY -s0HB21wEckG/D+AN/Vvef6PCgULDjiDUOiugEPonDvX6ZMcusRXuNXt0ZJYDYREK -ybaTWtYaUEX5rR9EO3pfrkOmx+zd6c09vtR8g4ZntUTnMyqZp0YgEFnI0REIHnk1 -7sk0EQKBgGTzNU9Ir6cEZh4j+Qf5rA38bejkD8aRAYg5ozQcbNRJnrC7QnpmZPeD -X/E9MZ6wj1BVXEn2oNC63n3QB+B8OhrIDAYDbnaCLzVDl/BTuom3uTCYk0beKncz -AurSDpc15RFYjqn0DPBSii/DTaQIz0Rg+seZrOp5Ii2LrSlsnDPf ------END RSA PRIVATE KEY-----" -_APP_VCS_GITHUB_APP_ID=337303 -_APP_VCS_GITHUB_CLIENT_ID=Iv1.306ee38582d3f948 -_APP_VCS_GITHUB_CLIENT_SECRET=eafc638eaeebe95c0db0fdf59a0a99b9e41832eb -_APP_VCS_GITHUB_WEBHOOK_SECRET=gzbrfuenqodiefbrg39u +_APP_VCS_GITHUB_APP_NAME= +_APP_VCS_GITHUB_PRIVATE_KEY=disabled +_APP_VCS_GITHUB_APP_ID= +_APP_VCS_GITHUB_CLIENT_ID= +_APP_VCS_GITHUB_CLIENT_SECRET= +_APP_VCS_GITHUB_WEBHOOK_SECRET= _APP_MIGRATIONS_FIREBASE_CLIENT_ID= _APP_MIGRATIONS_FIREBASE_CLIENT_SECRET= _APP_ASSISTANT_OPENAI_API_KEY= \ No newline at end of file From 141b864a56c509f4adb561b98244a8edfdda7fc0 Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Tue, 14 Nov 2023 19:49:23 +0530 Subject: [PATCH 21/34] Update permissions for create repository document --- app/controllers/api/functions.php | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 268acc0692..9720a68889 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -761,11 +761,9 @@ App::put('/v1/functions/:functionId') $repository = $dbForConsole->createDocument('repositories', new Document([ '$id' => ID::unique(), '$permissions' => [ - Permission::read(Role::team(ID::custom($teamId))), - Permission::update(Role::team(ID::custom($teamId), 'owner')), - Permission::update(Role::team(ID::custom($teamId), 'developer')), - Permission::delete(Role::team(ID::custom($teamId), 'owner')), - Permission::delete(Role::team(ID::custom($teamId), 'developer')), + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), ], 'installationId' => $installation->getId(), 'installationInternalId' => $installation->getInternalId(), From c2e5849c49738b59001d61606dc0a642b9702e0b Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 14 Nov 2023 16:49:45 +0200 Subject: [PATCH 22/34] removing blank line --- src/Appwrite/Usage/Calculators/TimeSeries.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Appwrite/Usage/Calculators/TimeSeries.php b/src/Appwrite/Usage/Calculators/TimeSeries.php index 9ead48e93f..38574b9b03 100644 --- a/src/Appwrite/Usage/Calculators/TimeSeries.php +++ b/src/Appwrite/Usage/Calculators/TimeSeries.php @@ -423,7 +423,6 @@ class TimeSeries extends Calculator */ private function createOrUpdateMetric(string $projectId, string $time, string $period, string $metric, int $value, int $type): void { - $id = \md5("{$time}_{$period}_{$metric}"); $project = $this->database->getDocument('projects', $projectId); $database = call_user_func($this->getProjectDB, $project); From 7e1b618769448fd222d51551f64080d49443c1fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 14 Nov 2023 16:50:59 +0100 Subject: [PATCH 23/34] Fix permission issues with repositories collection --- app/controllers/api/functions.php | 18 ++++++++++++------ app/controllers/api/vcs.php | 24 ++++++++++++------------ 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 9720a68889..cbdbd3a1cb 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -242,12 +242,16 @@ App::post('/v1/functions') // Git connect logic if (!empty($providerRepositoryId)) { + $teamId = $project->getAttribute('teamId', ''); + $repository = $dbForConsole->createDocument('repositories', new Document([ '$id' => ID::unique(), '$permissions' => [ - Permission::read(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), + Permission::read(Role::team(ID::custom($teamId))), + Permission::update(Role::team(ID::custom($teamId), 'owner')), + Permission::update(Role::team(ID::custom($teamId), 'developer')), + Permission::delete(Role::team(ID::custom($teamId), 'owner')), + Permission::delete(Role::team(ID::custom($teamId), 'developer')), ], 'installationId' => $installation->getId(), 'installationInternalId' => $installation->getInternalId(), @@ -761,9 +765,11 @@ App::put('/v1/functions/:functionId') $repository = $dbForConsole->createDocument('repositories', new Document([ '$id' => ID::unique(), '$permissions' => [ - Permission::read(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), + Permission::read(Role::team(ID::custom($teamId))), + Permission::update(Role::team(ID::custom($teamId), 'owner')), + Permission::update(Role::team(ID::custom($teamId), 'developer')), + Permission::delete(Role::team(ID::custom($teamId), 'owner')), + Permission::delete(Role::team(ID::custom($teamId), 'developer')), ], 'installationId' => $installation->getId(), 'installationInternalId' => $installation->getInternalId(), diff --git a/app/controllers/api/vcs.php b/app/controllers/api/vcs.php index 8b61580b76..68a84d0a1d 100644 --- a/app/controllers/api/vcs.php +++ b/app/controllers/api/vcs.php @@ -857,10 +857,10 @@ App::post('/v1/vcs/github/events') $github->initializeVariables($providerInstallationId, $privateKey, $githubAppId); //find functionId from functions table - $repositories = $dbForConsole->find('repositories', [ + $repositories = Authorization::skip(fn () => $dbForConsole->find('repositories', [ Query::equal('providerRepositoryId', [$providerRepositoryId]), Query::limit(100), - ]); + ])); // create new deployment only on push and not when branch is created if (!$providerBranchCreated) { @@ -877,13 +877,13 @@ App::post('/v1/vcs/github/events') ]); foreach ($installations as $installation) { - $repositories = $dbForConsole->find('repositories', [ + $repositories = Authorization::skip(fn () => $dbForConsole->find('repositories', [ Query::equal('installationInternalId', [$installation->getInternalId()]), Query::limit(1000) - ]); + ])); foreach ($repositories as $repository) { - $dbForConsole->deleteDocument('repositories', $repository->getId()); + Authorization::skip(fn () => $dbForConsole->deleteDocument('repositories', $repository->getId())); } $dbForConsole->deleteDocument('installations', $installation->getId()); @@ -915,10 +915,10 @@ App::post('/v1/vcs/github/events') $providerCommitAuthor = $commitDetails["commitAuthor"] ?? ''; $providerCommitMessage = $commitDetails["commitMessage"] ?? ''; - $repositories = $dbForConsole->find('repositories', [ + $repositories = Authorization::skip(fn () => $dbForConsole->find('repositories', [ Query::equal('providerRepositoryId', [$providerRepositoryId]), Query::orderDesc('$createdAt') - ]); + ])); $createGitDeployments($github, $providerInstallationId, $repositories, $providerBranch, $providerBranchUrl, $providerRepositoryName, $providerRepositoryUrl, $providerRepositoryOwner, $providerCommitHash, $providerCommitAuthor, $providerCommitAuthorUrl, $providerCommitMessage, $providerCommitUrl, $providerPullRequestId, $external, $dbForConsole, $queueForBuilds, $getProjectDB, $request); } elseif ($parsedPayload["action"] == "closed") { @@ -929,10 +929,10 @@ App::post('/v1/vcs/github/events') $external = $parsedPayload["external"] ?? true; if ($external) { - $repositories = $dbForConsole->find('repositories', [ + $repositories = Authorization::skip(fn () => $dbForConsole->find('repositories', [ Query::equal('providerRepositoryId', [$providerRepositoryId]), Query::orderDesc('$createdAt') - ]); + ])); foreach ($repositories as $repository) { $providerPullRequestIds = $repository->getAttribute('providerPullRequestIds', []); @@ -1092,9 +1092,9 @@ App::patch('/v1/vcs/github/installations/:installationId/repositories/:repositor throw new Exception(Exception::INSTALLATION_NOT_FOUND); } - $repository = $dbForConsole->getDocument('repositories', $repositoryId, [ + $repository = Authorization::skip(fn () => $dbForConsole->getDocument('repositories', $repositoryId, [ Query::equal('projectInternalId', [$project->getInternalId()]) - ]); + ])); if ($repository->isEmpty()) { throw new Exception(Exception::REPOSITORY_NOT_FOUND); @@ -1109,7 +1109,7 @@ App::patch('/v1/vcs/github/installations/:installationId/repositories/:repositor // TODO: Delete from array when PR is closed - $repository = $dbForConsole->updateDocument('repositories', $repository->getId(), $repository); + $repository = Authorization::skip(fn () => $dbForConsole->updateDocument('repositories', $repository->getId(), $repository)); $privateKey = App::getEnv('_APP_VCS_GITHUB_PRIVATE_KEY'); $githubAppId = App::getEnv('_APP_VCS_GITHUB_APP_ID'); From 1c4fea0fc3e77b80cecad66a556ba9856d9d9076 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 14 Nov 2023 17:45:02 +0100 Subject: [PATCH 24/34] Fix patch script, make errors silent --- .../patchRecreateRepositoriesDocuments.php | 28 +++++++++++++++---- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/patchRecreateRepositoriesDocuments.php b/src/Appwrite/Platform/Tasks/patchRecreateRepositoriesDocuments.php index 4d04802f50..93e6c527bb 100644 --- a/src/Appwrite/Platform/Tasks/patchRecreateRepositoriesDocuments.php +++ b/src/Appwrite/Platform/Tasks/patchRecreateRepositoriesDocuments.php @@ -37,9 +37,17 @@ class PatchRecreateRepositoriesDocuments extends Action $startTime = microtime(true); if (!empty($projectId)) { - $project = $dbForConsole->getDocument('projects', $projectId); - $dbForProject = call_user_func($getProjectDB, $project); - $this->recreateRepositories($dbForConsole, $dbForProject, $project); + try { + $project = $dbForConsole->getDocument('projects', $projectId); + $dbForProject = call_user_func($getProjectDB, $project); + $this->recreateRepositories($dbForConsole, $dbForProject, $project); + } catch (\Throwable $th) { + Console::error("Unexpected error occured with Project ID {$projectId}"); + Console::error('[Error] Type: ' . get_class($th)); + Console::error('[Error] Message: ' . $th->getMessage()); + Console::error('[Error] File: ' . $th->getFile()); + Console::error('[Error] Line: ' . $th->getLine()); + } } else { $queries = []; if (!empty($after)) { @@ -50,8 +58,18 @@ class PatchRecreateRepositoriesDocuments extends Action Console::info("Iterating all projects"); } $this->foreachDocument($dbForConsole, 'projects', $queries, function (Document $project) use ($getProjectDB, $dbForConsole) { - $dbForProject = call_user_func($getProjectDB, $project); - $this->recreateRepositories($dbForConsole, $dbForProject, $project); + $projectId = $project->getId(); + + try { + $dbForProject = call_user_func($getProjectDB, $project); + $this->recreateRepositories($dbForConsole, $dbForProject, $project); + } catch (\Throwable $th) { + Console::error("Unexpected error occured with Project ID {$projectId}"); + Console::error('[Error] Type: ' . get_class($th)); + Console::error('[Error] Message: ' . $th->getMessage()); + Console::error('[Error] File: ' . $th->getFile()); + Console::error('[Error] Line: ' . $th->getLine()); + } }); } From c3015340ce7ae0d3686ce8aa5a11b36b4226c5f7 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 15 Nov 2023 16:46:43 +1300 Subject: [PATCH 25/34] Update database --- composer.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/composer.lock b/composer.lock index 25f0c3964d..d362f3aa9c 100644 --- a/composer.lock +++ b/composer.lock @@ -1906,16 +1906,16 @@ }, { "name": "utopia-php/database", - "version": "0.45.1", + "version": "0.45.2", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "0e76f996439b80794ab73c2fffdb51ebd6676e4b" + "reference": "dc789f2c1fd8b5ee07ff883e11c9ad7970824788" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/0e76f996439b80794ab73c2fffdb51ebd6676e4b", - "reference": "0e76f996439b80794ab73c2fffdb51ebd6676e4b", + "url": "https://api.github.com/repos/utopia-php/database/zipball/dc789f2c1fd8b5ee07ff883e11c9ad7970824788", + "reference": "dc789f2c1fd8b5ee07ff883e11c9ad7970824788", "shasum": "" }, "require": { @@ -1956,9 +1956,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.45.1" + "source": "https://github.com/utopia-php/database/tree/0.45.2" }, - "time": "2023-11-01T08:30:19+00:00" + "time": "2023-11-15T03:38:47+00:00" }, { "name": "utopia-php/domains", From 94931ff4ce5d8bfb1057c4743db488d2566874a3 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 15 Nov 2023 16:47:49 +1300 Subject: [PATCH 26/34] Fix non-PSR-compliant file name --- ...toriesDocuments.php => PatchRecreateRepositoriesDocuments.php} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/Appwrite/Platform/Tasks/{patchRecreateRepositoriesDocuments.php => PatchRecreateRepositoriesDocuments.php} (100%) diff --git a/src/Appwrite/Platform/Tasks/patchRecreateRepositoriesDocuments.php b/src/Appwrite/Platform/Tasks/PatchRecreateRepositoriesDocuments.php similarity index 100% rename from src/Appwrite/Platform/Tasks/patchRecreateRepositoriesDocuments.php rename to src/Appwrite/Platform/Tasks/PatchRecreateRepositoriesDocuments.php From 2e70ed59af5efcccbf3bf55e14176310e240e5e0 Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Wed, 15 Nov 2023 11:41:42 +0530 Subject: [PATCH 27/34] Fix git installation deletion --- app/controllers/api/vcs.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/controllers/api/vcs.php b/app/controllers/api/vcs.php index 68a84d0a1d..1b0c993e11 100644 --- a/app/controllers/api/vcs.php +++ b/app/controllers/api/vcs.php @@ -1046,8 +1046,8 @@ App::delete('/v1/vcs/installations/:installationId') ->inject('response') ->inject('project') ->inject('dbForConsole') - ->inject('deletes') - ->action(function (string $installationId, Response $response, Document $project, Database $dbForConsole, Delete $deletes) { + ->inject('queueForDeletes') + ->action(function (string $installationId, Response $response, Document $project, Database $dbForConsole, Delete $queueForDeletes) { $installation = $dbForConsole->getDocument('installations', $installationId); if ($installation->isEmpty()) { @@ -1058,7 +1058,7 @@ App::delete('/v1/vcs/installations/:installationId') throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove installation from DB'); } - $deletes + $queueForDeletes ->setType(DELETE_TYPE_DOCUMENT) ->setDocument($installation); From 7efecbca8b0637d0a16c7d78ded922d43a44a5d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BE=B7=E5=8D=8E=20=E5=88=98?= Date: Wed, 15 Nov 2023 17:20:50 +0000 Subject: [PATCH 28/34] chore: fixed indentation --- src/Appwrite/Usage/Calculators/TimeSeries.php | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Appwrite/Usage/Calculators/TimeSeries.php b/src/Appwrite/Usage/Calculators/TimeSeries.php index 38574b9b03..6dbf49c44c 100644 --- a/src/Appwrite/Usage/Calculators/TimeSeries.php +++ b/src/Appwrite/Usage/Calculators/TimeSeries.php @@ -432,13 +432,13 @@ class TimeSeries extends Calculator $document = $database->getDocument('stats', $id); if ($document->isEmpty()) { $database->createDocument('stats', new Document([ - '$id' => $id, - 'period' => $period, - 'time' => $time, - 'metric' => $metric, - 'value' => $value, - 'type' => $type, - 'region' => $this->region, + '$id' => $id, + 'period' => $period, + 'time' => $time, + 'metric' => $metric, + 'value' => $value, + 'type' => $type, + 'region' => $this->region, ])); } else { $database->updateDocument( From 31d5b75034e95c196666e9516f9e2eb01ac97d9a Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Wed, 15 Nov 2023 17:23:35 +0000 Subject: [PATCH 29/34] Bump console to version 3.2.7 --- .gitmodules | 2 +- app/console | 2 +- app/init.php | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index ff2e0a6aab..af12124355 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "app/console"] path = app/console url = https://github.com/appwrite/console - branch = 3.2.6 + branch = 3.2.7 diff --git a/app/console b/app/console index f7c34a1b37..49d039ed07 160000 --- a/app/console +++ b/app/console @@ -1 +1 @@ -Subproject commit f7c34a1b37d53dd5f28c83b4e12a4e68fcd9b484 +Subproject commit 49d039ed07628155e7f56e2c997fcef90ecde267 diff --git a/app/init.php b/app/init.php index be6b440498..2572d01151 100644 --- a/app/init.php +++ b/app/init.php @@ -109,7 +109,7 @@ const APP_LIMIT_LIST_DEFAULT = 25; // Default maximum number of items to return const APP_KEY_ACCCESS = 24 * 60 * 60; // 24 hours const APP_USER_ACCCESS = 24 * 60 * 60; // 24 hours const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours -const APP_CACHE_BUSTER = 516; +const APP_CACHE_BUSTER = 327; const APP_VERSION_STABLE = '1.4.11'; const APP_DATABASE_ATTRIBUTE_EMAIL = 'email'; const APP_DATABASE_ATTRIBUTE_ENUM = 'enum'; From 6b5e734dfaedfb6fb291d7dab6a7a883dcc5a555 Mon Sep 17 00:00:00 2001 From: shimon Date: Wed, 15 Nov 2023 20:09:50 +0200 Subject: [PATCH 30/34] indentation small fix --- src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php index ee9931bde4..5f0ecbe1db 100644 --- a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php +++ b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php @@ -112,9 +112,9 @@ class DeleteOrphanedProjects extends Action $dbForConsole->deleteCachedDocument('projects', $project->getId()); } - Console::info('--Deleting project no (' . $project->getInternalId() . ')'); + Console::info('--Deleting project no (' . $project->getInternalId() . ')'); - $orphans++; + $orphans++; } catch (\Throwable $th) { Console::error('Error: ' . $th->getMessage()); } finally { From 37e1e251705f11f7f192bbc2002b343a226508aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BE=B7=E5=8D=8E=20=E5=88=98?= Date: Wed, 15 Nov 2023 18:31:33 +0000 Subject: [PATCH 31/34] chore: update versions and changelog --- CHANGES.md | 4 ++++ README-CN.md | 6 +++--- README.md | 6 +++--- app/init.php | 2 +- src/Appwrite/Migration/Migration.php | 1 + 5 files changed, 12 insertions(+), 7 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index e8f0c08c1f..513ee2ec90 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,7 @@ +# Version 1.4.12 + +## Bug fixes + # Version 1.4.11 ## Miscellaneous diff --git a/README-CN.md b/README-CN.md index 4e45c423c4..1bbc8eb234 100644 --- a/README-CN.md +++ b/README-CN.md @@ -66,7 +66,7 @@ docker run -it --rm \ --volume /var/run/docker.sock:/var/run/docker.sock \ --volume "$(pwd)"/appwrite:/usr/src/code/appwrite:rw \ --entrypoint="install" \ - appwrite/appwrite:1.4.11 + appwrite/appwrite:1.4.12 ``` ### Windows @@ -78,7 +78,7 @@ docker run -it --rm ^ --volume //var/run/docker.sock:/var/run/docker.sock ^ --volume "%cd%"/appwrite:/usr/src/code/appwrite:rw ^ --entrypoint="install" ^ - appwrite/appwrite:1.4.11 + appwrite/appwrite:1.4.12 ``` #### PowerShell @@ -88,7 +88,7 @@ docker run -it --rm ` --volume /var/run/docker.sock:/var/run/docker.sock ` --volume ${pwd}/appwrite:/usr/src/code/appwrite:rw ` --entrypoint="install" ` - appwrite/appwrite:1.4.11 + appwrite/appwrite:1.4.12 ``` 运行后，可以在浏览器上访问 http://localhost 找到 Appwrite 控制台。在非 Linux 的本机主机上完成安装后，服务器可能需要几分钟才能启动。 diff --git a/README.md b/README.md index 88615a355a..3a1d2dbe9f 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ docker run -it --rm \ --volume /var/run/docker.sock:/var/run/docker.sock \ --volume "$(pwd)"/appwrite:/usr/src/code/appwrite:rw \ --entrypoint="install" \ - appwrite/appwrite:1.4.11 + appwrite/appwrite:1.4.12 ``` ### Windows @@ -88,7 +88,7 @@ docker run -it --rm ^ --volume //var/run/docker.sock:/var/run/docker.sock ^ --volume "%cd%"/appwrite:/usr/src/code/appwrite:rw ^ --entrypoint="install" ^ - appwrite/appwrite:1.4.11 + appwrite/appwrite:1.4.12 ``` #### PowerShell @@ -98,7 +98,7 @@ docker run -it --rm ` --volume /var/run/docker.sock:/var/run/docker.sock ` --volume ${pwd}/appwrite:/usr/src/code/appwrite:rw ` --entrypoint="install" ` - appwrite/appwrite:1.4.11 + appwrite/appwrite:1.4.12 ``` Once the Docker installation is complete, go to http://localhost to access the Appwrite console from your browser. Please note that on non-Linux native hosts, the server might take a few minutes to start after completing the installation. diff --git a/app/init.php b/app/init.php index 2572d01151..2c0219eec2 100644 --- a/app/init.php +++ b/app/init.php @@ -110,7 +110,7 @@ const APP_KEY_ACCCESS = 24 * 60 * 60; // 24 hours const APP_USER_ACCCESS = 24 * 60 * 60; // 24 hours const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours const APP_CACHE_BUSTER = 327; -const APP_VERSION_STABLE = '1.4.11'; +const APP_VERSION_STABLE = '1.4.12'; const APP_DATABASE_ATTRIBUTE_EMAIL = 'email'; const APP_DATABASE_ATTRIBUTE_ENUM = 'enum'; const APP_DATABASE_ATTRIBUTE_IP = 'ip'; diff --git a/src/Appwrite/Migration/Migration.php b/src/Appwrite/Migration/Migration.php index 406acae7df..8f68e31be4 100644 --- a/src/Appwrite/Migration/Migration.php +++ b/src/Appwrite/Migration/Migration.php @@ -76,6 +76,7 @@ abstract class Migration '1.4.9' => 'V19', '1.4.10' => 'V19', '1.4.11' => 'V19', + '1.4.12' => 'V19' ]; /** From e145ad4cb4ed9c0bc6e6c8613bf51c17e5b5ba5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BE=B7=E5=8D=8E=20=E5=88=98?= Date: Wed, 15 Nov 2023 18:41:57 +0000 Subject: [PATCH 32/34] chore: update versions and changelog --- CHANGES.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 513ee2ec90..889f65e1e7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,16 @@ # Version 1.4.12 +## Miscellaneous +* Bump console to version 3.2.7 [#7148](https://github.com/appwrite/appwrite/pull/7148) +* Chore update database to 0.45.2 [#7138](https://github.com/appwrite/appwrite/pull/7138) +* Implement queue thresholds for the health API [#7123](https://github.com/appwrite/appwrite/pull/7123) +* Add Authorization::skip to the usage worker [#7124](https://github.com/appwrite/appwrite/pull/7124) + ## Bug fixes +* fix: use queueForDeletes in git installation delete endpoint [#7140](https://github.com/appwrite/appwrite/pull/7140) +* fix: patch script, make errors silent [#7134](https://github.com/appwrite/appwrite/pull/7134) +* fix: repositories recreation script [#7133](https://github.com/appwrite/appwrite/pull/7133) +* fix: Only delete repositories linked to the particular project [#7131](https://github.com/appwrite/appwrite/pull/7131) # Version 1.4.11 From dceb1858b0698ffc3ae7291012ad615257ffbbc7 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 16 Nov 2023 20:31:53 +0200 Subject: [PATCH 33/34] comparison fix --- src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php index 5f0ecbe1db..26ada31416 100644 --- a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php +++ b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php @@ -90,7 +90,7 @@ class DeleteOrphanedProjects extends Action /** * +2 = audit+abuse */ - if ($collectionsCreated === (count($collectionsConfig) + 2)) { + if ($collectionsCreated >= (count($collectionsConfig) + 2)) { Console::log($msg . ' ignoring....'); continue; } From 8c9a662e1427d5beedd0ff63338c87bcdef3a4f4 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 16 Nov 2023 20:33:37 +0200 Subject: [PATCH 34/34] comparison fix --- src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php index 26ada31416..753240b66a 100644 --- a/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php +++ b/src/Appwrite/Platform/Tasks/DeleteOrphanedProjects.php @@ -54,7 +54,7 @@ class DeleteOrphanedProjects extends Action $totalProjects = $dbForConsole->count('projects'); Console::success("Found a total of: {$totalProjects} projects"); - $orphans = 1; + $orphans = 0; $cnt = 0; $count = 0; $limit = 30; @@ -135,6 +135,6 @@ class DeleteOrphanedProjects extends Action $count = $count + $sum; } - Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects found ' . $orphans - 1 . ' orphans'); + Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects found ' . $orphans . ' orphans'); } }