From a503803725c0a6d569a912f8b4c7b48d28df36be Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Mon, 21 Oct 2024 16:33:57 +0200 Subject: [PATCH 01/25] Add secret attribute to variables --- app/config/collections.php | 11 +++ app/controllers/api/functions.php | 4 +- app/controllers/api/project.php | 5 +- .../Utopia/Response/Model/Variable.php | 22 ++++++ .../Functions/FunctionsConsoleClientTest.php | 62 ++++++++++++++++- .../Projects/ProjectsConsoleClientTest.php | 68 ++++++++++++++++++- 6 files changed, 164 insertions(+), 8 deletions(-) diff --git a/app/config/collections.php b/app/config/collections.php index a55ab1abd0..cc8fc2d420 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -4031,6 +4031,17 @@ $projectCollections = array_merge([ 'array' => false, 'filters' => ['encrypt'] ], + [ + '$id' => ID::custom('secret'), + 'type' => Database::VAR_BOOLEAN, + 'format' => '', + 'size' => 0, + 'signed' => true, + 'required' => false, + 'default' => false, + 'array' => false, + 'filters' => [], + ], [ '$id' => ID::custom('search'), 'type' => Database::VAR_STRING, diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index c3051ef476..5bd8a993bb 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -2337,10 +2337,11 @@ App::post('/v1/functions/:functionId/variables') ->param('functionId', '', new UID(), 'Function unique ID.', false) ->param('key', null, new Text(Database::LENGTH_KEY), 'Variable key. Max length: ' . Database::LENGTH_KEY . ' chars.', false) ->param('value', null, new Text(8192, 0), 'Variable value. Max length: 8192 chars.', false) + ->param('secret', false, new Boolean(true), 'Is secret? Secret variables can only be updated or deleted, they cannot be read.', true) ->inject('response') ->inject('dbForProject') ->inject('dbForConsole') - ->action(function (string $functionId, string $key, string $value, Response $response, Database $dbForProject, Database $dbForConsole) { + ->action(function (string $functionId, string $key, string $value, mixed $secret, Response $response, Database $dbForProject, Database $dbForConsole) { $function = $dbForProject->getDocument('functions', $functionId); if ($function->isEmpty()) { @@ -2361,6 +2362,7 @@ App::post('/v1/functions/:functionId/variables') 'resourceType' => 'function', 'key' => $key, 'value' => $value, + 'secret' => $secret, 'search' => implode(' ', [$variableId, $function->getId(), $key, 'function']), ]); diff --git a/app/controllers/api/project.php b/app/controllers/api/project.php index 6053326308..d5957188a9 100644 --- a/app/controllers/api/project.php +++ b/app/controllers/api/project.php @@ -13,6 +13,7 @@ use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\Datetime as DateTimeValidator; use Utopia\Database\Validator\UID; +use Utopia\Validator\Boolean; use Utopia\Validator\Text; use Utopia\Validator\WhiteList; @@ -322,11 +323,12 @@ App::post('/v1/project/variables') ->label('sdk.response.model', Response::MODEL_VARIABLE) ->param('key', null, new Text(Database::LENGTH_KEY), 'Variable key. Max length: ' . Database::LENGTH_KEY . ' chars.', false) ->param('value', null, new Text(8192, 0), 'Variable value. Max length: 8192 chars.', false) + ->param('secret', false, new Boolean(true), 'Is secret? Secret variables can only be updated or deleted, they cannot be read.', true) ->inject('project') ->inject('response') ->inject('dbForProject') ->inject('dbForConsole') - ->action(function (string $key, string $value, Document $project, Response $response, Database $dbForProject, Database $dbForConsole) { + ->action(function (string $key, string $value, mixed $secret, Document $project, Response $response, Database $dbForProject, Database $dbForConsole) { $variableId = ID::unique(); $variable = new Document([ @@ -341,6 +343,7 @@ App::post('/v1/project/variables') 'resourceType' => 'project', 'key' => $key, 'value' => $value, + 'secret' => $secret, 'search' => implode(' ', [$variableId, $key, 'project']), ]); diff --git a/src/Appwrite/Utopia/Response/Model/Variable.php b/src/Appwrite/Utopia/Response/Model/Variable.php index 88fcd14ca1..d479eb541c 100644 --- a/src/Appwrite/Utopia/Response/Model/Variable.php +++ b/src/Appwrite/Utopia/Response/Model/Variable.php @@ -4,6 +4,7 @@ namespace Appwrite\Utopia\Response\Model; use Appwrite\Utopia\Response; use Appwrite\Utopia\Response\Model; +use Utopia\Database\Document; class Variable extends Model { @@ -41,6 +42,12 @@ class Variable extends Model 'default' => '', 'example' => 'myPa$$word1', ]) + ->addRule('secret', [ + 'type' => self::TYPE_BOOLEAN, + 'description' => 'Variable secret flag.', + 'default' => false, + 'example' => false, + ]) ->addRule('resourceType', [ 'type' => self::TYPE_STRING, 'description' => 'Service to which the variable belongs. Possible values are "project", "function"', @@ -56,6 +63,21 @@ class Variable extends Model ; } + /** + * Filter + * + * @param Document $document + * @return Document + */ + public function filter(Document $document): Document + { + $secret = $document->getAttribute('secret'); + if ($secret === true) { + $document->setAttribute('value', null); + } + return $document; + } + /** * Get Name * diff --git a/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php b/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php index 3a02cbcba2..0708d40aab 100644 --- a/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php @@ -118,6 +118,22 @@ class FunctionsConsoleClientTest extends Scope $variableId = $variable['body']['$id']; + // test for secret variable + $variable = $this->createVariable( + $data['functionId'], + [ + 'key' => 'APP_TEST_1', + 'value' => 'TESTINGVALUE_1', + 'secret' => true + ] + ); + + $this->assertEquals(201, $variable['headers']['status-code']); + $this->assertEquals('APP_TEST_1', $variable['body']['key']); + $this->assertEmpty($variable['body']['value']); + + $secretVariableId = $variable['body']['$id']; + /** * Test for FAILURE */ @@ -157,7 +173,8 @@ class FunctionsConsoleClientTest extends Scope return array_merge( $data, [ - 'variableId' => $variableId + 'variableId' => $variableId, + 'secretVariableId' => $secretVariableId ] ); } @@ -177,10 +194,12 @@ class FunctionsConsoleClientTest extends Scope ], $this->getHeaders())); $this->assertEquals(200, $response['headers']['status-code']); - $this->assertEquals(1, sizeof($response['body']['variables'])); - $this->assertEquals(1, $response['body']['total']); + $this->assertEquals(2, sizeof($response['body']['variables'])); + $this->assertEquals(2, $response['body']['total']); $this->assertEquals("APP_TEST", $response['body']['variables'][0]['key']); $this->assertEquals("TESTINGVALUE", $response['body']['variables'][0]['value']); + $this->assertEquals("APP_TEST_1", $response['body']['variables'][1]['key']); + $this->assertEmpty($response['body']['variables'][1]['value']); /** * Test for FAILURE @@ -207,6 +226,15 @@ class FunctionsConsoleClientTest extends Scope $this->assertEquals("APP_TEST", $response['body']['key']); $this->assertEquals("TESTINGVALUE", $response['body']['value']); + $response = $this->client->call(Client::METHOD_GET, '/functions/' . $data['functionId'] . '/variables/' . $data['secretVariableId'], array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals("APP_TEST_1", $response['body']['key']); + $this->assertEmpty($response['body']['value']); + /** * Test for FAILURE */ @@ -251,6 +279,27 @@ class FunctionsConsoleClientTest extends Scope $this->assertEquals("APP_TEST_UPDATE", $variable['body']['key']); $this->assertEquals("TESTINGVALUEUPDATED", $variable['body']['value']); + $response = $this->client->call(Client::METHOD_PUT, '/functions/' . $data['functionId'] . '/variables/' . $data['secretVariableId'], array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'key' => 'APP_TEST_UPDATE_1', + 'value' => 'TESTINGVALUEUPDATED_1' + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals("APP_TEST_UPDATE_1", $response['body']['key']); + $this->assertEmpty($response['body']['value']); + + $variable = $this->client->call(Client::METHOD_GET, '/functions/' . $data['functionId'] . '/variables/' . $data['secretVariableId'], array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(200, $variable['headers']['status-code']); + $this->assertEquals("APP_TEST_UPDATE_1", $variable['body']['key']); + $this->assertEmpty($variable['body']['value']); + $response = $this->client->call(Client::METHOD_PUT, '/functions/' . $data['functionId'] . '/variables/' . $data['variableId'], array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], @@ -332,6 +381,13 @@ class FunctionsConsoleClientTest extends Scope $this->assertEquals(204, $response['headers']['status-code']); + $response = $this->client->call(Client::METHOD_DELETE, '/functions/' . $data['functionId'] . '/variables/' . $data['secretVariableId'], array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertEquals(204, $response['headers']['status-code']); + $response = $this->client->call(Client::METHOD_GET, '/functions/' . $data['functionId'] . '/variables', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], diff --git a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php index 7b0847126c..48210435e7 100644 --- a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php +++ b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php @@ -3814,6 +3814,23 @@ class ProjectsConsoleClientTest extends Scope $this->assertEquals(201, $variable['headers']['status-code']); $variableId = $variable['body']['$id']; + // test for secret variable + $variable = $this->client->call(Client::METHOD_POST, '/project/variables', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $data['projectId'], + 'x-appwrite-mode' => 'admin', + ], $this->getHeaders()), [ + 'key' => 'APP_TEST_1', + 'value' => 'TESTINGVALUE_1', + 'secret' => true + ]); + + $this->assertEquals(201, $variable['headers']['status-code']); + $this->assertEquals('APP_TEST_1', $variable['body']['key']); + $this->assertEmpty($variable['body']['value']); + + $secretVariableId = $variable['body']['$id']; + /** * Test for FAILURE */ @@ -3857,6 +3874,7 @@ class ProjectsConsoleClientTest extends Scope $data, [ 'variableId' => $variableId, + 'secretVariableId' => $secretVariableId ] ); } @@ -3877,10 +3895,12 @@ class ProjectsConsoleClientTest extends Scope ], $this->getHeaders())); $this->assertEquals(200, $response['headers']['status-code']); - $this->assertCount(1, $response['body']['variables']); - $this->assertEquals(1, $response['body']['total']); + $this->assertCount(2, $response['body']['variables']); + $this->assertEquals(2, $response['body']['total']); $this->assertEquals("APP_TEST", $response['body']['variables'][0]['key']); $this->assertEquals("TESTINGVALUE", $response['body']['variables'][0]['value']); + $this->assertEquals("APP_TEST_1", $response['body']['variables'][1]['key']); + $this->assertEmpty($response['body']['variables'][1]['value']); return $data; } @@ -3903,6 +3923,16 @@ class ProjectsConsoleClientTest extends Scope $this->assertEquals("APP_TEST", $response['body']['key']); $this->assertEquals("TESTINGVALUE", $response['body']['value']); + $response = $this->client->call(Client::METHOD_GET, '/project/variables/' . $data['secretVariableId'], array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $data['projectId'], + 'x-appwrite-mode' => 'admin', + ], $this->getHeaders())); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals("APP_TEST_1", $response['body']['key']); + $this->assertEmpty($response['body']['value']); + /** * Test for FAILURE */ @@ -3950,6 +3980,29 @@ class ProjectsConsoleClientTest extends Scope $this->assertEquals("APP_TEST_UPDATE", $variable['body']['key']); $this->assertEquals("TESTINGVALUEUPDATED", $variable['body']['value']); + $response = $this->client->call(Client::METHOD_PUT, '/project/variables/' . $data['secretVariableId'], array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $data['projectId'], + 'x-appwrite-mode' => 'admin', + ], $this->getHeaders()), [ + 'key' => 'APP_TEST_UPDATE_1', + 'value' => 'TESTINGVALUEUPDATED_1' + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals("APP_TEST_UPDATE_1", $response['body']['key']); + $this->assertEmpty($response['body']['value']); + + $variable = $this->client->call(Client::METHOD_GET, '/project/variables/' . $data['secretVariableId'], array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $data['projectId'], + 'x-appwrite-mode' => 'admin', + ], $this->getHeaders())); + + $this->assertEquals(200, $variable['headers']['status-code']); + $this->assertEquals("APP_TEST_UPDATE_1", $variable['body']['key']); + $this->assertEmpty($variable['body']['value']); + $response = $this->client->call(Client::METHOD_GET, '/project/variables', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $data['projectId'], @@ -3957,8 +4010,9 @@ class ProjectsConsoleClientTest extends Scope ], $this->getHeaders())); $this->assertEquals(200, $response['headers']['status-code']); - $this->assertCount(1, $response['body']['variables']); + $this->assertCount(2, $response['body']['variables']); $this->assertEquals("APP_TEST_UPDATE", $response['body']['variables'][0]['key']); + $this->assertEquals("APP_TEST_UPDATE_1", $response['body']['variables'][1]['key']); /** * Test for FAILURE @@ -4037,6 +4091,14 @@ class ProjectsConsoleClientTest extends Scope $this->assertEquals(204, $response['headers']['status-code']); + $this->assertEquals(204, $response['headers']['status-code']); + + $response = $this->client->call(Client::METHOD_DELETE, '/project/variables/' . $data['secretVariableId'], array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $data['projectId'], + 'x-appwrite-mode' => 'admin', + ], $this->getHeaders())); + $response = $this->client->call(Client::METHOD_GET, '/project/variables', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $data['projectId'], From 4240089d41706020d83031bf441b3f7dc82490a6 Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Tue, 22 Oct 2024 19:45:49 +0200 Subject: [PATCH 02/25] Add create deployment endpoint --- .env | 1 + app/config/errors.php | 5 + app/init.php | 5 + app/worker.php | 4 + docker-compose.yml | 5 + src/Appwrite/Extend/Exception.php | 1 + .../Modules/Functions/Workers/Builds.php | 34 +-- .../Http/Deployments/CreateDeployment.php | 266 ++++++++++++++++++ .../Platform/Modules/Sites/Services/Http.php | 2 + src/Appwrite/Platform/Workers/Deletes.php | 15 +- 10 files changed, 305 insertions(+), 33 deletions(-) create mode 100644 src/Appwrite/Platform/Modules/Sites/Http/Deployments/CreateDeployment.php diff --git a/.env b/.env index 6b6a4cfa47..a663c0d278 100644 --- a/.env +++ b/.env @@ -67,6 +67,7 @@ _APP_SMS_FROM=+123456789 _APP_SMS_PROJECTS_DENY_LIST= _APP_STORAGE_LIMIT=30000000 _APP_STORAGE_PREVIEW_LIMIT=20000000 +_APP_SITES_SIZE_LIMIT=30000000 _APP_FUNCTIONS_SIZE_LIMIT=30000000 _APP_FUNCTIONS_TIMEOUT=900 _APP_FUNCTIONS_BUILD_TIMEOUT=900 diff --git a/app/config/errors.php b/app/config/errors.php index 6e05d87052..df8cb45c98 100644 --- a/app/config/errors.php +++ b/app/config/errors.php @@ -536,6 +536,11 @@ return [ ], /** Sites */ + Exception::SITE_NOT_FOUND => [ + 'name' => Exception::SITE_NOT_FOUND, + 'description' => 'Site with the requested ID could not be found.', + 'code' => 404, + ], Exception::SITE_FRAMEWORK_UNSUPPORTED => [ 'name' => Exception::SITE_FRAMEWORK_UNSUPPORTED, 'description' => 'The requested framework is either inactive or unsupported. Please check the value of the _APP_SITES_FRAMEWORKS environment variable.', diff --git a/app/init.php b/app/init.php index 9867c59e8f..078db328ca 100644 --- a/app/init.php +++ b/app/init.php @@ -132,6 +132,7 @@ const APP_DATABASE_ATTRIBUTE_STRING_MAX_LENGTH = 1_073_741_824; // 2^32 bits / 4 const APP_DATABASE_TIMEOUT_MILLISECONDS = 15_000; const APP_DATABASE_QUERY_MAX_VALUES = 500; const APP_STORAGE_UPLOADS = '/storage/uploads'; +const APP_STORAGE_SITES = '/storage/sites'; const APP_STORAGE_FUNCTIONS = '/storage/functions'; const APP_STORAGE_BUILDS = '/storage/builds'; const APP_STORAGE_CACHE = '/storage/cache'; @@ -1523,6 +1524,10 @@ App::setResource('deviceForFiles', function ($project) { return getDevice(APP_STORAGE_UPLOADS . '/app-' . $project->getId()); }, ['project']); +App::setResource('deviceForSites', function ($project) { + return getDevice(APP_STORAGE_SITES . '/app-' . $project->getId()); +}, ['project']); + App::setResource('deviceForFunctions', function ($project) { return getDevice(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId()); }, ['project']); diff --git a/app/worker.php b/app/worker.php index 2d59259284..2928522bac 100644 --- a/app/worker.php +++ b/app/worker.php @@ -256,6 +256,10 @@ Server::setResource('pools', function (Registry $register) { return $register->get('pools'); }, ['register']); +Server::setResource('deviceForSites', function (Document $project) { + return getDevice(APP_STORAGE_SITES . '/app-' . $project->getId()); +}, ['project']); + Server::setResource('deviceForFunctions', function (Document $project) { return getDevice(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId()); }, ['project']); diff --git a/docker-compose.yml b/docker-compose.yml index 479ca38b8f..b0292045ca 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -161,6 +161,11 @@ services: - _APP_FUNCTIONS_CPUS - _APP_FUNCTIONS_MEMORY - _APP_FUNCTIONS_RUNTIMES + - _APP_SITES_FRAMEWORKS + - _APP_SITES_CPUS + - _APP_SITES_MEMORY + - _APP_SITES_SIZE_LIMIT + - _APP_DOMAIN_SITES - _APP_EXECUTOR_SECRET - _APP_EXECUTOR_HOST - _APP_LOGGING_CONFIG diff --git a/src/Appwrite/Extend/Exception.php b/src/Appwrite/Extend/Exception.php index 29d0cf959b..4412505cae 100644 --- a/src/Appwrite/Extend/Exception.php +++ b/src/Appwrite/Extend/Exception.php @@ -152,6 +152,7 @@ class Exception extends \Exception public const GENERAL_PROVIDER_FAILURE = 'general_provider_failure'; /** Sites */ + public const SITE_NOT_FOUND = 'site_not_found'; public const SITE_FRAMEWORK_UNSUPPORTED = 'site_framework_unsupported'; /** Functions */ diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index d2d7f13afc..4f662f566a 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -155,31 +155,7 @@ class Builds extends Action $runtime = $runtimes[$key] ?? null; if ($isSite) { - // $key = "{$this->key}-{$version->version}"; - // $list[$key] = array_merge( - // [ - // 'key' => $this->key, - // 'name' => $this->name, - // 'logo' => "{$this->key}.png", - // 'startCommand' => $this->startCommand, - // ], - // [ - // 'version' => $this->version, - // 'base' => $this->base, - // 'image' => $this->image, - // 'supports' => $this->supports, - // ] - // ); - $runtime = [ - 'key' => 'static-for-now', - 'name' => 'Static', - 'logo' => 'node.png', - 'startCommand' => null, - 'version' => 'v1', - 'base' => 'rtsp/lighttpd', - 'image' => 'rtsp/lighttpd', - 'supports' => [System::X86, System::ARM64, System::ARMV7, System::ARMV8] - ]; + $runtime = $runtimes['node-18.0']; } if (\is_null($runtime)) { @@ -591,9 +567,13 @@ class Builds extends Action $isCanceled = false; Co::join([ - Co\go(function () use ($executor, &$response, $project, $deployment, $source, $resource, $runtime, $vars, $command, $cpus, $memory, &$err) { + Co\go(function () use ($executor, &$response, $project, $deployment, $source, $resource, $runtime, $vars, $command, $cpus, $memory, &$err, $isSite) { try { + $version = $resource->getAttribute('version', 'v2'); + if ($isSite) { + $version = 'v4'; + } $command = $version === 'v2' ? 'tar -zxf /tmp/code.tar.gz -C /usr/code && cd /usr/local/src/ && ./build.sh' : 'tar -zxf /tmp/code.tar.gz -C /mnt/code && helpers/build.sh "' . \trim(\escapeshellarg($command), "\'") . '"'; $response = $executor->createRuntime( @@ -605,7 +585,7 @@ class Builds extends Action cpus: $cpus, memory: $memory, remove: true, - entrypoint: $deployment->getAttribute('entrypoint'), + entrypoint: $deployment->getAttribute('entrypoint', 'package.json'), // TODO: change this later so that sites don't need to have an entrypoint destination: APP_STORAGE_BUILDS . "/app-{$project->getId()}", variables: $vars, command: $command diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/CreateDeployment.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/CreateDeployment.php new file mode 100644 index 0000000000..b710305dc0 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/CreateDeployment.php @@ -0,0 +1,266 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_POST) + ->setHttpPath('/v1/sites/:siteId/deployments') + ->desc('Create deployment') + ->groups(['api', 'sites']) + ->label('scope', 'functions.write') //TODO: Update the scope to sites later + ->label('event', 'sites.[siteId].deployments.[deploymentId].create') + ->label('audits.event', 'deployment.create') + ->label('audits.resource', 'site/{request.siteId}') + ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) + ->label('sdk.namespace', 'sites') + ->label('sdk.method', 'createDeployment') + ->label('sdk.methodType', 'upload') + ->label('sdk.description', '/docs/references/sites/create-deployment.md') //TODO: Create new docs + ->label('sdk.packaging', true) + ->label('sdk.request.type', 'multipart/form-data') + ->label('sdk.response.code', Response::STATUS_CODE_ACCEPTED) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_DEPLOYMENT) + ->param('siteId', '', new UID(), 'Site ID.') + ->param('installCommand', null, new Text(8192, 0), 'Install Commands.', true) + ->param('buildCommand', null, new Text(8192, 0), 'Build Commands.', true) + ->param('outputDirectory', null, new Text(8192, 0), 'Output Directory.', true) + ->param('code', [], new File(), 'Gzip file with your code package. When used with the Appwrite CLI, pass the path to your code directory, and the CLI will automatically package your code. Use a path that is within the current directory.', skipValidation: true) + ->param('activate', false, new Boolean(true), 'Automatically activate the deployment when it is finished building.') + ->inject('request') + ->inject('response') + ->inject('dbForProject') + ->inject('queueForEvents') + ->inject('project') + ->inject('deviceForSites') + ->inject('deviceForFunctions') // TODO: Remove this later once volume is added to executor + ->inject('deviceForLocal') + ->inject('queueForBuilds') + ->callback([$this, 'action']); + } + + public function action(string $siteId, ?string $installCommand, ?string $buildCommand, ?string $outputDirectory, mixed $code, mixed $activate, Request $request, Response $response, Database $dbForProject, Event $queueForEvents, Document $project, Device $deviceForSites, Device $deviceForFunctions, Device $deviceForLocal, Build $queueForBuilds) + { + $activate = \strval($activate) === 'true' || \strval($activate) === '1'; + + $site = $dbForProject->getDocument('sites', $siteId); + + if ($site->isEmpty()) { + throw new Exception(Exception::SITE_NOT_FOUND); + } + + if ($installCommand === null) { + $installCommand = $site->getAttribute('installCommand', ''); + } + + if ($buildCommand === null) { + $buildCommand = $site->getAttribute('buildCommand', ''); + } + + if ($outputDirectory === null) { + $outputDirectory = $site->getAttribute('outputDirectory', ''); + } + + $file = $request->getFiles('code'); + + // GraphQL multipart spec adds files with index keys + if (empty($file)) { + $file = $request->getFiles(0); + } + + if (empty($file)) { + throw new Exception(Exception::STORAGE_FILE_EMPTY, 'No file sent'); + } + + $fileExt = new FileExt([FileExt::TYPE_GZIP]); + $fileSizeValidator = new FileSize(System::getEnv('_APP_SITES_SIZE_LIMIT', '30000000')); + $upload = new Upload(); + + // Make sure we handle a single file and multiple files the same way + $fileName = (\is_array($file['name']) && isset($file['name'][0])) ? $file['name'][0] : $file['name']; + $fileTmpName = (\is_array($file['tmp_name']) && isset($file['tmp_name'][0])) ? $file['tmp_name'][0] : $file['tmp_name']; + $fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size']; + + if (!$fileExt->isValid($file['name'])) { // Check if file type is allowed + throw new Exception(Exception::STORAGE_FILE_TYPE_UNSUPPORTED); + } + + $contentRange = $request->getHeader('content-range'); + $deploymentId = ID::unique(); + $chunk = 1; + $chunks = 1; + + if (!empty($contentRange)) { + $start = $request->getContentRangeStart(); + $end = $request->getContentRangeEnd(); + $fileSize = $request->getContentRangeSize(); + $deploymentId = $request->getHeader('x-appwrite-id', $deploymentId); + // TODO make `end >= $fileSize` in next breaking version + if (is_null($start) || is_null($end) || is_null($fileSize) || $end > $fileSize) { + throw new Exception(Exception::STORAGE_INVALID_CONTENT_RANGE); + } + + // TODO remove the condition that checks `$end === $fileSize` in next breaking version + if ($end === $fileSize - 1 || $end === $fileSize) { + //if it's a last chunks the chunk size might differ, so we set the $chunks and $chunk to notify it's last chunk + $chunks = $chunk = -1; + } else { + // Calculate total number of chunks based on the chunk size i.e ($rangeEnd - $rangeStart) + $chunks = (int) ceil($fileSize / ($end + 1 - $start)); + $chunk = (int) ($start / ($end + 1 - $start)) + 1; + } + } + + if (!$fileSizeValidator->isValid($fileSize)) { // Check if file size is exceeding allowed limit + throw new Exception(Exception::STORAGE_INVALID_FILE_SIZE); + } + + if (!$upload->isValid($fileTmpName)) { + throw new Exception(Exception::STORAGE_INVALID_FILE); + } + + // Save to storage + $fileSize ??= $deviceForLocal->getFileSize($fileTmpName); + $path = $deviceForFunctions->getPath($deploymentId . '.' . \pathinfo($fileName, PATHINFO_EXTENSION)); + $deployment = $dbForProject->getDocument('deployments', $deploymentId); + + $metadata = ['content_type' => $deviceForLocal->getFileMimeType($fileTmpName)]; + if (!$deployment->isEmpty()) { + $chunks = $deployment->getAttribute('chunksTotal', 1); + $metadata = $deployment->getAttribute('metadata', []); + if ($chunk === -1) { + $chunk = $chunks; + } + } + + $chunksUploaded = $deviceForFunctions->upload($fileTmpName, $path, $chunk, $chunks, $metadata); + + if (empty($chunksUploaded)) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed moving file'); + } + + $type = $request->getHeader('x-sdk-language') === 'cli' ? 'cli' : 'manual'; + + if ($chunksUploaded === $chunks) { + if ($activate) { + // Remove deploy for all other deployments. + $activeDeployments = $dbForProject->find('deployments', [ + Query::equal('activate', [true]), + Query::equal('resourceId', [$siteId]), + Query::equal('resourceType', ['sites']) + ]); + + foreach ($activeDeployments as $activeDeployment) { + $activeDeployment->setAttribute('activate', false); + $dbForProject->updateDocument('deployments', $activeDeployment->getId(), $activeDeployment); + } + } + + $fileSize = $deviceForFunctions->getFileSize($path); + + if ($deployment->isEmpty()) { + $deployment = $dbForProject->createDocument('deployments', new Document([ + '$id' => $deploymentId, + '$permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + 'resourceInternalId' => $site->getInternalId(), + 'resourceId' => $site->getId(), + 'resourceType' => 'sites', + 'buildInternalId' => '', + 'installCommand' => $installCommand, + 'buildCommand' => $buildCommand, + 'outputDirectory' => $outputDirectory, + 'path' => $path, + 'size' => $fileSize, + 'search' => implode(' ', [$deploymentId]), + 'activate' => $activate, + 'metadata' => $metadata, + 'type' => $type + ])); + } else { + $deployment = $dbForProject->updateDocument('deployments', $deploymentId, $deployment->setAttribute('size', $fileSize)->setAttribute('metadata', $metadata)); + } + + // Start the build + $queueForBuilds + ->setType(BUILD_TYPE_DEPLOYMENT) + ->setResource($site) + ->setDeployment($deployment); + } else { + if ($deployment->isEmpty()) { + $deployment = $dbForProject->createDocument('deployments', new Document([ + '$id' => $deploymentId, + '$permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + 'resourceInternalId' => $site->getInternalId(), + 'resourceId' => $site->getId(), + 'resourceType' => 'sites', + 'buildInternalId' => '', + 'installCommand' => $installCommand, + 'buildCommand' => $buildCommand, + 'outputDirectory' => $outputDirectory, + 'path' => $path, + 'size' => $fileSize, + 'chunksTotal' => $chunks, + 'chunksUploaded' => $chunksUploaded, + 'search' => implode(' ', [$deploymentId]), + 'activate' => $activate, + 'metadata' => $metadata, + 'type' => $type + ])); + } else { + $deployment = $dbForProject->updateDocument('deployments', $deploymentId, $deployment->setAttribute('chunksUploaded', $chunksUploaded)->setAttribute('metadata', $metadata)); + } + } + + $metadata = null; + + $queueForEvents + ->setParam('siteId', $site->getId()) + ->setParam('deploymentId', $deployment->getId()); + + $response + ->setStatusCode(Response::STATUS_CODE_ACCEPTED) + ->dynamic($deployment, Response::MODEL_DEPLOYMENT); + } +} diff --git a/src/Appwrite/Platform/Modules/Sites/Services/Http.php b/src/Appwrite/Platform/Modules/Sites/Services/Http.php index 1730426a73..d2d2641cb5 100644 --- a/src/Appwrite/Platform/Modules/Sites/Services/Http.php +++ b/src/Appwrite/Platform/Modules/Sites/Services/Http.php @@ -2,6 +2,7 @@ namespace Appwrite\Platform\Modules\Sites\Services; +use Appwrite\Platform\Modules\Sites\Http\Deployments\CreateDeployment; use Appwrite\Platform\Modules\Sites\Http\Sites\CreateSite; use Utopia\Platform\Service; @@ -11,5 +12,6 @@ class Http extends Service { $this->type = Service::TYPE_HTTP; $this->addAction(CreateSite::getName(), new CreateSite()); + $this->addAction(CreateDeployment::getName(), new CreateDeployment()); } } diff --git a/src/Appwrite/Platform/Workers/Deletes.php b/src/Appwrite/Platform/Workers/Deletes.php index 48e4014f1e..676120b741 100644 --- a/src/Appwrite/Platform/Workers/Deletes.php +++ b/src/Appwrite/Platform/Workers/Deletes.php @@ -47,6 +47,7 @@ class Deletes extends Action ->inject('dbForConsole') ->inject('getProjectDB') ->inject('deviceForFiles') + ->inject('deviceForSites') ->inject('deviceForFunctions') ->inject('deviceForBuilds') ->inject('deviceForCache') @@ -54,14 +55,14 @@ class Deletes extends Action ->inject('executionRetention') ->inject('auditRetention') ->inject('log') - ->callback(fn ($message, $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => $this->action($message, $dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $abuseRetention, $executionRetention, $auditRetention, $log)); + ->callback(fn ($message, $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForSites, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log) => $this->action($message, $dbForConsole, $getProjectDB, $deviceForFiles, $deviceForSites, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $abuseRetention, $executionRetention, $auditRetention, $log)); } /** * @throws Exception * @throws Throwable */ - public function action(Message $message, Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void + public function action(Message $message, Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForSites, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, string $abuseRetention, string $executionRetention, string $auditRetention, Log $log): void { $payload = $message->getPayload() ?? []; @@ -84,7 +85,7 @@ class Deletes extends Action case DELETE_TYPE_DOCUMENT: switch ($document->getCollection()) { case DELETE_TYPE_PROJECTS: - $this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $document); + $this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForSites, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $document); break; case DELETE_TYPE_FUNCTIONS: $this->deleteFunction($dbForConsole, $getProjectDB, $deviceForFunctions, $deviceForBuilds, $document, $project); @@ -451,11 +452,12 @@ class Deletes extends Action foreach ($projects as $project) { $deviceForFiles = getDevice(APP_STORAGE_UPLOADS . '/app-' . $project->getId()); + $deviceForSites = getDevice(APP_STORAGE_SITES . '/app-' . $project->getId()); $deviceForFunctions = getDevice(APP_STORAGE_FUNCTIONS . '/app-' . $project->getId()); $deviceForBuilds = getDevice(APP_STORAGE_BUILDS . '/app-' . $project->getId()); $deviceForCache = getDevice(APP_STORAGE_CACHE . '/app-' . $project->getId()); - $this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $project); + $this->deleteProject($dbForConsole, $getProjectDB, $deviceForFiles, $deviceForSites, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $project); $dbForConsole->deleteDocument('projects', $project->getId()); } } @@ -473,7 +475,7 @@ class Deletes extends Action * @throws Authorization * @throws DatabaseException */ - private function deleteProject(Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, Document $document): void + private function deleteProject(Database $dbForConsole, callable $getProjectDB, Device $deviceForFiles, Device $deviceForSites, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForCache, Document $document): void { $projectInternalId = $document->getInternalId(); $projectId = $document->getId(); @@ -503,7 +505,7 @@ class Deletes extends Action try { $dbForProject->deleteCollection($collection->getId()); } catch (Throwable $e) { - Console::error('Error deleting '.$collection->getId().' '.$e->getMessage()); + Console::error('Error deleting ' . $collection->getId() . ' ' . $e->getMessage()); /** * Ignore junction tables; @@ -579,6 +581,7 @@ class Deletes extends Action // Delete all storage directories $deviceForFiles->delete($deviceForFiles->getRoot(), true); + $deviceForSites->delete($deviceForSites->getRoot(), true); $deviceForFunctions->delete($deviceForFunctions->getRoot(), true); $deviceForBuilds->delete($deviceForBuilds->getRoot(), true); $deviceForCache->delete($deviceForCache->getRoot(), true); From a4bdc23aef1a5df0850c40961ac082986667e898 Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Wed, 23 Oct 2024 12:19:07 +0200 Subject: [PATCH 03/25] Add more endpoints and models for sites --- .../Modules/Sites/Http/Sites/CreateSite.php | 2 +- .../Modules/Sites/Http/Sites/GetSite.php | 53 ++++ .../Sites/Http/Sites/ListFrameworks.php | 63 +++++ .../Modules/Sites/Http/Sites/ListSites.php | 93 +++++++ .../Modules/Sites/Http/Sites/UpdateSite.php | 231 ++++++++++++++++++ .../Platform/Modules/Sites/Services/Http.php | 8 + .../Database/Validator/Queries/Sites.php | 26 ++ src/Appwrite/Utopia/Response.php | 5 + .../Utopia/Response/Model/Framework.php | 66 +++++ 9 files changed, 546 insertions(+), 1 deletion(-) create mode 100644 src/Appwrite/Platform/Modules/Sites/Http/Sites/GetSite.php create mode 100644 src/Appwrite/Platform/Modules/Sites/Http/Sites/ListFrameworks.php create mode 100644 src/Appwrite/Platform/Modules/Sites/Http/Sites/ListSites.php create mode 100644 src/Appwrite/Platform/Modules/Sites/Http/Sites/UpdateSite.php create mode 100644 src/Appwrite/Utopia/Database/Validator/Queries/Sites.php create mode 100644 src/Appwrite/Utopia/Response/Model/Framework.php diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Sites/CreateSite.php b/src/Appwrite/Platform/Modules/Sites/Http/Sites/CreateSite.php index 086aeddc35..476aad070e 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Sites/CreateSite.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Sites/CreateSite.php @@ -79,7 +79,7 @@ class CreateSite extends Base Config::getParam('framework-specifications', []), App::getEnv('_APP_SITES_CPUS', APP_SITE_CPUS_DEFAULT), App::getEnv('_APP_SITES_MEMORY', APP_SITE_MEMORY_DEFAULT) - ), 'Runtime specification for the site and builds.', true, ['plan']) + ), 'Framework specification for the site and builds.', true, ['plan']) ->inject('request') ->inject('response') ->inject('dbForProject') diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Sites/GetSite.php b/src/Appwrite/Platform/Modules/Sites/Http/Sites/GetSite.php new file mode 100644 index 0000000000..03c6e390e4 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Sites/Http/Sites/GetSite.php @@ -0,0 +1,53 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/sites/:siteId') + ->desc('Get site') + ->groups(['api', 'sites']) + ->label('scope', 'functions.read') // TODO: Update scope to sites.read + ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) + ->label('sdk.namespace', 'sites') + ->label('sdk.method', 'get') + ->label('sdk.description', '/docs/references/sites/get-site.md') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_SITE) + ->param('siteId', '', new UID(), 'Site ID.') + ->inject('response') + ->inject('dbForProject') + ->callback([$this, 'action']); + } + + public function action(string $siteId, Response $response, Database $dbForProject) + { + $site = $dbForProject->getDocument('sites', $siteId); + + if ($site->isEmpty()) { + throw new Exception(Exception::SITE_NOT_FOUND); + } + + $response->dynamic($site, Response::MODEL_SITE); + } +} diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Sites/ListFrameworks.php b/src/Appwrite/Platform/Modules/Sites/Http/Sites/ListFrameworks.php new file mode 100644 index 0000000000..e3e5aaa3e2 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Sites/Http/Sites/ListFrameworks.php @@ -0,0 +1,63 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/sites/frameworks') + ->desc('List frameworks') + ->groups(['api', 'sites']) + ->label('scope', 'functions.read') // TODO: Update scope to sites.read + ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) + ->label('sdk.namespace', 'sites') + ->label('sdk.method', 'listFrameworks') + ->label('sdk.description', '/docs/references/sites/list-frameworks.md') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_FRAMEWORK_LIST) + ->inject('response') + ->callback([$this, 'action']); + } + + public function action(Response $response) + { + $frameworks = Config::getParam('frameworks'); + + $allowList = \array_filter(\explode(',', System::getEnv('_APP_SITES_FRAMEWORKS', ''))); + + $allowed = []; + foreach ($frameworks as $id => $framework) { + if (!empty($allowList) && !\in_array($id, $allowList)) { + continue; + } + + $framework['$id'] = $id; + $allowed[] = $framework; + } + + $response->dynamic(new Document([ + 'total' => count($allowed), + 'frameworks' => $allowed + ]), Response::MODEL_FRAMEWORK_LIST); + } +} diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Sites/ListSites.php b/src/Appwrite/Platform/Modules/Sites/Http/Sites/ListSites.php new file mode 100644 index 0000000000..df3715f3bd --- /dev/null +++ b/src/Appwrite/Platform/Modules/Sites/Http/Sites/ListSites.php @@ -0,0 +1,93 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/sites') + ->desc('List sites') + ->groups(['api', 'sites']) + ->label('scope', 'functions.write') // TODO: Update scope to sites.write + ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) + ->label('sdk.namespace', 'sites') + ->label('sdk.method', 'list') + ->label('sdk.description', '/docs/references/sites/list-sites.md') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_SITE_LIST) + ->param('queries', [], new Sites(), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long. You may filter on the following attributes: ' . implode(', ', Sites::ALLOWED_ATTRIBUTES), true) + ->param('search', '', new Text(256), 'Search term to filter your list results. Max length: 256 chars.', true) + ->inject('response') + ->inject('dbForProject') + ->callback([$this, 'action']); + } + + public function action(array $queries, string $search, Response $response, Database $dbForProject) + { + try { + $queries = Query::parseQueries($queries); + } catch (QueryException $e) { + throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); + } + + if (!empty($search)) { + $queries[] = Query::search('search', $search); + } + + /** + * Get cursor document if there was a cursor query, we use array_filter and reset for reference $cursor to $queries + */ + $cursor = \array_filter($queries, function ($query) { + return \in_array($query->getMethod(), [Query::TYPE_CURSOR_AFTER, Query::TYPE_CURSOR_BEFORE]); + }); + $cursor = reset($cursor); + if ($cursor) { + /** @var Query $cursor */ + + $validator = new Cursor(); + if (!$validator->isValid($cursor)) { + throw new Exception(Exception::GENERAL_QUERY_INVALID, $validator->getDescription()); + } + + $siteId = $cursor->getValue(); + $cursorDocument = $dbForProject->getDocument('sites', $siteId); + + if ($cursorDocument->isEmpty()) { + throw new Exception(Exception::GENERAL_CURSOR_NOT_FOUND, "Site '{$siteId}' for the 'cursor' value not found."); + } + + $cursor->setValue($cursorDocument); + } + + $filterQueries = Query::groupByType($queries)['filters']; + + $response->dynamic(new Document([ + 'sites' => $dbForProject->find('sites', $queries), + 'total' => $dbForProject->count('sites', $filterQueries, APP_LIMIT_COUNT), + ]), Response::MODEL_SITE_LIST); + } +} diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Sites/UpdateSite.php b/src/Appwrite/Platform/Modules/Sites/Http/Sites/UpdateSite.php new file mode 100644 index 0000000000..67b47f00fc --- /dev/null +++ b/src/Appwrite/Platform/Modules/Sites/Http/Sites/UpdateSite.php @@ -0,0 +1,231 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_PUT) + ->setHttpPath('/v1/sites/:siteId') + ->desc('Update site') + ->groups(['api', 'sites']) + ->label('scope', 'functions.write') // TODO: update it to sites.write later + ->label('event', 'sites.[siteId].update') + ->label('audits.event', 'sites.update') + ->label('audits.resource', 'site/{response.$id}') + ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) + ->label('sdk.namespace', 'sites') + ->label('sdk.method', 'update') + ->label('sdk.description', '/docs/references/sites/update-site.md') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_SITE) + ->param('siteId', '', new UID(), 'Site ID.') + ->param('name', '', new Text(128), 'Site name. Max length: 128 chars.') + ->param('framework', '', new WhiteList(Config::getParam('frameworks'), true), 'Sites framework.') + ->param('enabled', true, new Boolean(), 'Is site enabled? When set to \'disabled\', users cannot access the site but Server SDKs with and API key can still access the site. No data is lost when this is toggled.', true) // TODO: Add logging param later + ->param('installCommand', '', new Text(8192, 0), 'Install Command.', true) + ->param('buildCommand', '', new Text(8192, 0), 'Build Command.', true) + ->param('outputDirectory', '', new Text(8192, 0), 'Output Directory for site.', true) + ->param('fallbackRedirect', '', new Text(8192, 0), 'Fallback Redirect URL for site in case a route is not found.', true) + ->param('scopes', [], new ArrayList(new WhiteList(array_keys(Config::getParam('scopes')), true), APP_LIMIT_ARRAY_PARAMS_SIZE), 'List of scopes allowed for API key auto-generated for every execution. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' scopes are allowed.', true) //TODO: Update description of scopes + ->param('installationId', '', new Text(128, 0), 'Appwrite Installation ID for VCS (Version Control System) deployment.', true) + ->param('providerRepositoryId', '', new Text(128, 0), 'Repository ID of the repo linked to the site.', true) + ->param('providerBranch', '', new Text(128, 0), 'Production branch for the repo linked to the site.', true) + ->param('providerSilentMode', false, new Boolean(), 'Is the VCS (Version Control System) connection in silent mode for the repo linked to the site? In silent mode, comments will not be made on commits and pull requests.', true) + ->param('providerRootDirectory', '', new Text(128, 0), 'Path to site code in the linked repo.', true) + ->param('specification', APP_SITE_SPECIFICATION_DEFAULT, fn (array $plan) => new FrameworkSpecification( + $plan, + Config::getParam('framework-specifications', []), + App::getEnv('_APP_SITES_CPUS', APP_SITE_CPUS_DEFAULT), + App::getEnv('_APP_SITES_MEMORY', APP_SITE_MEMORY_DEFAULT) + ), 'Framework specification for the site and builds.', true, ['plan']) + ->inject('request') + ->inject('response') + ->inject('dbForProject') + ->inject('project') + ->inject('queueForEvents') + ->inject('queueForBuilds') + ->inject('dbForConsole') + ->inject('gitHub') + ->callback([$this, 'action']); + } + + public function action(string $siteId, string $name, string $framework, bool $enabled, string $installCommand, string $buildCommand, string $outputDirectory, string $fallbackRedirect, array $scopes, string $installationId, ?string $providerRepositoryId, string $providerBranch, bool $providerSilentMode, string $providerRootDirectory, string $specification, Request $request, Response $response, Database $dbForProject, Document $project, Event $queueForEvents, Build $queueForBuilds, Database $dbForConsole, GitHub $github) + { + // TODO: If only branch changes, re-deploy + $site = $dbForProject->getDocument('sites', $siteId); + + if ($site->isEmpty()) { + throw new Exception(Exception::SITE_NOT_FOUND); + } + + $installation = $dbForConsole->getDocument('installations', $installationId); + + if (!empty($installationId) && $installation->isEmpty()) { + throw new Exception(Exception::INSTALLATION_NOT_FOUND); + } + + if (!empty($providerRepositoryId) && (empty($installationId) || empty($providerBranch))) { + throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'When connecting to VCS (Version Control System), you need to provide "installationId" and "providerBranch".'); + } + + if ($site->isEmpty()) { + throw new Exception(Exception::SITE_NOT_FOUND); + } + + if (empty($framework)) { + $framework = $site->getAttribute('framework'); + } + + $enabled ??= $site->getAttribute('enabled', true); + + $repositoryId = $site->getAttribute('repositoryId', ''); + $repositoryInternalId = $site->getAttribute('repositoryInternalId', ''); + + $isConnected = !empty($site->getAttribute('providerRepositoryId', '')); + + // Git disconnect logic. Disconnecting only when providerRepositoryId is empty, allowing for continue updates without disconnecting git + if ($isConnected && ($providerRepositoryId !== null && empty($providerRepositoryId))) { + $repositories = $dbForConsole->find('repositories', [ + Query::equal('projectInternalId', [$project->getInternalId()]), + Query::equal('resourceInternalId', [$site->getInternalId()]), + Query::equal('resourceType', ['site']), + Query::limit(100), + ]); + + foreach ($repositories as $repository) { + $dbForConsole->deleteDocument('repositories', $repository->getId()); + } + + $providerRepositoryId = ''; + $installationId = ''; + $providerBranch = ''; + $providerRootDirectory = ''; + $providerSilentMode = true; + $repositoryId = ''; + $repositoryInternalId = ''; + } + + // Git connect logic + if (!$isConnected && !empty($providerRepositoryId)) { + $teamId = $project->getAttribute('teamId', ''); + + $repository = $dbForConsole->createDocument('repositories', new Document([ + '$id' => ID::unique(), + '$permissions' => [ + Permission::read(Role::team(ID::custom($teamId))), + Permission::update(Role::team(ID::custom($teamId), 'owner')), + Permission::update(Role::team(ID::custom($teamId), 'developer')), + Permission::delete(Role::team(ID::custom($teamId), 'owner')), + Permission::delete(Role::team(ID::custom($teamId), 'developer')), + ], + 'installationId' => $installation->getId(), + 'installationInternalId' => $installation->getInternalId(), + 'projectId' => $project->getId(), + 'projectInternalId' => $project->getInternalId(), + 'providerRepositoryId' => $providerRepositoryId, + 'resourceId' => $site->getId(), + 'resourceInternalId' => $site->getInternalId(), + 'resourceType' => 'site', + 'providerPullRequestIds' => [] + ])); + + $repositoryId = $repository->getId(); + $repositoryInternalId = $repository->getInternalId(); + } + + $live = true; + + if ( + $site->getAttribute('name') !== $name || + $site->getAttribute('buildCommand') !== $buildCommand || + $site->getAttribute('installCommand') !== $installCommand || + $site->getAttribute('outputDirectory') !== $outputDirectory || + $site->getAttribute('fallbackRedirect') !== $fallbackRedirect || + $site->getAttribute('providerRootDirectory') !== $providerRootDirectory || + $site->getAttribute('framework') !== $framework + ) { + $live = false; + } + + $spec = Config::getParam('framework-specifications')[$specification] ?? []; + + // Enforce Cold Start if spec limits change. + if ($site->getAttribute('specification') !== $specification && !empty($site->getAttribute('deploymentId'))) { + $executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); + try { + $executor->deleteRuntime($project->getId(), $site->getAttribute('deploymentId')); + } catch (\Throwable $th) { + // Don't throw if the deployment doesn't exist + if ($th->getCode() !== 404) { + throw $th; + } + } + } + + $site = $dbForProject->updateDocument('sites', $site->getId(), new Document(array_merge($site->getArrayCopy(), [ + 'name' => $name, + 'framework' => $framework, + 'enabled' => $enabled, + 'live' => $live, + 'buildCommand' => $buildCommand, + 'installCommand' => $installCommand, + 'outputDirectory' => $outputDirectory, + 'fallbackRedirect' => $fallbackRedirect, + 'scopes' => $scopes, + 'installationId' => $installation->getId(), + 'installationInternalId' => $installation->getInternalId(), + 'providerRepositoryId' => $providerRepositoryId, + 'repositoryId' => $repositoryId, + 'repositoryInternalId' => $repositoryInternalId, + 'providerBranch' => $providerBranch, + 'providerRootDirectory' => $providerRootDirectory, + 'providerSilentMode' => $providerSilentMode, + 'specification' => $specification, + 'search' => implode(' ', [$siteId, $name, $framework]), + ]))); + + // Redeploy logic + if (!$isConnected && !empty($providerRepositoryId)) { + $this->redeployVcsFunction($request, $site, $project, $installation, $dbForProject, $queueForBuilds, new Document(), $github); + } + + $queueForEvents->setParam('siteId', $site->getId()); + + $response->dynamic($site, Response::MODEL_SITE); + } +} diff --git a/src/Appwrite/Platform/Modules/Sites/Services/Http.php b/src/Appwrite/Platform/Modules/Sites/Services/Http.php index d2d2641cb5..fcfd122c02 100644 --- a/src/Appwrite/Platform/Modules/Sites/Services/Http.php +++ b/src/Appwrite/Platform/Modules/Sites/Services/Http.php @@ -4,6 +4,10 @@ namespace Appwrite\Platform\Modules\Sites\Services; use Appwrite\Platform\Modules\Sites\Http\Deployments\CreateDeployment; use Appwrite\Platform\Modules\Sites\Http\Sites\CreateSite; +use Appwrite\Platform\Modules\Sites\Http\Sites\GetSite; +use Appwrite\Platform\Modules\Sites\Http\Sites\ListFrameworks; +use Appwrite\Platform\Modules\Sites\Http\Sites\ListSites; +use Appwrite\Platform\Modules\Sites\Http\Sites\UpdateSite; use Utopia\Platform\Service; class Http extends Service @@ -12,6 +16,10 @@ class Http extends Service { $this->type = Service::TYPE_HTTP; $this->addAction(CreateSite::getName(), new CreateSite()); + $this->addAction(GetSite::getName(), new GetSite()); + $this->addAction(ListSites::getName(), new ListSites()); + $this->addAction(UpdateSite::getName(), new UpdateSite()); + $this->addAction(ListFrameworks::getName(), new ListFrameworks()); $this->addAction(CreateDeployment::getName(), new CreateDeployment()); } } diff --git a/src/Appwrite/Utopia/Database/Validator/Queries/Sites.php b/src/Appwrite/Utopia/Database/Validator/Queries/Sites.php new file mode 100644 index 0000000000..35d4bdb5ef --- /dev/null +++ b/src/Appwrite/Utopia/Database/Validator/Queries/Sites.php @@ -0,0 +1,26 @@ +setModel(new BaseList('Installations List', self::MODEL_INSTALLATION_LIST, 'installations', self::MODEL_INSTALLATION)) ->setModel(new BaseList('Provider Repositories List', self::MODEL_PROVIDER_REPOSITORY_LIST, 'providerRepositories', self::MODEL_PROVIDER_REPOSITORY)) ->setModel(new BaseList('Branches List', self::MODEL_BRANCH_LIST, 'branches', self::MODEL_BRANCH)) + ->setModel(new BaseList('Frameworks List', self::MODEL_FRAMEWORK_LIST, 'frameworks', self::MODEL_FRAMEWORK)) ->setModel(new BaseList('Runtimes List', self::MODEL_RUNTIME_LIST, 'runtimes', self::MODEL_RUNTIME)) ->setModel(new BaseList('Deployments List', self::MODEL_DEPLOYMENT_LIST, 'deployments', self::MODEL_DEPLOYMENT)) ->setModel(new BaseList('Executions List', self::MODEL_EXECUTION_LIST, 'executions', self::MODEL_EXECUTION)) @@ -439,6 +443,7 @@ class Response extends SwooleResponse ->setModel(new VcsContent()) ->setModel(new Branch()) ->setModel(new Runtime()) + ->setModel(new Framework()) ->setModel(new Deployment()) ->setModel(new Execution()) ->setModel(new Build()) diff --git a/src/Appwrite/Utopia/Response/Model/Framework.php b/src/Appwrite/Utopia/Response/Model/Framework.php new file mode 100644 index 0000000000..dcdf31ad1f --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/Framework.php @@ -0,0 +1,66 @@ +addRule('$id', [ + 'type' => self::TYPE_STRING, + 'description' => 'Framework ID.', + 'default' => '', + 'example' => 'sveltekit', + ]) + ->addRule('key', [ + 'type' => self::TYPE_STRING, + 'description' => 'Parent framework key.', + 'default' => '', + 'example' => 'sveltekit', + ]) + ->addRule('name', [ + 'type' => self::TYPE_STRING, + 'description' => 'Framework Name.', + 'default' => '', + 'example' => 'SvelteKit' + ]) + ->addRule('logo', [ + 'type' => self::TYPE_STRING, + 'description' => 'Name of the logo image.', + 'default' => '', + 'example' => 'sveltekit.png', + ]) + ->addRule('supports', [ + 'type' => self::TYPE_STRING, + 'description' => 'List of supported architectures.', + 'default' => '', + 'example' => 'amd64', + 'array' => true, + ]) + ; + } + + /** + * Get Name + * + * @return string + */ + public function getName(): string + { + return 'Framework'; + } + + /** + * Get Type + * + * @return string + */ + public function getType(): string + { + return Response::MODEL_FRAMEWORK; + } +} From 9793cf4ece402c98ec36341d79f3166b3d3841fa Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Wed, 23 Oct 2024 17:07:28 +0200 Subject: [PATCH 04/25] Add all deployment endpoints for sites --- .../Http/Deployments/CancelDeployment.php | 124 ++++++++++++++++++ .../Http/Deployments/CreateDeployment.php | 3 +- .../Http/Deployments/DeleteDeployment.php | 96 ++++++++++++++ .../Http/Deployments/DownloadDeployment.php | 115 ++++++++++++++++ .../Sites/Http/Deployments/GetDeployment.php | 70 ++++++++++ .../Http/Deployments/ListDeployments.php | 116 ++++++++++++++++ .../Http/Deployments/RebuildDeployment.php | 99 ++++++++++++++ .../Http/Deployments/UpdateDeployment.php | 83 ++++++++++++ .../Platform/Modules/Sites/Services/Http.php | 14 ++ 9 files changed, 718 insertions(+), 2 deletions(-) create mode 100644 src/Appwrite/Platform/Modules/Sites/Http/Deployments/CancelDeployment.php create mode 100644 src/Appwrite/Platform/Modules/Sites/Http/Deployments/DeleteDeployment.php create mode 100644 src/Appwrite/Platform/Modules/Sites/Http/Deployments/DownloadDeployment.php create mode 100644 src/Appwrite/Platform/Modules/Sites/Http/Deployments/GetDeployment.php create mode 100644 src/Appwrite/Platform/Modules/Sites/Http/Deployments/ListDeployments.php create mode 100644 src/Appwrite/Platform/Modules/Sites/Http/Deployments/RebuildDeployment.php create mode 100644 src/Appwrite/Platform/Modules/Sites/Http/Deployments/UpdateDeployment.php diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/CancelDeployment.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/CancelDeployment.php new file mode 100644 index 0000000000..ef6acefdc5 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/CancelDeployment.php @@ -0,0 +1,124 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_PATCH) + ->setHttpPath('/v1/sites/:siteId/deployments/:deploymentId/build') + ->desc('Cancel deployment') + ->groups(['api', 'sites']) + ->label('scope', 'functions.write') //TODO: Update the scope to sites later + ->label('audits.event', 'deployment.update') + ->label('audits.resource', 'site/{request.siteId}') + ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) + ->label('sdk.namespace', 'sites') + ->label('sdk.method', 'updateDeploymentBuild') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_BUILD) + ->param('siteId', '', new UID(), 'Site ID.') + ->param('deploymentId', '', new UID(), 'Deployment ID.') + ->inject('response') + ->inject('dbForProject') + ->inject('project') + ->inject('queueForEvents') + ->callback([$this, 'action']); + } + + public function action(string $siteId, string $deploymentId, Response $response, Database $dbForProject, Document $project, Event $queueForEvents) + { + $site = $dbForProject->getDocument('sites', $siteId); + + if ($site->isEmpty()) { + throw new Exception(Exception::SITE_NOT_FOUND); + } + + $deployment = $dbForProject->getDocument('deployments', $deploymentId); + + if ($deployment->isEmpty()) { + throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); + } + + $build = Authorization::skip(fn () => $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', ''))); + + if ($build->isEmpty()) { + $buildId = ID::unique(); + $build = $dbForProject->createDocument('builds', new Document([ + '$id' => $buildId, + '$permissions' => [], + 'startTime' => DateTime::now(), + 'deploymentInternalId' => $deployment->getInternalId(), + 'deploymentId' => $deployment->getId(), + 'status' => 'canceled', + 'path' => '', + 'runtime' => $site->getAttribute('framework'), + 'source' => $deployment->getAttribute('path', ''), + 'sourceType' => '', + 'logs' => '', + 'duration' => 0, + 'size' => 0 + ])); + + $deployment->setAttribute('buildId', $build->getId()); + $deployment->setAttribute('buildInternalId', $build->getInternalId()); + $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment); + } else { + if (\in_array($build->getAttribute('status'), ['ready', 'failed'])) { + throw new Exception(Exception::BUILD_ALREADY_COMPLETED); + } + + $startTime = new \DateTime($build->getAttribute('startTime')); + $endTime = new \DateTime('now'); + $duration = $endTime->getTimestamp() - $startTime->getTimestamp(); + + $build = $dbForProject->updateDocument('builds', $build->getId(), $build->setAttributes([ + 'endTime' => DateTime::now(), + 'duration' => $duration, + 'status' => 'canceled' + ])); + } + + $dbForProject->purgeCachedDocument('deployments', $deployment->getId()); + + try { + $executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST')); + $executor->deleteRuntime($project->getId(), $deploymentId . "-build"); + } catch (\Throwable $th) { + // Don't throw if the deployment doesn't exist + if ($th->getCode() !== 404) { + throw $th; + } + } + + $queueForEvents + ->setParam('siteId', $site->getId()) + ->setParam('deploymentId', $deployment->getId()); + + $response->dynamic($build, Response::MODEL_BUILD); + } +} diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/CreateDeployment.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/CreateDeployment.php index b710305dc0..b35708fb50 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/CreateDeployment.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/CreateDeployment.php @@ -65,7 +65,6 @@ class CreateDeployment extends Action ->inject('response') ->inject('dbForProject') ->inject('queueForEvents') - ->inject('project') ->inject('deviceForSites') ->inject('deviceForFunctions') // TODO: Remove this later once volume is added to executor ->inject('deviceForLocal') @@ -73,7 +72,7 @@ class CreateDeployment extends Action ->callback([$this, 'action']); } - public function action(string $siteId, ?string $installCommand, ?string $buildCommand, ?string $outputDirectory, mixed $code, mixed $activate, Request $request, Response $response, Database $dbForProject, Event $queueForEvents, Document $project, Device $deviceForSites, Device $deviceForFunctions, Device $deviceForLocal, Build $queueForBuilds) + public function action(string $siteId, ?string $installCommand, ?string $buildCommand, ?string $outputDirectory, mixed $code, mixed $activate, Request $request, Response $response, Database $dbForProject, Event $queueForEvents, Device $deviceForSites, Device $deviceForFunctions, Device $deviceForLocal, Build $queueForBuilds) { $activate = \strval($activate) === 'true' || \strval($activate) === '1'; diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/DeleteDeployment.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/DeleteDeployment.php new file mode 100644 index 0000000000..313870bb97 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/DeleteDeployment.php @@ -0,0 +1,96 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_DELETE) + ->setHttpPath('/v1/sites/:siteId/deployments/:deploymentId') + ->desc('Delete deployment') + ->groups(['api', 'sites']) + ->label('scope', 'functions.write') //TODO: Update the scope to sites later + ->label('event', 'sites.[siteId].deployments.[deploymentId].delete') + ->label('audits.event', 'deployment.delete') + ->label('audits.resource', 'site/{request.siteId}') + ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) + ->label('sdk.namespace', 'sites') + ->label('sdk.method', 'deleteDeployment') + ->label('sdk.description', '/docs/references/sites/delete-deployment.md') + ->label('sdk.response.code', Response::STATUS_CODE_NOCONTENT) + ->label('sdk.response.model', Response::MODEL_NONE) + ->param('siteId', '', new UID(), 'Site ID.') + ->param('deploymentId', '', new UID(), 'Deployment ID.') + ->inject('response') + ->inject('dbForProject') + ->inject('queueForDeletes') + ->inject('queueForEvents') + ->inject('deviceForSites') + ->inject('deviceForFunctions') //TODO: remove it later + ->callback([$this, 'action']); + } + + public function action(string $siteId, string $deploymentId, Response $response, Database $dbForProject, Delete $queueForDeletes, Event $queueForEvents, Device $deviceForSites, Device $deviceForFunctions) + { + $site = $dbForProject->getDocument('sites', $siteId); + if ($site->isEmpty()) { + throw new Exception(Exception::SITE_NOT_FOUND); + } + + $deployment = $dbForProject->getDocument('deployments', $deploymentId); + if ($deployment->isEmpty()) { + throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); + } + + if ($deployment->getAttribute('resourceId') !== $site->getId()) { + throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); + } + + if (!$dbForProject->deleteDocument('deployments', $deployment->getId())) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove deployment from DB'); + } + + if (!empty($deployment->getAttribute('path', ''))) { + if (!($deviceForFunctions->delete($deployment->getAttribute('path', '')))) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove deployment from storage'); + } + } + + if ($site->getAttribute('deployment') === $deployment->getId()) { // Reset site deployment + $site = $dbForProject->updateDocument('sites', $site->getId(), new Document(array_merge($site->getArrayCopy(), [ + 'deployment' => '', + 'deploymentInternalId' => '', + ]))); + } + + $queueForEvents + ->setParam('siteId', $site->getId()) + ->setParam('deploymentId', $deployment->getId()); + + $queueForDeletes + ->setType(DELETE_TYPE_DOCUMENT) + ->setDocument($deployment); + + $response->noContent(); + } +} diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/DownloadDeployment.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/DownloadDeployment.php new file mode 100644 index 0000000000..0d748514b1 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/DownloadDeployment.php @@ -0,0 +1,115 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/sites/:siteId/deployments/:deploymentId/download') + ->desc('Download deployment') + ->groups(['api', 'sites']) + ->label('scope', 'functions.read') //TODO: Update the scope to sites later + ->label('sdk.auth', [APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT]) + ->label('sdk.namespace', 'sites') + ->label('sdk.method', 'getDeploymentDownload') + ->label('sdk.description', '/docs/references/sites/get-deployment-download.md') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', '*/*') + ->label('sdk.methodType', 'location') + ->param('siteId', '', new UID(), 'Site ID.') + ->param('deploymentId', '', new UID(), 'Deployment ID.') + ->inject('response') + ->inject('request') + ->inject('dbForProject') + ->inject('deviceForSites') + ->inject('deviceForFunctions') //TODO: Remove this later + ->callback([$this, 'action']); + } + + public function action(string $siteId, string $deploymentId, Response $response, Request $request, Database $dbForProject, Device $deviceForSites, Device $deviceForFunctions) + { + $site = $dbForProject->getDocument('sites', $siteId); + if ($site->isEmpty()) { + throw new Exception(Exception::SITE_NOT_FOUND); + } + + $deployment = $dbForProject->getDocument('deployments', $deploymentId); + if ($deployment->isEmpty()) { + throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); + } + + if ($deployment->getAttribute('resourceId') !== $site->getId()) { + throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); + } + + $path = $deployment->getAttribute('path', ''); + if (!$deviceForFunctions->exists($path)) { + throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); + } + + $response + ->setContentType('application/gzip') + ->addHeader('Expires', \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)) . ' GMT') // 45 days cache + ->addHeader('X-Peak', \memory_get_peak_usage()) + ->addHeader('Content-Disposition', 'attachment; filename="' . $deploymentId . '.tar.gz"'); + + $size = $deviceForFunctions->getFileSize($path); + $rangeHeader = $request->getHeader('range'); + + if (!empty($rangeHeader)) { + $start = $request->getRangeStart(); + $end = $request->getRangeEnd(); + $unit = $request->getRangeUnit(); + + if ($end === null) { + $end = min(($start + MAX_OUTPUT_CHUNK_SIZE - 1), ($size - 1)); + } + + if ($unit !== 'bytes' || $start >= $end || $end >= $size) { + throw new Exception(Exception::STORAGE_INVALID_RANGE); + } + + $response + ->addHeader('Accept-Ranges', 'bytes') + ->addHeader('Content-Range', 'bytes ' . $start . '-' . $end . '/' . $size) + ->addHeader('Content-Length', $end - $start + 1) + ->setStatusCode(Response::STATUS_CODE_PARTIALCONTENT); + + $response->send($deviceForFunctions->read($path, $start, ($end - $start + 1))); + } + + if ($size > APP_STORAGE_READ_BUFFER) { + for ($i = 0; $i < ceil($size / MAX_OUTPUT_CHUNK_SIZE); $i++) { + $response->chunk( + $deviceForFunctions->read( + $path, + ($i * MAX_OUTPUT_CHUNK_SIZE), + min(MAX_OUTPUT_CHUNK_SIZE, $size - ($i * MAX_OUTPUT_CHUNK_SIZE)) + ), + (($i + 1) * MAX_OUTPUT_CHUNK_SIZE) >= $size + ); + } + } else { + $response->send($deviceForFunctions->read($path)); + } + } +} diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/GetDeployment.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/GetDeployment.php new file mode 100644 index 0000000000..b83aa75c6e --- /dev/null +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/GetDeployment.php @@ -0,0 +1,70 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/sites/:siteId/deployments/:deploymentId') + ->desc('Get deployment') + ->groups(['api', 'sites']) + ->label('scope', 'functions.read') //TODO: Update the scope to sites later + ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) + ->label('sdk.namespace', 'sites') + ->label('sdk.method', 'getDeployment') + ->label('sdk.description', '/docs/references/sites/get-deployment.md') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_DEPLOYMENT) + ->param('siteId', '', new UID(), 'Site ID.') + ->param('deploymentId', '', new UID(), 'Deployment ID.') + ->inject('response') + ->inject('dbForProject') + ->callback([$this, 'action']); + } + + public function action(string $siteId, string $deploymentId, Response $response, Database $dbForProject) + { + $site = $dbForProject->getDocument('sites', $siteId); + + if ($site->isEmpty()) { + throw new Exception(Exception::SITE_NOT_FOUND); + } + + $deployment = $dbForProject->getDocument('deployments', $deploymentId); + + if ($deployment->getAttribute('resourceId') !== $site->getId()) { + throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); + } + + if ($deployment->isEmpty()) { + throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); + } + + $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', '')); + $deployment->setAttribute('status', $build->getAttribute('status', 'waiting')); + $deployment->setAttribute('buildLogs', $build->getAttribute('logs', '')); + $deployment->setAttribute('buildTime', $build->getAttribute('duration', 0)); + $deployment->setAttribute('buildSize', $build->getAttribute('size', 0)); + $deployment->setAttribute('size', $deployment->getAttribute('size', 0)); + + $response->dynamic($deployment, Response::MODEL_DEPLOYMENT); + } +} diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/ListDeployments.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/ListDeployments.php new file mode 100644 index 0000000000..2d2adb9572 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/ListDeployments.php @@ -0,0 +1,116 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/sites/:siteId/deployments') + ->desc('List deployments') + ->groups(['api', 'sites']) + ->label('scope', 'functions.read') //TODO: Update the scope to sites later + ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) + ->label('sdk.namespace', 'sites') + ->label('sdk.method', 'listDeployments') + ->label('sdk.description', '/docs/references/sites/list-deployments.md') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_DEPLOYMENT_LIST) + ->param('siteId', '', new UID(), 'Site ID.') + ->param('queries', [], new Deployments(), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long. You may filter on the following attributes: ' . implode(', ', Deployments::ALLOWED_ATTRIBUTES), true) + ->param('search', '', new Text(256), 'Search term to filter your list results. Max length: 256 chars.', true) + ->inject('response') + ->inject('dbForProject') + ->callback([$this, 'action']); + } + + public function action(string $siteId, array $queries, string $search, Response $response, Database $dbForProject) + { + $site = $dbForProject->getDocument('sites', $siteId); + + if ($site->isEmpty()) { + throw new Exception(Exception::SITE_NOT_FOUND); + } + + try { + $queries = Query::parseQueries($queries); + } catch (QueryException $e) { + throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage()); + } + + if (!empty($search)) { + $queries[] = Query::search('search', $search); + } + + // Set resource queries + $queries[] = Query::equal('resourceInternalId', [$site->getInternalId()]); + $queries[] = Query::equal('resourceType', ['sites']); + + /** + * Get cursor document if there was a cursor query, we use array_filter and reset for reference $cursor to $queries + */ + $cursor = \array_filter($queries, function ($query) { + return \in_array($query->getMethod(), [Query::TYPE_CURSOR_AFTER, Query::TYPE_CURSOR_BEFORE]); + }); + $cursor = reset($cursor); + if ($cursor) { + /** @var Query $cursor */ + + $validator = new Cursor(); + if (!$validator->isValid($cursor)) { + throw new Exception(Exception::GENERAL_QUERY_INVALID, $validator->getDescription()); + } + + $deploymentId = $cursor->getValue(); + $cursorDocument = $dbForProject->getDocument('deployments', $deploymentId); + + if ($cursorDocument->isEmpty()) { + throw new Exception(Exception::GENERAL_CURSOR_NOT_FOUND, "Deployment '{$deploymentId}' for the 'cursor' value not found."); + } + + $cursor->setValue($cursorDocument); + } + + $filterQueries = Query::groupByType($queries)['filters']; + + $results = $dbForProject->find('deployments', $queries); + $total = $dbForProject->count('deployments', $filterQueries, APP_LIMIT_COUNT); + + foreach ($results as $result) { + $build = $dbForProject->getDocument('builds', $result->getAttribute('buildId', '')); + $result->setAttribute('status', $build->getAttribute('status', 'processing')); + $result->setAttribute('buildLogs', $build->getAttribute('logs', '')); + $result->setAttribute('buildTime', $build->getAttribute('duration', 0)); + $result->setAttribute('buildSize', $build->getAttribute('size', 0)); + $result->setAttribute('size', $result->getAttribute('size', 0)); + } + + $response->dynamic(new Document([ + 'deployments' => $results, + 'total' => $total, + ]), Response::MODEL_DEPLOYMENT_LIST); + } +} diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/RebuildDeployment.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/RebuildDeployment.php new file mode 100644 index 0000000000..ee09cf9fd0 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/RebuildDeployment.php @@ -0,0 +1,99 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_POST) + ->setHttpPath('/v1/sites/:siteId/deployments/:deploymentId/build') + ->desc('Rebuild deployment') + ->groups(['api', 'sites']) + ->label('scope', 'functions.write') //TODO: Update the scope to sites later + ->label('event', 'sites.[siteId].deployments.[deploymentId].update') + ->label('audits.event', 'deployment.update') + ->label('audits.resource', 'site/{request.siteId}') + ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) + ->label('sdk.namespace', 'sites') + ->label('sdk.method', 'createBuild') + ->label('sdk.response.code', Response::STATUS_CODE_NOCONTENT) + ->label('sdk.response.model', Response::MODEL_NONE) + ->param('siteId', '', new UID(), 'Site ID.') + ->param('deploymentId', '', new UID(), 'Deployment ID.') + ->inject('response') + ->inject('dbForProject') + ->inject('queueForEvents') + ->inject('queueForBuilds') + ->inject('deviceForSites') + ->inject('deviceForFunctions') //TODO: remove it later + ->callback([$this, 'action']); + } + + public function action(string $siteId, string $deploymentId, Response $response, Database $dbForProject, Event $queueForEvents, Build $queueForBuilds, Device $deviceForSites, Device $deviceForFunctions) + { + $site = $dbForProject->getDocument('sites', $siteId); + + if ($site->isEmpty()) { + throw new Exception(Exception::SITE_NOT_FOUND); + } + $deployment = $dbForProject->getDocument('deployments', $deploymentId); + + if ($deployment->isEmpty()) { + throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); + } + + $path = $deployment->getAttribute('path'); + if (empty($path) || !$deviceForFunctions->exists($path)) { + throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); + } + + $deploymentId = ID::unique(); + + $destination = $deviceForFunctions->getPath($deploymentId . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION)); + $deviceForFunctions->transfer($path, $destination, $deviceForFunctions); + + $deployment->removeAttribute('$internalId'); + $deployment = $dbForProject->createDocument('deployments', $deployment->setAttributes([ + '$internalId' => '', + '$id' => $deploymentId, + 'buildId' => '', + 'buildInternalId' => '', + 'path' => $destination, + 'buildCommand' => $site->getAttribute('buildCommand', ''), + 'installCommand' => $site->getAttribute('installCommand', ''), + 'outputDirectory' => $site->getAttribute('outputDirectory', ''), + 'search' => implode(' ', [$deploymentId]), + ])); + + $queueForBuilds + ->setType(BUILD_TYPE_DEPLOYMENT) + ->setResource($site) + ->setDeployment($deployment); + + $queueForEvents + ->setParam('siteId', $site->getId()) + ->setParam('deploymentId', $deployment->getId()); + + $response->noContent(); + } +} diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/UpdateDeployment.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/UpdateDeployment.php new file mode 100644 index 0000000000..ea798a1513 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/UpdateDeployment.php @@ -0,0 +1,83 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_PATCH) + ->setHttpPath('/v1/sites/:siteId/deployments/:deploymentId') + ->desc('Update deployment') + ->groups(['api', 'sites']) + ->label('scope', 'functions.write') //TODO: Update the scope to sites later + ->label('event', 'sites.[siteId].deployments.[deploymentId].update') + ->label('audits.event', 'deployment.update') + ->label('audits.resource', 'site/{request.siteId}') + ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) + ->label('sdk.namespace', 'sites') + ->label('sdk.method', 'updateDeployment') + ->label('sdk.description', '/docs/references/sites/update-site-deployment.md') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_SITE) + ->param('siteId', '', new UID(), 'Site ID.') + ->param('deploymentId', '', new UID(), 'Deployment ID.') + ->inject('response') + ->inject('dbForProject') + ->inject('queueForEvents') + ->inject('dbForConsole') + ->callback([$this, 'action']); + } + + public function action(string $siteId, string $deploymentId, Response $response, Database $dbForProject, Event $queueForEvents, Database $dbForConsole) + { + $site = $dbForProject->getDocument('sites', $siteId); + $deployment = $dbForProject->getDocument('deployments', $deploymentId); + $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId', '')); + + if ($site->isEmpty()) { + throw new Exception(Exception::SITE_NOT_FOUND); + } + + if ($deployment->isEmpty()) { + throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); + } + + if ($build->isEmpty()) { + throw new Exception(Exception::BUILD_NOT_FOUND); + } + + if ($build->getAttribute('status') !== 'ready') { + throw new Exception(Exception::BUILD_NOT_READY); + } + + $site = $dbForProject->updateDocument('sites', $site->getId(), new Document(array_merge($site->getArrayCopy(), [ + 'deploymentInternalId' => $deployment->getInternalId(), + 'deploymentId' => $deployment->getId(), + ]))); + + $queueForEvents + ->setParam('siteId', $site->getId()) + ->setParam('deploymentId', $deployment->getId()); + + $response->dynamic($site, Response::MODEL_SITE); + } +} diff --git a/src/Appwrite/Platform/Modules/Sites/Services/Http.php b/src/Appwrite/Platform/Modules/Sites/Services/Http.php index fcfd122c02..7a66646ab3 100644 --- a/src/Appwrite/Platform/Modules/Sites/Services/Http.php +++ b/src/Appwrite/Platform/Modules/Sites/Services/Http.php @@ -2,7 +2,14 @@ namespace Appwrite\Platform\Modules\Sites\Services; +use Appwrite\Platform\Modules\Sites\Http\Deployments\CancelDeployment; use Appwrite\Platform\Modules\Sites\Http\Deployments\CreateDeployment; +use Appwrite\Platform\Modules\Sites\Http\Deployments\DeleteDeployment; +use Appwrite\Platform\Modules\Sites\Http\Deployments\DownloadDeployment; +use Appwrite\Platform\Modules\Sites\Http\Deployments\GetDeployment; +use Appwrite\Platform\Modules\Sites\Http\Deployments\ListDeployments; +use Appwrite\Platform\Modules\Sites\Http\Deployments\RebuildDeployment; +use Appwrite\Platform\Modules\Sites\Http\Deployments\UpdateDeployment; use Appwrite\Platform\Modules\Sites\Http\Sites\CreateSite; use Appwrite\Platform\Modules\Sites\Http\Sites\GetSite; use Appwrite\Platform\Modules\Sites\Http\Sites\ListFrameworks; @@ -21,5 +28,12 @@ class Http extends Service $this->addAction(UpdateSite::getName(), new UpdateSite()); $this->addAction(ListFrameworks::getName(), new ListFrameworks()); $this->addAction(CreateDeployment::getName(), new CreateDeployment()); + $this->addAction(GetDeployment::getName(), new GetDeployment()); + $this->addAction(ListDeployments::getName(), new ListDeployments()); + $this->addAction(UpdateDeployment::getName(), new UpdateDeployment()); + $this->addAction(DeleteDeployment::getName(), new DeleteDeployment()); + $this->addAction(DownloadDeployment::getName(), new DownloadDeployment()); + $this->addAction(RebuildDeployment::getName(), new RebuildDeployment()); + $this->addAction(CancelDeployment::getName(), new CancelDeployment()); } } From 57e596fea2b93f9a89a54e5c08f5f270e9d23aa5 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Wed, 23 Oct 2024 18:36:26 +0200 Subject: [PATCH 05/25] feat: static runtime for site --- app/controllers/general.php | 54 ++++++++++++++----- .../Modules/Functions/Workers/Builds.php | 14 ++++- 2 files changed, 52 insertions(+), 16 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 483fa88a12..62cc3b9001 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -100,9 +100,9 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo $type = $route->getAttribute('resourceType'); - if ($type === 'function' || $type === 'sites') { + if ($type === 'function' || $type === 'site') { $isFunction = $type === 'function' ; - $isSite = $type === 'sites'; + $isSite = $type === 'site'; $utopia->getRoute()?->label('sdk.namespace', 'functions'); $utopia->getRoute()?->label('sdk.method', 'createExecution'); @@ -132,6 +132,7 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo $project = Authorization::skip(fn () => $dbForConsole->getDocument('projects', $projectId)); + /** @var Database $dbForProject */ $dbForProject = $getProjectDB($project); $function = Authorization::skip(fn () => $dbForProject->getDocument($isSite ? 'sites' : 'functions', $resourceId)); @@ -146,11 +147,28 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo $runtime = (isset($runtimes[$function->getAttribute('runtime', '')])) ? $runtimes[$function->getAttribute('runtime', '')] : null; - if (\is_null($runtime)) { - throw new AppwriteException(AppwriteException::FUNCTION_RUNTIME_UNSUPPORTED, 'Runtime "' . $function->getAttribute('runtime', '') . '" is not supported'); + // todo: fallback for static sites runtime + if ($isSite) { + $runtime = [ + 'key' => 'static-for-now', + 'name' => 'Static', + 'logo' => 'node.png', + 'startCommand' => null, + 'version' => 'v1', + 'base' => 'static:1.0', + 'image' => 'static:1.0', + 'supports' => [System::X86, System::ARM64, System::ARMV7, System::ARMV8] + ]; } - $deployment = Authorization::skip(fn () => $dbForProject->getDocument('deployments', $function->getAttribute('deployment', ''))); + //todo: figure out for sites/functions + if ($isFunction) { + if (\is_null($runtime)) { + throw new AppwriteException(AppwriteException::FUNCTION_RUNTIME_UNSUPPORTED, 'Runtime "' . $function->getAttribute('runtime', '') . '" is not supported'); + } + } + $deploymentId = $isSite ? $function->getAttribute('deploymentId', '') : $function->getAttribute('deployment', ''); + $deployment = Authorization::skip(fn () => $dbForProject->getDocument('deployments', $deploymentId)); if ($deployment->getAttribute('resourceId') !== $function->getId()) { throw new AppwriteException(AppwriteException::DEPLOYMENT_NOT_FOUND, 'Deployment not found. Create a deployment before trying to execute a function'); @@ -170,10 +188,13 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo throw new AppwriteException(AppwriteException::BUILD_NOT_READY); } - $permissions = $function->getAttribute('execute'); + //todo: figure out for sites/functions + if ($isFunction) { + $permissions = $function->getAttribute('execute'); - if (!(\in_array('any', $permissions)) && !(\in_array('guests', $permissions))) { - throw new AppwriteException(AppwriteException::USER_UNAUTHORIZED, 'To execute function using domain, execute permissions must include "any" or "guests"'); + if (!(\in_array('any', $permissions)) && !(\in_array('guests', $permissions))) { + throw new AppwriteException(AppwriteException::USER_UNAUTHORIZED, 'To execute function using domain, execute permissions must include "any" or "guests"'); + } } $jwtExpiry = $function->getAttribute('timeout', 900); @@ -299,22 +320,28 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo $executor = new Executor(System::getEnv('_APP_EXECUTOR_HOST')); try { $version = $function->getAttribute('version', 'v2'); - $command = $runtime['startCommand']; - $command = $version === 'v2' ? '' : 'cp /tmp/code.tar.gz /mnt/code/code.tar.gz && nohup helpers/start.sh "' . $command . '"'; + $entrypoint = $deployment->getAttribute('entrypoint', ''); + // todo: figure out site specific settings + if ($isSite) { + $version = 'v4'; + $entrypoint = 'placeholder'; + } + $runtimeEntrypoint = $version === 'v2' ? '' : 'cp /tmp/code.tar.gz /mnt/code/code.tar.gz && nohup helpers/start.sh "' . $runtime['startCommand'] . '"'; $executionResponse = $executor->createExecution( projectId: $project->getId(), deploymentId: $deployment->getId(), body: \strlen($body) > 0 ? $body : null, variables: $vars, - timeout: $function->getAttribute('timeout', 0), + // todo: figure out timeouts for sites + timeout: $function->getAttribute('timeout', 30), image: $runtime['image'], source: $build->getAttribute('path', ''), - entrypoint: $deployment->getAttribute('entrypoint', ''), + entrypoint: $entrypoint, version: $version, path: $path, method: $method, headers: $headers, - runtimeEntrypoint: $command, + runtimeEntrypoint: $runtimeEntrypoint, cpus: $spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT, memory: $spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT, logging: $function->getAttribute('logging', true), @@ -336,7 +363,6 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo $execution->setAttribute('logs', $executionResponse['logs']); $execution->setAttribute('errors', $executionResponse['errors']); $execution->setAttribute('duration', $executionResponse['duration']); - } catch (\Throwable $th) { $durationEnd = \microtime(true); diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index 4f662f566a..312b9edcf3 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -145,6 +145,8 @@ class Builds extends Action } $version = $resource->getAttribute('version', 'v2'); + + // todo: fallback for sites if ($isSite) { $version = 'v4'; } @@ -154,6 +156,7 @@ class Builds extends Action $key = $resource->getAttribute('runtime'); $runtime = $runtimes[$key] ?? null; + // todo: fallback for sites if ($isSite) { $runtime = $runtimes['node-18.0']; } @@ -687,9 +690,16 @@ class Builds extends Action /** Set auto deploy */ if ($deployment->getAttribute('activate') === true) { $resource->setAttribute('deploymentInternalId', $deployment->getInternalId()); - $resource->setAttribute('deployment', $deployment->getId()); $resource->setAttribute('live', true); - $resource = $dbForProject->updateDocument('functions', $resource->getId(), $resource); + // todo: fix here how clean this is + if ($isSite) { + $resource->setAttribute('deploymentId', $deployment->getId()); + $resource = $dbForProject->updateDocument('sites', $resource->getId(), $resource); + } + if ($isFunction) { + $resource->setAttribute('deployment', $deployment->getId()); + $resource = $dbForProject->updateDocument('functions', $resource->getId(), $resource); + } } if ($dbForProject->getDocument('builds', $buildId)->getAttribute('status') === 'canceled') { From 8f5fa849962977349f438aede076244a3cca7505 Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Thu, 24 Oct 2024 10:49:43 +0200 Subject: [PATCH 06/25] Add variable endpoints for sites --- app/init.php | 2 +- .../Modules/Sites/Http/Sites/DeleteSite.php | 69 ++++++++++++++ .../Sites/Http/Variables/CreateVariable.php | 91 +++++++++++++++++++ .../Sites/Http/Variables/DeleteVariable.php | 68 ++++++++++++++ .../Sites/Http/Variables/GetVariable.php | 68 ++++++++++++++ .../Sites/Http/Variables/ListVariables.php | 57 ++++++++++++ .../Sites/Http/Variables/UpdateVariable.php | 82 +++++++++++++++++ .../Platform/Modules/Sites/Services/Http.php | 12 +++ 8 files changed, 448 insertions(+), 1 deletion(-) create mode 100644 src/Appwrite/Platform/Modules/Sites/Http/Sites/DeleteSite.php create mode 100644 src/Appwrite/Platform/Modules/Sites/Http/Variables/CreateVariable.php create mode 100644 src/Appwrite/Platform/Modules/Sites/Http/Variables/DeleteVariable.php create mode 100644 src/Appwrite/Platform/Modules/Sites/Http/Variables/GetVariable.php create mode 100644 src/Appwrite/Platform/Modules/Sites/Http/Variables/ListVariables.php create mode 100644 src/Appwrite/Platform/Modules/Sites/Http/Variables/UpdateVariable.php diff --git a/app/init.php b/app/init.php index 078db328ca..1b9625175a 100644 --- a/app/init.php +++ b/app/init.php @@ -558,7 +558,7 @@ Database::addFilter( return $database ->find('variables', [ Query::equal('resourceInternalId', [$document->getInternalId()]), - Query::equal('resourceType', ['function']), + Query::equal('resourceType', ['function', 'site']), Query::limit(APP_LIMIT_SUBQUERY), ]); } diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Sites/DeleteSite.php b/src/Appwrite/Platform/Modules/Sites/Http/Sites/DeleteSite.php new file mode 100644 index 0000000000..c1ac277436 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Sites/Http/Sites/DeleteSite.php @@ -0,0 +1,69 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_DELETE) + ->setHttpPath('/v1/sites/:siteId') + ->desc('Delete site') + ->groups(['api', 'sites']) + ->label('scope', 'functions.write') // TODO: Update scope to sites.write + ->label('event', 'sites.[siteId].delete') + ->label('audits.event', 'site.delete') + ->label('audits.resource', 'site/{request.siteId}') + ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) + ->label('sdk.namespace', 'sites') + ->label('sdk.method', 'delete') + ->label('sdk.description', '/docs/references/sites/delete-site.md') + ->label('sdk.response.code', Response::STATUS_CODE_NOCONTENT) + ->label('sdk.response.model', Response::MODEL_NONE) + ->param('siteId', '', new UID(), 'Site ID.') + ->inject('response') + ->inject('dbForProject') + ->inject('queueForDeletes') + ->inject('queueForEvents') + ->callback([$this, 'action']); + } + + public function action(string $siteId, Response $response, Database $dbForProject, Delete $queueForDeletes, Event $queueForEvents) + { + $site = $dbForProject->getDocument('sites', $siteId); + + if ($site->isEmpty()) { + throw new Exception(Exception::SITE_NOT_FOUND); + } + + if (!$dbForProject->deleteDocument('sites', $site->getId())) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove site from DB'); + } + + $queueForDeletes + ->setType(DELETE_TYPE_DOCUMENT) + ->setDocument($site); + + $queueForEvents->setParam('siteId', $site->getId()); + + $response->noContent(); + } +} diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Variables/CreateVariable.php b/src/Appwrite/Platform/Modules/Sites/Http/Variables/CreateVariable.php new file mode 100644 index 0000000000..fab3a953a8 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Sites/Http/Variables/CreateVariable.php @@ -0,0 +1,91 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_POST) + ->setHttpPath('/v1/sites/:siteId/variables') + ->desc('Create variable') + ->groups(['api', 'sites']) + ->label('scope', 'functions.write') // TODO: Update scope to sites.write + ->label('audits.event', 'variable.create') + ->label('audits.resource', 'site/{request.siteId}') + ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) + ->label('sdk.namespace', 'sites') + ->label('sdk.method', 'createVariable') + ->label('sdk.description', '/docs/references/sites/create-variable.md') + ->label('sdk.response.code', Response::STATUS_CODE_CREATED) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_VARIABLE) + ->param('siteId', '', new UID(), 'Site unique ID.', false) + ->param('key', null, new Text(Database::LENGTH_KEY), 'Variable key. Max length: ' . Database::LENGTH_KEY . ' chars.', false) + ->param('value', null, new Text(8192, 0), 'Variable value. Max length: 8192 chars.', false) + ->inject('response') + ->inject('dbForProject') + ->inject('dbForConsole') + ->callback([$this, 'action']); + } + + public function action(string $siteId, string $key, string $value, Response $response, Database $dbForProject, Database $dbForConsole) + { + $site = $dbForProject->getDocument('sites', $siteId); + + if ($site->isEmpty()) { + throw new Exception(Exception::SITE_NOT_FOUND); + } + + $variableId = ID::unique(); + + $variable = new Document([ + '$id' => $variableId, + '$permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + 'resourceInternalId' => $site->getInternalId(), + 'resourceId' => $site->getId(), + 'resourceType' => 'site', + 'key' => $key, + 'value' => $value, + 'search' => implode(' ', [$variableId, $site->getId(), $key, 'site']), + ]); + + try { + $variable = $dbForProject->createDocument('variables', $variable); + } catch (DuplicateException $th) { + throw new Exception(Exception::VARIABLE_ALREADY_EXISTS); + } + + $dbForProject->updateDocument('sites', $site->getId(), $site->setAttribute('live', false)); + + $response + ->setStatusCode(Response::STATUS_CODE_CREATED) + ->dynamic($variable, Response::MODEL_VARIABLE); + } +} diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Variables/DeleteVariable.php b/src/Appwrite/Platform/Modules/Sites/Http/Variables/DeleteVariable.php new file mode 100644 index 0000000000..45f6905763 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Sites/Http/Variables/DeleteVariable.php @@ -0,0 +1,68 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_DELETE) + ->setHttpPath('/v1/sites/:siteId/variables/:variableId') + ->desc('Delete variable') + ->groups(['api', 'sites']) + ->label('scope', 'functions.write') // TODO: Update scope to sites + ->label('audits.event', 'variable.delete') + ->label('audits.resource', 'site/{request.siteId}') + ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) + ->label('sdk.namespace', 'sites') + ->label('sdk.method', 'deleteVariable') + ->label('sdk.description', '/docs/references/sites/delete-variable.md') + ->label('sdk.response.code', Response::STATUS_CODE_NOCONTENT) + ->label('sdk.response.model', Response::MODEL_NONE) + ->param('siteId', '', new UID(), 'Site unique ID.', false) + ->param('variableId', '', new UID(), 'Variable unique ID.', false) + ->inject('response') + ->inject('dbForProject') + ->callback([$this, 'action']); + } + + public function action(string $siteId, string $variableId, Response $response, Database $dbForProject) + { + $site = $dbForProject->getDocument('sites', $siteId); + + if ($site->isEmpty()) { + throw new Exception(Exception::SITE_NOT_FOUND); + } + + $variable = $dbForProject->getDocument('variables', $variableId); + if ($variable === false || $variable->isEmpty() || $variable->getAttribute('resourceInternalId') !== $site->getInternalId() || $variable->getAttribute('resourceType') !== 'site') { + throw new Exception(Exception::VARIABLE_NOT_FOUND); + } + + if ($variable === false || $variable->isEmpty()) { + throw new Exception(Exception::VARIABLE_NOT_FOUND); + } + + $dbForProject->deleteDocument('variables', $variable->getId()); + + $dbForProject->updateDocument('sites', $site->getId(), $site->setAttribute('live', false)); + + $response->noContent(); + } +} diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Variables/GetVariable.php b/src/Appwrite/Platform/Modules/Sites/Http/Variables/GetVariable.php new file mode 100644 index 0000000000..cb9a57a2e8 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Sites/Http/Variables/GetVariable.php @@ -0,0 +1,68 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/sites/:siteId/variables/:variableId') + ->desc('Get variable') + ->groups(['api', 'sites']) + ->label('scope', 'functions.read') // TODO: Update scope to sites + ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) + ->label('sdk.namespace', 'sites') + ->label('sdk.method', 'getVariable') + ->label('sdk.description', '/docs/references/sites/get-variable.md') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_VARIABLE) + ->param('siteId', '', new UID(), 'Site unique ID.', false) + ->param('variableId', '', new UID(), 'Variable unique ID.', false) + ->inject('response') + ->inject('dbForProject') + ->callback([$this, 'action']); + } + + public function action(string $siteId, string $variableId, Response $response, Database $dbForProject) + { + $site = $dbForProject->getDocument('sites', $siteId); + + if ($site->isEmpty()) { + throw new Exception(Exception::SITE_NOT_FOUND); + } + + $variable = $dbForProject->getDocument('variables', $variableId); + if ( + $variable === false || + $variable->isEmpty() || + $variable->getAttribute('resourceInternalId') !== $site->getInternalId() || + $variable->getAttribute('resourceType') !== 'site' + ) { + throw new Exception(Exception::VARIABLE_NOT_FOUND); + } + + if ($variable === false || $variable->isEmpty()) { + throw new Exception(Exception::VARIABLE_NOT_FOUND); + } + + $response->dynamic($variable, Response::MODEL_VARIABLE); + } +} diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Variables/ListVariables.php b/src/Appwrite/Platform/Modules/Sites/Http/Variables/ListVariables.php new file mode 100644 index 0000000000..7233cb234b --- /dev/null +++ b/src/Appwrite/Platform/Modules/Sites/Http/Variables/ListVariables.php @@ -0,0 +1,57 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/sites/:siteId/variables') + ->desc('List variables') + ->groups(['api', 'sites']) + ->label('scope', 'functions.read') // TODO: Update scope to sites + ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) + ->label('sdk.namespace', 'sites') + ->label('sdk.method', 'listVariables') + ->label('sdk.description', '/docs/references/sites/list-variables.md') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_VARIABLE_LIST) + ->param('siteId', '', new UID(), 'Site unique ID.', false) + ->inject('response') + ->inject('dbForProject') + ->callback([$this, 'action']); + } + + public function action(string $siteId, Response $response, Database $dbForProject) + { + $site = $dbForProject->getDocument('sites', $siteId); + + if ($site->isEmpty()) { + throw new Exception(Exception::SITE_NOT_FOUND); + } + + $response->dynamic(new Document([ + 'variables' => $site->getAttribute('vars', []), + 'total' => \count($site->getAttribute('vars', [])), + ]), Response::MODEL_VARIABLE_LIST); + } +} diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Variables/UpdateVariable.php b/src/Appwrite/Platform/Modules/Sites/Http/Variables/UpdateVariable.php new file mode 100644 index 0000000000..abd023e182 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Sites/Http/Variables/UpdateVariable.php @@ -0,0 +1,82 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_PUT) + ->setHttpPath('/v1/sites/:siteId/variables/:variableId') + ->desc('Update variable') + ->groups(['api', 'sites']) + ->label('scope', 'functions.write') // TODO: Update scope to sites + ->label('audits.event', 'variable.update') + ->label('audits.resource', 'site/{request.siteId}') + ->label('sdk.auth', [APP_AUTH_TYPE_KEY]) + ->label('sdk.namespace', 'sites') + ->label('sdk.method', 'updateVariable') + ->label('sdk.description', '/docs/references/sites/update-variable.md') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_VARIABLE) + ->param('siteId', '', new UID(), 'Site unique ID.', false) + ->param('variableId', '', new UID(), 'Variable unique ID.', false) + ->param('key', null, new Text(255), 'Variable key. Max length: 255 chars.', false) + ->param('value', null, new Text(8192, 0), 'Variable value. Max length: 8192 chars.', true) + ->inject('response') + ->inject('dbForProject') + ->callback([$this, 'action']); + } + + public function action(string $siteId, string $variableId, string $key, ?string $value, Response $response, Database $dbForProject) + { + $site = $dbForProject->getDocument('sites', $siteId); + + if ($site->isEmpty()) { + throw new Exception(Exception::SITE_NOT_FOUND); + } + + $variable = $dbForProject->getDocument('variables', $variableId); + if ($variable === false || $variable->isEmpty() || $variable->getAttribute('resourceInternalId') !== $site->getInternalId() || $variable->getAttribute('resourceType') !== 'site') { + throw new Exception(Exception::VARIABLE_NOT_FOUND); + } + + if ($variable === false || $variable->isEmpty()) { + throw new Exception(Exception::VARIABLE_NOT_FOUND); + } + + $variable + ->setAttribute('key', $key) + ->setAttribute('value', $value ?? $variable->getAttribute('value')) + ->setAttribute('search', implode(' ', [$variableId, $site->getId(), $key, 'site'])); + + try { + $dbForProject->updateDocument('variables', $variable->getId(), $variable); + } catch (DuplicateException $th) { + throw new Exception(Exception::VARIABLE_ALREADY_EXISTS); + } + + $dbForProject->updateDocument('sites', $site->getId(), $site->setAttribute('live', false)); + + $response->dynamic($variable, Response::MODEL_VARIABLE); + } +} diff --git a/src/Appwrite/Platform/Modules/Sites/Services/Http.php b/src/Appwrite/Platform/Modules/Sites/Services/Http.php index 7a66646ab3..ee72d434f4 100644 --- a/src/Appwrite/Platform/Modules/Sites/Services/Http.php +++ b/src/Appwrite/Platform/Modules/Sites/Services/Http.php @@ -11,10 +11,16 @@ use Appwrite\Platform\Modules\Sites\Http\Deployments\ListDeployments; use Appwrite\Platform\Modules\Sites\Http\Deployments\RebuildDeployment; use Appwrite\Platform\Modules\Sites\Http\Deployments\UpdateDeployment; use Appwrite\Platform\Modules\Sites\Http\Sites\CreateSite; +use Appwrite\Platform\Modules\Sites\Http\Sites\DeleteSite; use Appwrite\Platform\Modules\Sites\Http\Sites\GetSite; use Appwrite\Platform\Modules\Sites\Http\Sites\ListFrameworks; use Appwrite\Platform\Modules\Sites\Http\Sites\ListSites; use Appwrite\Platform\Modules\Sites\Http\Sites\UpdateSite; +use Appwrite\Platform\Modules\Sites\Http\Variables\CreateVariable; +use Appwrite\Platform\Modules\Sites\Http\Variables\DeleteVariable; +use Appwrite\Platform\Modules\Sites\Http\Variables\GetVariable; +use Appwrite\Platform\Modules\Sites\Http\Variables\ListVariables; +use Appwrite\Platform\Modules\Sites\Http\Variables\UpdateVariable; use Utopia\Platform\Service; class Http extends Service @@ -26,6 +32,7 @@ class Http extends Service $this->addAction(GetSite::getName(), new GetSite()); $this->addAction(ListSites::getName(), new ListSites()); $this->addAction(UpdateSite::getName(), new UpdateSite()); + $this->addAction(DeleteSite::getName(), new DeleteSite()); $this->addAction(ListFrameworks::getName(), new ListFrameworks()); $this->addAction(CreateDeployment::getName(), new CreateDeployment()); $this->addAction(GetDeployment::getName(), new GetDeployment()); @@ -35,5 +42,10 @@ class Http extends Service $this->addAction(DownloadDeployment::getName(), new DownloadDeployment()); $this->addAction(RebuildDeployment::getName(), new RebuildDeployment()); $this->addAction(CancelDeployment::getName(), new CancelDeployment()); + $this->addAction(CreateVariable::getName(), new CreateVariable()); + $this->addAction(GetVariable::getName(), new GetVariable()); + $this->addAction(ListVariables::getName(), new ListVariables()); + $this->addAction(UpdateVariable::getName(), new UpdateVariable()); + $this->addAction(DeleteVariable::getName(), new DeleteVariable()); } } From 12bddc3b1fdd5a5c8a4a0adfea6ecd0d8671a60d Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Thu, 24 Oct 2024 10:59:47 +0200 Subject: [PATCH 07/25] Change secret type to bool --- app/controllers/api/functions.php | 4 ++-- app/controllers/api/project.php | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 5bd8a993bb..6823f0c802 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -2337,11 +2337,11 @@ App::post('/v1/functions/:functionId/variables') ->param('functionId', '', new UID(), 'Function unique ID.', false) ->param('key', null, new Text(Database::LENGTH_KEY), 'Variable key. Max length: ' . Database::LENGTH_KEY . ' chars.', false) ->param('value', null, new Text(8192, 0), 'Variable value. Max length: 8192 chars.', false) - ->param('secret', false, new Boolean(true), 'Is secret? Secret variables can only be updated or deleted, they cannot be read.', true) + ->param('secret', false, new Boolean(), 'Is secret? Secret variables can only be updated or deleted, they cannot be read.', true) ->inject('response') ->inject('dbForProject') ->inject('dbForConsole') - ->action(function (string $functionId, string $key, string $value, mixed $secret, Response $response, Database $dbForProject, Database $dbForConsole) { + ->action(function (string $functionId, string $key, string $value, bool $secret, Response $response, Database $dbForProject, Database $dbForConsole) { $function = $dbForProject->getDocument('functions', $functionId); if ($function->isEmpty()) { diff --git a/app/controllers/api/project.php b/app/controllers/api/project.php index d5957188a9..7ac49466a2 100644 --- a/app/controllers/api/project.php +++ b/app/controllers/api/project.php @@ -323,12 +323,12 @@ App::post('/v1/project/variables') ->label('sdk.response.model', Response::MODEL_VARIABLE) ->param('key', null, new Text(Database::LENGTH_KEY), 'Variable key. Max length: ' . Database::LENGTH_KEY . ' chars.', false) ->param('value', null, new Text(8192, 0), 'Variable value. Max length: 8192 chars.', false) - ->param('secret', false, new Boolean(true), 'Is secret? Secret variables can only be updated or deleted, they cannot be read.', true) + ->param('secret', false, new Boolean(), 'Is secret? Secret variables can only be updated or deleted, they cannot be read.', true) ->inject('project') ->inject('response') ->inject('dbForProject') ->inject('dbForConsole') - ->action(function (string $key, string $value, mixed $secret, Document $project, Response $response, Database $dbForProject, Database $dbForConsole) { + ->action(function (string $key, string $value, bool $secret, Document $project, Response $response, Database $dbForProject, Database $dbForConsole) { $variableId = ID::unique(); $variable = new Document([ From 8bda7f1e1efe4ba959bb719e0567d0967c04b999 Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Thu, 24 Oct 2024 11:47:09 +0200 Subject: [PATCH 08/25] Add listSiteTemplates endpoint and response models --- app/config/site-templates.php | 45 +++++++ app/controllers/api/functions.php | 2 +- app/init.php | 1 + .../Sites/Http/Sites/ListSiteTemplates.php | 71 +++++++++++ .../Platform/Modules/Sites/Services/Http.php | 2 + src/Appwrite/Utopia/Response.php | 8 ++ .../Response/Model/TemplateFramework.php | 70 +++++++++++ .../Utopia/Response/Model/TemplateSite.php | 116 ++++++++++++++++++ 8 files changed, 314 insertions(+), 1 deletion(-) create mode 100644 app/config/site-templates.php create mode 100644 src/Appwrite/Platform/Modules/Sites/Http/Sites/ListSiteTemplates.php create mode 100644 src/Appwrite/Utopia/Response/Model/TemplateFramework.php create mode 100644 src/Appwrite/Utopia/Response/Model/TemplateSite.php diff --git a/app/config/site-templates.php b/app/config/site-templates.php new file mode 100644 index 0000000000..baaf78469f --- /dev/null +++ b/app/config/site-templates.php @@ -0,0 +1,45 @@ + [ + 'name' => 'sveltekit' + ], + 'NEXTJS' => [ + 'name' => 'nextjs' + ], +]; + +function getFrameworks($framework, $installCommand, $buildCommand, $outputDirectory, $fallbackRedirect, $providerRootDirectory) +{ + return array_map(function ($version) use ($framework, $installCommand, $buildCommand, $outputDirectory, $fallbackRedirect, $providerRootDirectory) { + return [ + 'name' => $framework['name'], + 'installCommand' => $installCommand, + 'buildCommand' => $buildCommand, + 'outputDirectory' => $outputDirectory, + 'fallbackRedirect' => $fallbackRedirect, + 'providerRootDirectory' => $providerRootDirectory + ]; + }, $framework); +} + +return [ + [ + 'icon' => 'icon-lightning-bolt', + 'id' => 'starter', + 'name' => 'Starter site', + 'tagline' => + 'A simple site to get started. Edit this site to explore endless possibilities with Appwrite Sites.', + 'useCases' => ['starter'], + 'frameworks' => [ + ...getFrameworks(TEMPLATE_FRAMEWORKS['SVELTEKIT'], 'npm install', 'npm run build', 'build', 'index.html', 'node/starter') + ], + 'instructions' => 'For documentation and instructions check out file.', + 'vcsProvider' => 'github', + 'providerRepositoryId' => 'templates', + 'providerOwner' => 'appwrite', + 'providerVersion' => '0.2.*', + 'variables' => [], + 'scopes' => ['users.read'] + ] +]; diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 444e05fb21..b581f3b5f1 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -2565,7 +2565,7 @@ App::get('/v1/functions/templates') ->desc('List function templates') ->label('scope', 'public') ->label('sdk.namespace', 'functions') - ->label('sdk.method', 'listTemplates') + ->label('sdk.method', 'listTemplates') // TODO: Change to listFunctionTemplates later ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.description', '/docs/references/functions/list-templates.md') ->label('sdk.response.code', Response::STATUS_CODE_OK) diff --git a/app/init.php b/app/init.php index 1b9625175a..05824389e5 100644 --- a/app/init.php +++ b/app/init.php @@ -336,6 +336,7 @@ Config::load('storage-inputs', __DIR__ . '/config/storage/inputs.php'); Config::load('storage-outputs', __DIR__ . '/config/storage/outputs.php'); Config::load('runtime-specifications', __DIR__ . '/config/runtimes/specifications.php'); Config::load('function-templates', __DIR__ . '/config/function-templates.php'); +Config::load('site-templates', __DIR__ . '/config/site-templates.php'); /** * New DB Filters diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Sites/ListSiteTemplates.php b/src/Appwrite/Platform/Modules/Sites/Http/Sites/ListSiteTemplates.php new file mode 100644 index 0000000000..38e907267a --- /dev/null +++ b/src/Appwrite/Platform/Modules/Sites/Http/Sites/ListSiteTemplates.php @@ -0,0 +1,71 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/sites/templates') + ->desc('List site templates') + ->groups(['api']) + ->label('scope', 'public') + ->label('sdk.namespace', 'sites') + ->label('sdk.method', 'listSiteTemplates') + ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) + ->label('sdk.description', '/docs/references/sites/list-templates.md') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_TEMPLATE_SITE_LIST) + ->param('frameworks', [], new ArrayList(new WhiteList(array_keys(Config::getParam('frameworks')), true), APP_LIMIT_ARRAY_PARAMS_SIZE), 'List of frameworks allowed for filtering site templates. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' frameworks are allowed.', true) + ->param('useCases', [], new ArrayList(new WhiteList(['dev-tools', 'starter', 'databases', 'ai', 'messaging', 'utilities']), APP_LIMIT_ARRAY_PARAMS_SIZE), 'List of use cases allowed for filtering site templates. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' use cases are allowed.', true) + ->param('limit', 25, new Range(1, 5000), 'Limit the number of templates returned in the response. Default limit is 25, and maximum limit is 5000.', true) + ->param('offset', 0, new Range(0, 5000), 'Offset the list of returned templates. Maximum offset is 5000.', true) + ->inject('response') + ->callback([$this, 'action']); + } + + public function action(array $frameworks, array $usecases, int $limit, int $offset, Response $response) + { + $templates = Config::getParam('site-templates', []); + + var_dump($templates); + + if (!empty($frameworks)) { + $templates = \array_filter($templates, function ($template) use ($frameworks) { + return \count(\array_intersect($frameworks, \array_column($template['frameworks'], 'name'))) > 0; + }); + } + + if (!empty($usecases)) { + $templates = \array_filter($templates, function ($template) use ($usecases) { + return \count(\array_intersect($usecases, $template['useCases'])) > 0; + }); + } + + $responseTemplates = \array_slice($templates, $offset, $limit); + $response->dynamic(new Document([ + 'templates' => $responseTemplates, + 'total' => \count($responseTemplates), + ]), Response::MODEL_TEMPLATE_SITE_LIST); + } +} diff --git a/src/Appwrite/Platform/Modules/Sites/Services/Http.php b/src/Appwrite/Platform/Modules/Sites/Services/Http.php index ee72d434f4..92fdc8d842 100644 --- a/src/Appwrite/Platform/Modules/Sites/Services/Http.php +++ b/src/Appwrite/Platform/Modules/Sites/Services/Http.php @@ -15,6 +15,7 @@ use Appwrite\Platform\Modules\Sites\Http\Sites\DeleteSite; use Appwrite\Platform\Modules\Sites\Http\Sites\GetSite; use Appwrite\Platform\Modules\Sites\Http\Sites\ListFrameworks; use Appwrite\Platform\Modules\Sites\Http\Sites\ListSites; +use Appwrite\Platform\Modules\Sites\Http\Sites\ListSiteTemplates; use Appwrite\Platform\Modules\Sites\Http\Sites\UpdateSite; use Appwrite\Platform\Modules\Sites\Http\Variables\CreateVariable; use Appwrite\Platform\Modules\Sites\Http\Variables\DeleteVariable; @@ -47,5 +48,6 @@ class Http extends Service $this->addAction(ListVariables::getName(), new ListVariables()); $this->addAction(UpdateVariable::getName(), new UpdateVariable()); $this->addAction(DeleteVariable::getName(), new DeleteVariable()); + $this->addAction(ListSiteTemplates::getName(), new ListSiteTemplates()); } } diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index 816d72392e..e94d53abbc 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -91,8 +91,10 @@ use Appwrite\Utopia\Response\Model\Subscriber; use Appwrite\Utopia\Response\Model\Target; use Appwrite\Utopia\Response\Model\Team; use Appwrite\Utopia\Response\Model\TemplateEmail; +use Appwrite\Utopia\Response\Model\TemplateFramework; use Appwrite\Utopia\Response\Model\TemplateFunction; use Appwrite\Utopia\Response\Model\TemplateRuntime; +use Appwrite\Utopia\Response\Model\TemplateSite; use Appwrite\Utopia\Response\Model\TemplateSMS; use Appwrite\Utopia\Response\Model\TemplateVariable; use Appwrite\Utopia\Response\Model\Token; @@ -251,6 +253,9 @@ class Response extends SwooleResponse public const MODEL_SITE_LIST = 'siteList'; public const MODEL_FRAMEWORK = 'framework'; public const MODEL_FRAMEWORK_LIST = 'frameworkList'; + public const MODEL_TEMPLATE_SITE = 'templateSite'; + public const MODEL_TEMPLATE_SITE_LIST = 'templateSiteList'; + public const MODEL_TEMPLATE_FRAMEWORK = 'templateFramework'; // Functions public const MODEL_FUNCTION = 'function'; @@ -360,6 +365,7 @@ class Response extends SwooleResponse ->setModel(new BaseList('Teams List', self::MODEL_TEAM_LIST, 'teams', self::MODEL_TEAM)) ->setModel(new BaseList('Memberships List', self::MODEL_MEMBERSHIP_LIST, 'memberships', self::MODEL_MEMBERSHIP)) ->setModel(new BaseList('Sites List', self::MODEL_SITE_LIST, 'sites', self::MODEL_SITE)) + ->setModel(new BaseList('Site Templates List', self::MODEL_TEMPLATE_SITE_LIST, 'templates', self::MODEL_TEMPLATE_SITE)) ->setModel(new BaseList('Functions List', self::MODEL_FUNCTION_LIST, 'functions', self::MODEL_FUNCTION)) ->setModel(new BaseList('Function Templates List', self::MODEL_TEMPLATE_FUNCTION_LIST, 'templates', self::MODEL_TEMPLATE_FUNCTION)) ->setModel(new BaseList('Installations List', self::MODEL_INSTALLATION_LIST, 'installations', self::MODEL_INSTALLATION)) @@ -433,6 +439,8 @@ class Response extends SwooleResponse ->setModel(new Team()) ->setModel(new Membership()) ->setModel(new Site()) + ->setModel(new TemplateSite()) + ->setModel(new TemplateFramework()) ->setModel(new Func()) ->setModel(new TemplateFunction()) ->setModel(new TemplateRuntime()) diff --git a/src/Appwrite/Utopia/Response/Model/TemplateFramework.php b/src/Appwrite/Utopia/Response/Model/TemplateFramework.php new file mode 100644 index 0000000000..8acfcf0017 --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/TemplateFramework.php @@ -0,0 +1,70 @@ +addRule('name', [ + 'type' => self::TYPE_STRING, + 'description' => 'Framework Name.', + 'default' => '', + 'example' => 'sveltekit', + ]) + ->addRule('installCommand', [ + 'type' => self::TYPE_STRING, + 'description' => 'The install command used to install the dependencies.', + 'default' => '', + 'example' => 'npm install', + ]) + ->addRule('buildCommand', [ + 'type' => self::TYPE_STRING, + 'description' => 'The build command used to build the deployment.', + 'default' => '', + 'example' => 'npm run build', + ]) + ->addRule('outputDirectory', [ + 'type' => self::TYPE_STRING, + 'description' => 'The output directory to store the build output.', + 'default' => '', + 'example' => 'build', + ]) + ->addRule('fallbackRedirect', [ + 'type' => self::TYPE_STRING, + 'description' => 'The fallback redirect for the site when a route is not found.', + 'default' => '', + 'example' => 'index.html', + ]) + ->addRule('providerRootDirectory', [ + 'type' => self::TYPE_STRING, + 'description' => 'Path to site in VCS (Version Control System) repository', + 'default' => '', + 'example' => 'node/starter', + ]); + } + + /** + * Get Name + * + * @return string + */ + public function getName(): string + { + return 'Template Framework'; + } + + /** + * Get Type + * + * @return string + */ + public function getType(): string + { + return Response::MODEL_TEMPLATE_FRAMEWORK; + } +} diff --git a/src/Appwrite/Utopia/Response/Model/TemplateSite.php b/src/Appwrite/Utopia/Response/Model/TemplateSite.php new file mode 100644 index 0000000000..7eeff22076 --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/TemplateSite.php @@ -0,0 +1,116 @@ +addRule('icon', [ + 'type' => self::TYPE_STRING, + 'description' => 'Site Template Icon.', + 'default' => '', + 'example' => 'icon-lightning-bolt', + ]) + ->addRule('id', [ + 'type' => self::TYPE_STRING, + 'description' => 'Site Template ID.', + 'default' => '', + 'example' => 'starter', + ]) + ->addRule('name', [ + 'type' => self::TYPE_STRING, + 'description' => 'Site Template Name.', + 'default' => '', + 'example' => 'Starter site', + ]) + ->addRule('tagline', [ + 'type' => self::TYPE_STRING, + 'description' => 'Site Template Tagline.', + 'default' => '', + 'example' => 'A simple site to get started.', + ]) + ->addRule('useCases', [ + 'type' => self::TYPE_STRING, + 'description' => 'Site use cases.', + 'default' => [], + 'example' => 'Starter', + 'array' => true, + ]) + ->addRule('frameworks', [ + 'type' => Response::MODEL_TEMPLATE_FRAMEWORK, + 'description' => 'List of frameworks that can be used with this template.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('instructions', [ + 'type' => self::TYPE_STRING, + 'description' => 'Site Template Instructions.', + 'default' => '', + 'example' => 'For documentation and instructions check out .', + ]) + ->addRule('vcsProvider', [ + 'type' => self::TYPE_STRING, + 'description' => 'VCS (Version Control System) Provider.', + 'default' => '', + 'example' => 'github', + ]) + ->addRule('providerRepositoryId', [ + 'type' => self::TYPE_STRING, + 'description' => 'VCS (Version Control System) Repository ID', + 'default' => '', + 'example' => 'templates', + ]) + ->addRule('providerOwner', [ + 'type' => self::TYPE_STRING, + 'description' => 'VCS (Version Control System) Owner.', + 'default' => '', + 'example' => 'appwrite', + ]) + ->addRule('providerVersion', [ + 'type' => self::TYPE_STRING, + 'description' => 'VCS (Version Control System) branch version (tag).', + 'default' => '', + 'example' => 'main', + ]) + ->addRule('variables', [ + 'type' => Response::MODEL_TEMPLATE_VARIABLE, + 'description' => 'Site variables.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('scopes', [ + 'type' => self::TYPE_STRING, + 'description' => 'Site scopes.', + 'default' => [], + 'example' => 'users.read', + 'array' => true, + ]); + } + + /** + * Get Name + * + * @return string + */ + public function getName(): string + { + return 'Template Site'; + } + + /** + * Get Type + * + * @return string + */ + public function getType(): string + { + return Response::MODEL_TEMPLATE_SITE; + } +} From 2f9d00fc07abd0ad394cd6b5402b353a0fb6ffc7 Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Thu, 24 Oct 2024 12:08:33 +0200 Subject: [PATCH 09/25] Update the description for secret var --- src/Appwrite/Utopia/Response/Model/Variable.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Utopia/Response/Model/Variable.php b/src/Appwrite/Utopia/Response/Model/Variable.php index d479eb541c..22f76e44d4 100644 --- a/src/Appwrite/Utopia/Response/Model/Variable.php +++ b/src/Appwrite/Utopia/Response/Model/Variable.php @@ -44,7 +44,7 @@ class Variable extends Model ]) ->addRule('secret', [ 'type' => self::TYPE_BOOLEAN, - 'description' => 'Variable secret flag.', + 'description' => 'Variable secret flag. Secret variables can only be updated or deleted, but never read.', 'default' => false, 'example' => false, ]) From 83b5aecbfde4e9225daf76a22f2991b56e6f71b9 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Thu, 24 Oct 2024 12:19:10 +0200 Subject: [PATCH 10/25] chore: add todo comment --- app/controllers/general.php | 1 + 1 file changed, 1 insertion(+) diff --git a/app/controllers/general.php b/app/controllers/general.php index 62cc3b9001..88ece71e8a 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -167,6 +167,7 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo throw new AppwriteException(AppwriteException::FUNCTION_RUNTIME_UNSUPPORTED, 'Runtime "' . $function->getAttribute('runtime', '') . '" is not supported'); } } + //todo: find a better approach $deploymentId = $isSite ? $function->getAttribute('deploymentId', '') : $function->getAttribute('deployment', ''); $deployment = Authorization::skip(fn () => $dbForProject->getDocument('deployments', $deploymentId)); From d876085f6d39fe9b2bde6a1e4596fc2d9429151f Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Thu, 24 Oct 2024 15:21:40 +0200 Subject: [PATCH 11/25] Add usage endpoints --- app/controllers/api/functions.php | 4 +- app/controllers/shared/api.php | 9 ++ app/init.php | 7 + .../Modules/Sites/Http/Sites/GetSiteUsage.php | 129 +++++++++++++++++ .../Sites/Http/Sites/GetSitesUsage.php | 121 ++++++++++++++++ ...istSiteTemplates.php => ListTemplates.php} | 10 +- .../Platform/Modules/Sites/Services/Http.php | 8 +- src/Appwrite/Utopia/Response.php | 6 + .../Utopia/Response/Model/UsageSite.php | 119 ++++++++++++++++ .../Utopia/Response/Model/UsageSites.php | 132 ++++++++++++++++++ 10 files changed, 535 insertions(+), 10 deletions(-) create mode 100644 src/Appwrite/Platform/Modules/Sites/Http/Sites/GetSiteUsage.php create mode 100644 src/Appwrite/Platform/Modules/Sites/Http/Sites/GetSitesUsage.php rename src/Appwrite/Platform/Modules/Sites/Http/Sites/{ListSiteTemplates.php => ListTemplates.php} (93%) create mode 100644 src/Appwrite/Utopia/Response/Model/UsageSite.php create mode 100644 src/Appwrite/Utopia/Response/Model/UsageSites.php diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index b581f3b5f1..208cfea9d5 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -655,7 +655,7 @@ App::get('/v1/functions/:functionId/usage') App::get('/v1/functions/usage') ->desc('Get functions usage') - ->groups(['api', 'functions']) + ->groups(['api', 'functions', 'usage']) ->label('scope', 'functions.read') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'functions') @@ -2565,7 +2565,7 @@ App::get('/v1/functions/templates') ->desc('List function templates') ->label('scope', 'public') ->label('sdk.namespace', 'functions') - ->label('sdk.method', 'listTemplates') // TODO: Change to listFunctionTemplates later + ->label('sdk.method', 'listTemplates') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.description', '/docs/references/functions/list-templates.md') ->label('sdk.response.code', Response::STATUS_CODE_OK) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index f0d896c95a..dc35af190d 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -133,6 +133,15 @@ $databaseListener = function (string $event, Document $document, Document $proje $queueForUsage ->addMetric(METRIC_FUNCTIONS, $value); // per project + if ($event === Database::EVENT_DOCUMENT_DELETE) { + $queueForUsage + ->addReduce($document); + } + break; + case $document->getCollection() === 'sites': + $queueForUsage + ->addMetric(METRIC_SITES, $value); // per project + if ($event === Database::EVENT_DOCUMENT_DELETE) { $queueForUsage ->addReduce($document); diff --git a/app/init.php b/app/init.php index 05824389e5..3225d87c84 100644 --- a/app/init.php +++ b/app/init.php @@ -259,6 +259,7 @@ const METRIC_FILES = 'files'; const METRIC_FILES_STORAGE = 'files.storage'; const METRIC_BUCKET_ID_FILES = '{bucketInternalId}.files'; const METRIC_BUCKET_ID_FILES_STORAGE = '{bucketInternalId}.files.storage'; +const METRIC_SITES = 'sites'; const METRIC_FUNCTIONS = 'functions'; const METRIC_DEPLOYMENTS = 'deployments'; const METRIC_DEPLOYMENTS_STORAGE = 'deployments.storage'; @@ -286,6 +287,12 @@ const METRIC_EXECUTIONS_MB_SECONDS = 'executions.mbSeconds'; const METRIC_FUNCTION_ID_EXECUTIONS = '{functionInternalId}.executions'; const METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE = '{functionInternalId}.executions.compute'; const METRIC_FUNCTION_ID_EXECUTIONS_MB_SECONDS = '{functionInternalId}.executions.mbSeconds'; +const METRIC_SITE_ID_DEPLOYMENTS = '{resourceType}.{resourceInternalId}.deployments'; +const METRIC_SITE_ID_DEPLOYMENTS_STORAGE = '{resourceType}.{resourceInternalId}.deployments.storage'; +const METRIC_SITE_ID_BUILDS = '{siteInternalId}.builds'; +const METRIC_SITE_ID_BUILDS_STORAGE = '{siteInternalId}.builds.storage'; +const METRIC_SITE_ID_BUILDS_COMPUTE = '{siteInternalId}.builds.compute'; +const METRIC_SITE_ID_BUILDS_MB_SECONDS = '{siteInternalId}.builds.mbSeconds'; const METRIC_NETWORK_REQUESTS = 'network.requests'; const METRIC_NETWORK_INBOUND = 'network.inbound'; const METRIC_NETWORK_OUTBOUND = 'network.outbound'; diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Sites/GetSiteUsage.php b/src/Appwrite/Platform/Modules/Sites/Http/Sites/GetSiteUsage.php new file mode 100644 index 0000000000..a00f74a0ff --- /dev/null +++ b/src/Appwrite/Platform/Modules/Sites/Http/Sites/GetSiteUsage.php @@ -0,0 +1,129 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/sites/:siteId/usage') + ->desc('Get site usage') + ->groups(['api', 'sites', 'usage']) + ->label('scope', 'functions.read') // TODO: Update scope to sites.read + ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) + ->label('sdk.namespace', 'sites') + ->label('sdk.method', 'getSiteUsage') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_USAGE_SITE) + ->param('siteId', '', new UID(), 'Site ID.') + ->param('range', '30d', new WhiteList(['24h', '30d', '90d']), 'Date range.', true) + ->inject('response') + ->inject('dbForProject') + ->callback([$this, 'action']); + } + + public function action(string $siteId, string $range, Response $response, Database $dbForProject) + { + $site = $dbForProject->getDocument('sites', $siteId); + + if ($site->isEmpty()) { + throw new Exception(Exception::SITE_NOT_FOUND); + } + + $periods = Config::getParam('usage', []); + $stats = $usage = []; + $days = $periods[$range]; + $metrics = [ + str_replace(['{resourceType}', '{resourceInternalId}'], ['sites', $site->getInternalId()], METRIC_SITE_ID_DEPLOYMENTS), + str_replace(['{resourceType}', '{resourceInternalId}'], ['sites', $site->getInternalId()], METRIC_SITE_ID_DEPLOYMENTS_STORAGE), + str_replace('{siteInternalId}', $site->getInternalId(), METRIC_SITE_ID_BUILDS), + str_replace('{siteInternalId}', $site->getInternalId(), METRIC_SITE_ID_BUILDS_STORAGE), + str_replace('{siteInternalId}', $site->getInternalId(), METRIC_SITE_ID_BUILDS_COMPUTE), + str_replace('{siteInternalId}', $site->getInternalId(), METRIC_SITE_ID_BUILDS_MB_SECONDS) + ]; + + Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { + foreach ($metrics as $metric) { + $result = $dbForProject->findOne('stats', [ + Query::equal('metric', [$metric]), + Query::equal('period', ['inf']) + ]); + + $stats[$metric]['total'] = $result['value'] ?? 0; + $limit = $days['limit']; + $period = $days['period']; + $results = $dbForProject->find('stats', [ + Query::equal('metric', [$metric]), + Query::equal('period', [$period]), + Query::limit($limit), + Query::orderDesc('time'), + ]); + $stats[$metric]['data'] = []; + foreach ($results as $result) { + $stats[$metric]['data'][$result->getAttribute('time')] = [ + 'value' => $result->getAttribute('value'), + ]; + } + } + }); + + $format = match ($days['period']) { + '1h' => 'Y-m-d\TH:00:00.000P', + '1d' => 'Y-m-d\T00:00:00.000P', + }; + + foreach ($metrics as $metric) { + $usage[$metric]['total'] = $stats[$metric]['total']; + $usage[$metric]['data'] = []; + $leap = time() - ($days['limit'] * $days['factor']); + while ($leap < time()) { + $leap += $days['factor']; + $formatDate = date($format, $leap); + $usage[$metric]['data'][] = [ + 'value' => $stats[$metric]['data'][$formatDate]['value'] ?? 0, + 'date' => $formatDate, + ]; + } + } + + $response->dynamic(new Document([ + 'range' => $range, + 'deploymentsTotal' => $usage[$metrics[0]]['total'], + 'deploymentsStorageTotal' => $usage[$metrics[1]]['total'], + 'buildsTotal' => $usage[$metrics[2]]['total'], + 'buildsStorageTotal' => $usage[$metrics[3]]['total'], + 'buildsTimeTotal' => $usage[$metrics[4]]['total'], + 'deployments' => $usage[$metrics[0]]['data'], + 'deploymentsStorage' => $usage[$metrics[1]]['data'], + 'builds' => $usage[$metrics[2]]['data'], + 'buildsStorage' => $usage[$metrics[3]]['data'], + 'buildsTime' => $usage[$metrics[4]]['data'], + 'buildsMbSecondsTotal' => $usage[$metrics[7]]['total'], + 'buildsMbSeconds' => $usage[$metrics[7]]['data'] + // TODO: Add more metrics for requests, bandwidth, etc. + ]), Response::MODEL_USAGE_SITE); + } +} diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Sites/GetSitesUsage.php b/src/Appwrite/Platform/Modules/Sites/Http/Sites/GetSitesUsage.php new file mode 100644 index 0000000000..0c8b1f8e36 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Sites/Http/Sites/GetSitesUsage.php @@ -0,0 +1,121 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/sites/usage') + ->desc('Get sites usage') + ->groups(['api', 'sites', 'usage']) + ->label('scope', 'functions.read') // TODO: Update scope to sites.read + ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) + ->label('sdk.namespace', 'sites') + ->label('sdk.method', 'getUsage') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_USAGE_SITES) + ->param('range', '30d', new WhiteList(['24h', '30d', '90d']), 'Date range.', true) + ->inject('response') + ->inject('dbForProject') + ->callback([$this, 'action']); + } + + public function action(string $range, Response $response, Database $dbForProject) + { + $periods = Config::getParam('usage', []); + $stats = $usage = []; + $days = $periods[$range]; + $metrics = [ + METRIC_SITES, + METRIC_DEPLOYMENTS, + METRIC_DEPLOYMENTS_STORAGE, + METRIC_BUILDS, + METRIC_BUILDS_STORAGE, + METRIC_BUILDS_COMPUTE, + METRIC_BUILDS_MB_SECONDS, + ]; + + Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) { + foreach ($metrics as $metric) { + $result = $dbForProject->findOne('stats', [ + Query::equal('metric', [$metric]), + Query::equal('period', ['inf']) + ]); + + $stats[$metric]['total'] = $result['value'] ?? 0; + $limit = $days['limit']; + $period = $days['period']; + $results = $dbForProject->find('stats', [ + Query::equal('metric', [$metric]), + Query::equal('period', [$period]), + Query::limit($limit), + Query::orderDesc('time'), + ]); + $stats[$metric]['data'] = []; + foreach ($results as $result) { + $stats[$metric]['data'][$result->getAttribute('time')] = [ + 'value' => $result->getAttribute('value'), + ]; + } + } + }); + + $format = match ($days['period']) { + '1h' => 'Y-m-d\TH:00:00.000P', + '1d' => 'Y-m-d\T00:00:00.000P', + }; + + foreach ($metrics as $metric) { + $usage[$metric]['total'] = $stats[$metric]['total']; + $usage[$metric]['data'] = []; + $leap = time() - ($days['limit'] * $days['factor']); + while ($leap < time()) { + $leap += $days['factor']; + $formatDate = date($format, $leap); + $usage[$metric]['data'][] = [ + 'value' => $stats[$metric]['data'][$formatDate]['value'] ?? 0, + 'date' => $formatDate, + ]; + } + } + $response->dynamic(new Document([ + 'range' => $range, + 'sitesTotal' => $usage[$metrics[0]]['total'], + 'deploymentsTotal' => $usage[$metrics[1]]['total'], + 'deploymentsStorageTotal' => $usage[$metrics[2]]['total'], + 'buildsTotal' => $usage[$metrics[3]]['total'], + 'buildsStorageTotal' => $usage[$metrics[4]]['total'], + 'buildsTimeTotal' => $usage[$metrics[5]]['total'], + 'sites' => $usage[$metrics[0]]['data'], + 'deployments' => $usage[$metrics[1]]['data'], + 'deploymentsStorage' => $usage[$metrics[2]]['data'], + 'builds' => $usage[$metrics[3]]['data'], + 'buildsStorage' => $usage[$metrics[4]]['data'], + 'buildsTime' => $usage[$metrics[5]]['data'], + 'buildsMbSecondsTotal' => $usage[$metrics[8]]['total'], + 'buildsMbSeconds' => $usage[$metrics[8]]['data'] + ]), Response::MODEL_USAGE_SITES); + } +} diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Sites/ListSiteTemplates.php b/src/Appwrite/Platform/Modules/Sites/Http/Sites/ListTemplates.php similarity index 93% rename from src/Appwrite/Platform/Modules/Sites/Http/Sites/ListSiteTemplates.php rename to src/Appwrite/Platform/Modules/Sites/Http/Sites/ListTemplates.php index 38e907267a..a8c3de555b 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Sites/ListSiteTemplates.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Sites/ListTemplates.php @@ -12,13 +12,13 @@ use Utopia\Validator\ArrayList; use Utopia\Validator\Range; use Utopia\Validator\WhiteList; -class ListSiteTemplates extends Base +class ListTemplates extends Base { use HTTP; public static function getName() { - return 'listSiteTemplates'; + return 'listTemplates'; } public function __construct() @@ -26,11 +26,11 @@ class ListSiteTemplates extends Base $this ->setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) ->setHttpPath('/v1/sites/templates') - ->desc('List site templates') + ->desc('List templates') ->groups(['api']) ->label('scope', 'public') ->label('sdk.namespace', 'sites') - ->label('sdk.method', 'listSiteTemplates') + ->label('sdk.method', 'listTemplates') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.description', '/docs/references/sites/list-templates.md') ->label('sdk.response.code', Response::STATUS_CODE_OK) @@ -48,8 +48,6 @@ class ListSiteTemplates extends Base { $templates = Config::getParam('site-templates', []); - var_dump($templates); - if (!empty($frameworks)) { $templates = \array_filter($templates, function ($template) use ($frameworks) { return \count(\array_intersect($frameworks, \array_column($template['frameworks'], 'name'))) > 0; diff --git a/src/Appwrite/Platform/Modules/Sites/Services/Http.php b/src/Appwrite/Platform/Modules/Sites/Services/Http.php index 92fdc8d842..4739c5aebd 100644 --- a/src/Appwrite/Platform/Modules/Sites/Services/Http.php +++ b/src/Appwrite/Platform/Modules/Sites/Services/Http.php @@ -13,9 +13,11 @@ use Appwrite\Platform\Modules\Sites\Http\Deployments\UpdateDeployment; use Appwrite\Platform\Modules\Sites\Http\Sites\CreateSite; use Appwrite\Platform\Modules\Sites\Http\Sites\DeleteSite; use Appwrite\Platform\Modules\Sites\Http\Sites\GetSite; +use Appwrite\Platform\Modules\Sites\Http\Sites\GetSitesUsage; +use Appwrite\Platform\Modules\Sites\Http\Sites\GetSiteUsage; use Appwrite\Platform\Modules\Sites\Http\Sites\ListFrameworks; use Appwrite\Platform\Modules\Sites\Http\Sites\ListSites; -use Appwrite\Platform\Modules\Sites\Http\Sites\ListSiteTemplates; +use Appwrite\Platform\Modules\Sites\Http\Sites\ListTemplates; use Appwrite\Platform\Modules\Sites\Http\Sites\UpdateSite; use Appwrite\Platform\Modules\Sites\Http\Variables\CreateVariable; use Appwrite\Platform\Modules\Sites\Http\Variables\DeleteVariable; @@ -48,6 +50,8 @@ class Http extends Service $this->addAction(ListVariables::getName(), new ListVariables()); $this->addAction(UpdateVariable::getName(), new UpdateVariable()); $this->addAction(DeleteVariable::getName(), new DeleteVariable()); - $this->addAction(ListSiteTemplates::getName(), new ListSiteTemplates()); + $this->addAction(ListTemplates::getName(), new ListTemplates()); + $this->addAction(GetSiteUsage::getName(), new GetSiteUsage()); + $this->addAction(GetSitesUsage::getName(), new GetSitesUsage()); } } diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index e94d53abbc..05b6ada061 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -106,6 +106,8 @@ use Appwrite\Utopia\Response\Model\UsageDatabases; use Appwrite\Utopia\Response\Model\UsageFunction; use Appwrite\Utopia\Response\Model\UsageFunctions; use Appwrite\Utopia\Response\Model\UsageProject; +use Appwrite\Utopia\Response\Model\UsageSite; +use Appwrite\Utopia\Response\Model\UsageSites; use Appwrite\Utopia\Response\Model\UsageStorage; use Appwrite\Utopia\Response\Model\UsageUsers; use Appwrite\Utopia\Response\Model\User; @@ -144,6 +146,8 @@ class Response extends SwooleResponse public const MODEL_USAGE_STORAGE = 'usageStorage'; public const MODEL_USAGE_FUNCTIONS = 'usageFunctions'; public const MODEL_USAGE_FUNCTION = 'usageFunction'; + public const MODEL_USAGE_SITES = 'usageSites'; + public const MODEL_USAGE_SITE = 'usageSite'; public const MODEL_USAGE_PROJECT = 'usageProject'; // Database @@ -483,6 +487,8 @@ class Response extends SwooleResponse ->setModel(new UsageBuckets()) ->setModel(new UsageFunctions()) ->setModel(new UsageFunction()) + ->setModel(new UsageSites()) + ->setModel(new UsageSite()) ->setModel(new UsageProject()) ->setModel(new Headers()) ->setModel(new Specification()) diff --git a/src/Appwrite/Utopia/Response/Model/UsageSite.php b/src/Appwrite/Utopia/Response/Model/UsageSite.php new file mode 100644 index 0000000000..266cc4cade --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/UsageSite.php @@ -0,0 +1,119 @@ +addRule('range', [ + 'type' => self::TYPE_STRING, + 'description' => 'The time range of the usage stats.', + 'default' => '', + 'example' => '30d', + ]) + ->addRule('deploymentsTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of site deployments.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('deploymentsStorageTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated sum of site deployments storage.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('buildsTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of site builds.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('buildsStorageTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'total aggregated sum of site builds storage.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('buildsTimeTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated sum of site builds compute time.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('buildsMbSecondsTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated sum of site builds mbSeconds.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('deployments', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of site deployments per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('deploymentsStorage', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of site deployments storage per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('builds', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of site builds per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('buildsStorage', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated sum of site builds storage per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('buildsTime', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated sum of site builds compute time per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('buildsMbSeconds', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of site builds mbSeconds per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ; + } + + /** + * Get Name + * + * @return string + */ + public function getName(): string + { + return 'UsageSite'; + } + + /** + * Get Type + * + * @return string + */ + public function getType(): string + { + return Response::MODEL_USAGE_SITE; + } +} diff --git a/src/Appwrite/Utopia/Response/Model/UsageSites.php b/src/Appwrite/Utopia/Response/Model/UsageSites.php new file mode 100644 index 0000000000..8425b8cb74 --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/UsageSites.php @@ -0,0 +1,132 @@ +addRule('range', [ + 'type' => self::TYPE_STRING, + 'description' => 'Time range of the usage stats.', + 'default' => '', + 'example' => '30d', + ]) + ->addRule('sitesTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of sites.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('deploymentsTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of sites deployments.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('deploymentsStorageTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated sum of sites deployment storage.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('buildsTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated number of sites build.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('buildsStorageTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'total aggregated sum of sites build storage.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('buildsTimeTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated sum of sites build compute time.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('buildsMbSecondsTotal', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'Total aggregated sum of sites build mbSeconds.', + 'default' => 0, + 'example' => 0, + ]) + ->addRule('sites', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of sites per period.', + 'default' => 0, + 'example' => 0, + 'array' => true + ]) + ->addRule('deployments', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of sites deployment per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('deploymentsStorage', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of sites deployment storage per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('builds', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated number of sites build per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('buildsStorage', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated sum of sites build storage per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('buildsTime', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated sum of sites build compute time per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ->addRule('buildsMbSeconds', [ + 'type' => Response::MODEL_METRIC, + 'description' => 'Aggregated sum of sites build mbSeconds per period.', + 'default' => [], + 'example' => [], + 'array' => true + ]) + ; + } + + /** + * Get Name + * + * @return string + */ + public function getName(): string + { + return 'UsageSites'; + } + + /** + * Get Type + * + * @return string + */ + public function getType(): string + { + return Response::MODEL_USAGE_SITES; + } +} From 765d8241f27b3f21dc91c5f069fff06a4c587a86 Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Thu, 24 Oct 2024 16:32:53 +0200 Subject: [PATCH 12/25] Add secret param to site variables --- .../Platform/Modules/Sites/Http/Variables/CreateVariable.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Variables/CreateVariable.php b/src/Appwrite/Platform/Modules/Sites/Http/Variables/CreateVariable.php index fab3a953a8..b0d7983a1d 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Variables/CreateVariable.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Variables/CreateVariable.php @@ -14,6 +14,7 @@ use Utopia\Database\Helpers\Role; use Utopia\Database\Validator\UID; use Utopia\Platform\Action; use Utopia\Platform\Scope\HTTP; +use Utopia\Validator\Boolean; use Utopia\Validator\Text; class CreateVariable extends Base @@ -45,13 +46,14 @@ class CreateVariable extends Base ->param('siteId', '', new UID(), 'Site unique ID.', false) ->param('key', null, new Text(Database::LENGTH_KEY), 'Variable key. Max length: ' . Database::LENGTH_KEY . ' chars.', false) ->param('value', null, new Text(8192, 0), 'Variable value. Max length: 8192 chars.', false) + ->param('secret', false, new Boolean(), 'Is secret? Secret variables can only be updated or deleted, they cannot be read.', true) ->inject('response') ->inject('dbForProject') ->inject('dbForConsole') ->callback([$this, 'action']); } - public function action(string $siteId, string $key, string $value, Response $response, Database $dbForProject, Database $dbForConsole) + public function action(string $siteId, string $key, string $value, bool $secret, Response $response, Database $dbForProject, Database $dbForConsole) { $site = $dbForProject->getDocument('sites', $siteId); @@ -73,6 +75,7 @@ class CreateVariable extends Base 'resourceType' => 'site', 'key' => $key, 'value' => $value, + 'secret' => $secret, 'search' => implode(' ', [$variableId, $site->getId(), $key, 'site']), ]); From bbd589f6a8a4eb5f626696e09cd712c50c78ca2b Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Thu, 24 Oct 2024 18:02:26 +0200 Subject: [PATCH 13/25] Add download build endpoint to sites --- docker-compose.yml | 1 + .../Sites/Http/Deployments/DownloadBuild.php | 119 ++++++++++++++++++ .../Platform/Modules/Sites/Services/Http.php | 2 + 3 files changed, 122 insertions(+) create mode 100644 src/Appwrite/Platform/Modules/Sites/Http/Deployments/DownloadBuild.php diff --git a/docker-compose.yml b/docker-compose.yml index b0292045ca..91f39ff944 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -76,6 +76,7 @@ services: - appwrite-config:/storage/config:rw - appwrite-certificates:/storage/certificates:rw - appwrite-functions:/storage/functions:rw + - appwrite-builds:/storage/builds:rw - ./phpunit.xml:/usr/src/code/phpunit.xml - ./tests:/usr/src/code/tests - ./app:/usr/src/code/app diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Deployments/DownloadBuild.php b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/DownloadBuild.php new file mode 100644 index 0000000000..caa6d2389e --- /dev/null +++ b/src/Appwrite/Platform/Modules/Sites/Http/Deployments/DownloadBuild.php @@ -0,0 +1,119 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/sites/:siteId/deployments/:deploymentId/build/download') + ->desc('Download build') + ->groups(['api', 'sites']) + ->label('scope', 'functions.read') //TODO: Update the scope to sites later + ->label('sdk.auth', [APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT]) + ->label('sdk.namespace', 'sites') + ->label('sdk.method', 'getBuildDownload') + ->label('sdk.description', '/docs/references/sites/get-build-download.md') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', '*/*') + ->label('sdk.methodType', 'location') + ->param('siteId', '', new UID(), 'Site ID.') + ->param('deploymentId', '', new UID(), 'Deployment ID.') + ->inject('response') + ->inject('request') + ->inject('dbForProject') + ->inject('deviceForBuilds') + ->callback([$this, 'action']); + } + + public function action(string $siteId, string $deploymentId, Response $response, Request $request, Database $dbForProject, Device $deviceForBuilds) + { + $site = $dbForProject->getDocument('sites', $siteId); + if ($site->isEmpty()) { + throw new Exception(Exception::SITE_NOT_FOUND); + } + + $deployment = $dbForProject->getDocument('deployments', $deploymentId); + if ($deployment->isEmpty()) { + throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); + } + + if ($deployment->getAttribute('resourceId') !== $site->getId()) { + throw new Exception(Exception::DEPLOYMENT_NOT_FOUND); + } + + $build = $dbForProject->getDocument('builds', $deployment->getAttribute('buildId')); + if ($build->isEmpty()) { + throw new Exception(Exception::BUILD_NOT_FOUND); + } + + $path = $build->getAttribute('path', ''); + if (!$deviceForBuilds->exists($path)) { + throw new Exception(Exception::BUILD_NOT_FOUND); + } + + $response + ->setContentType('application/gzip') + ->addHeader('Expires', \date('D, d M Y H:i:s', \time() + (60 * 60 * 24 * 45)) . ' GMT') // 45 days cache + ->addHeader('X-Peak', \memory_get_peak_usage()) + ->addHeader('Content-Disposition', 'attachment; filename="' . $deploymentId . '.tar.gz"'); + + $size = $deviceForBuilds->getFileSize($path); + $rangeHeader = $request->getHeader('range'); + + if (!empty($rangeHeader)) { + $start = $request->getRangeStart(); + $end = $request->getRangeEnd(); + $unit = $request->getRangeUnit(); + + if ($end === null) { + $end = min(($start + MAX_OUTPUT_CHUNK_SIZE - 1), ($size - 1)); + } + + if ($unit !== 'bytes' || $start >= $end || $end >= $size) { + throw new Exception(Exception::STORAGE_INVALID_RANGE); + } + + $response + ->addHeader('Accept-Ranges', 'bytes') + ->addHeader('Content-Range', 'bytes ' . $start . '-' . $end . '/' . $size) + ->addHeader('Content-Length', $end - $start + 1) + ->setStatusCode(Response::STATUS_CODE_PARTIALCONTENT); + + $response->send($deviceForBuilds->read($path, $start, ($end - $start + 1))); + } + + if ($size > APP_STORAGE_READ_BUFFER) { + for ($i = 0; $i < ceil($size / MAX_OUTPUT_CHUNK_SIZE); $i++) { + $response->chunk( + $deviceForBuilds->read( + $path, + ($i * MAX_OUTPUT_CHUNK_SIZE), + min(MAX_OUTPUT_CHUNK_SIZE, $size - ($i * MAX_OUTPUT_CHUNK_SIZE)) + ), + (($i + 1) * MAX_OUTPUT_CHUNK_SIZE) >= $size + ); + } + } else { + $response->send($deviceForBuilds->read($path)); + } + } +} diff --git a/src/Appwrite/Platform/Modules/Sites/Services/Http.php b/src/Appwrite/Platform/Modules/Sites/Services/Http.php index 4739c5aebd..7552f16c4d 100644 --- a/src/Appwrite/Platform/Modules/Sites/Services/Http.php +++ b/src/Appwrite/Platform/Modules/Sites/Services/Http.php @@ -5,6 +5,7 @@ namespace Appwrite\Platform\Modules\Sites\Services; use Appwrite\Platform\Modules\Sites\Http\Deployments\CancelDeployment; use Appwrite\Platform\Modules\Sites\Http\Deployments\CreateDeployment; use Appwrite\Platform\Modules\Sites\Http\Deployments\DeleteDeployment; +use Appwrite\Platform\Modules\Sites\Http\Deployments\DownloadBuild; use Appwrite\Platform\Modules\Sites\Http\Deployments\DownloadDeployment; use Appwrite\Platform\Modules\Sites\Http\Deployments\GetDeployment; use Appwrite\Platform\Modules\Sites\Http\Deployments\ListDeployments; @@ -43,6 +44,7 @@ class Http extends Service $this->addAction(UpdateDeployment::getName(), new UpdateDeployment()); $this->addAction(DeleteDeployment::getName(), new DeleteDeployment()); $this->addAction(DownloadDeployment::getName(), new DownloadDeployment()); + $this->addAction(DownloadBuild::getName(), new DownloadBuild()); $this->addAction(RebuildDeployment::getName(), new RebuildDeployment()); $this->addAction(CancelDeployment::getName(), new CancelDeployment()); $this->addAction(CreateVariable::getName(), new CreateVariable()); From b16169f86768c7e257060c45c5412a35befaa58a Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Thu, 24 Oct 2024 18:09:43 +0200 Subject: [PATCH 14/25] Add secret to templateVariable model --- src/Appwrite/Utopia/Response/Model/TemplateVariable.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Appwrite/Utopia/Response/Model/TemplateVariable.php b/src/Appwrite/Utopia/Response/Model/TemplateVariable.php index c992083a87..e196b29032 100644 --- a/src/Appwrite/Utopia/Response/Model/TemplateVariable.php +++ b/src/Appwrite/Utopia/Response/Model/TemplateVariable.php @@ -28,6 +28,12 @@ class TemplateVariable extends Model 'default' => '', 'example' => '512', ]) + ->addRule('secret', [ + 'type' => self::TYPE_BOOLEAN, + 'description' => 'Variable secret flag. Secret variables can only be updated or deleted, but never read.', + 'default' => false, + 'example' => false, + ]) ->addRule('placeholder', [ 'type' => self::TYPE_STRING, 'description' => 'Variable Placeholder.', From 54d3668693860c1f3eacb908c38354f8ee65bc1a Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Thu, 24 Oct 2024 19:20:00 +0200 Subject: [PATCH 15/25] fix: todos for sites --- app/controllers/general.php | 144 +++++----- app/init.php | 10 +- .../Modules/Functions/Workers/Builds.php | 262 +++++++++++------- 3 files changed, 243 insertions(+), 173 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 88ece71e8a..adc2c82f49 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -54,19 +54,19 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo $host = $request->getHostname() ?? ''; - $route = Authorization::skip( + $rule = Authorization::skip( fn () => $dbForConsole->find('rules', [ Query::equal('domain', [$host]), Query::limit(1) ]) )[0] ?? null; - if ($route === null) { - if ($host === System::getEnv('_APP_DOMAIN_FUNCTIONS', '')) { + if ($rule === null) { + if ($host === System::getEnv('_APP_DOMAIN_FUNCTIONS', '') || $host === System::getEnv('_APP_DOMAIN_SITES', '')) { throw new AppwriteException(AppwriteException::GENERAL_ACCESS_FORBIDDEN, 'This domain cannot be used for security reasons. Please use any subdomain instead.'); } - if (\str_ends_with($host, System::getEnv('_APP_DOMAIN_FUNCTIONS', ''))) { + if (\str_ends_with($host, System::getEnv('_APP_DOMAIN_FUNCTIONS', '')) || \str_ends_with($host, System::getEnv('_APP_DOMAIN_SITES', ''))) { throw new AppwriteException(AppwriteException::GENERAL_ACCESS_FORBIDDEN, 'This domain is not connected to any Appwrite resource yet. Please configure custom domain or function domain to allow this request.'); } @@ -78,13 +78,12 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo // Act as API - no Proxy logic $utopia->getRoute()?->label('error', ''); + return false; } - $projectId = $route->getAttribute('projectId'); - $project = Authorization::skip( - fn () => $dbForConsole->getDocument('projects', $projectId) - ); + $projectId = $rule->getAttribute('projectId'); + $project = Authorization::skip(fn () => $dbForConsole->getDocument('projects', $projectId)); if (array_key_exists('proxy', $project->getAttribute('services', []))) { $status = $project->getAttribute('services', [])['proxy']; if (!$status) { @@ -98,11 +97,15 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo return false; } - $type = $route->getAttribute('resourceType'); + $type = $rule->getAttribute('resourceType'); if ($type === 'function' || $type === 'site') { - $isFunction = $type === 'function' ; - $isSite = $type === 'site'; + // $isFunction = $type === 'function' ; + // $isSite = $type === 'site'; + $resourceCollection = match($type) { + 'function' => 'functions', + 'site' => 'sites' + }; $utopia->getRoute()?->label('sdk.namespace', 'functions'); $utopia->getRoute()?->label('sdk.method', 'createExecution'); @@ -116,8 +119,8 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo } } - $resourceId = $route->getAttribute('resourceId'); - $projectId = $route->getAttribute('projectId'); + $resourceId = $rule->getAttribute('resourceId'); + $projectId = $rule->getAttribute('projectId'); $path = ($swooleRequest->server['request_uri'] ?? '/'); $query = ($swooleRequest->server['query_string'] ?? ''); @@ -135,21 +138,20 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo /** @var Database $dbForProject */ $dbForProject = $getProjectDB($project); - $function = Authorization::skip(fn () => $dbForProject->getDocument($isSite ? 'sites' : 'functions', $resourceId)); + $resource = Authorization::skip(fn () => $dbForProject->getDocument($resourceCollection, $resourceId)); - if ($function->isEmpty() || !$function->getAttribute('enabled')) { + if ($resource->isEmpty() || !$resource->getAttribute('enabled')) { throw new AppwriteException(AppwriteException::FUNCTION_NOT_FOUND); } - $version = $function->getAttribute('version', 'v2'); + $version = $resource->getAttribute('version', 'v2'); $runtimes = Config::getParam($version === 'v2' ? 'runtimes-v2' : 'runtimes', []); - $spec = Config::getParam('runtime-specifications')[$function->getAttribute('specification', APP_FUNCTION_SPECIFICATION_DEFAULT)]; + $spec = Config::getParam('runtime-specifications')[$resource->getAttribute('specification', APP_FUNCTION_SPECIFICATION_DEFAULT)]; - $runtime = (isset($runtimes[$function->getAttribute('runtime', '')])) ? $runtimes[$function->getAttribute('runtime', '')] : null; - - // todo: fallback for static sites runtime - if ($isSite) { - $runtime = [ + //todo: have runtime configs for sites + $runtime = match($type) { + 'function' => (isset($runtimes[$resource->getAttribute('runtime', '')])) ? $runtimes[$resource->getAttribute('runtime', '')] : null, + 'site' => [ 'key' => 'static-for-now', 'name' => 'Static', 'logo' => 'node.png', @@ -158,20 +160,22 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo 'base' => 'static:1.0', 'image' => 'static:1.0', 'supports' => [System::X86, System::ARM64, System::ARMV7, System::ARMV8] - ]; + ], + default => null + }; + + if (\is_null($runtime)) { + throw new AppwriteException(AppwriteException::FUNCTION_RUNTIME_UNSUPPORTED, 'Runtime "' . $resource->getAttribute('runtime', '') . '" is not supported'); } - //todo: figure out for sites/functions - if ($isFunction) { - if (\is_null($runtime)) { - throw new AppwriteException(AppwriteException::FUNCTION_RUNTIME_UNSUPPORTED, 'Runtime "' . $function->getAttribute('runtime', '') . '" is not supported'); - } - } - //todo: find a better approach - $deploymentId = $isSite ? $function->getAttribute('deploymentId', '') : $function->getAttribute('deployment', ''); + $deploymentId = match($type) { + 'function' => $resource->getAttribute('deploymentId', ''), + 'site' => $resource->getAttribute('deployment', '') + }; + $deployment = Authorization::skip(fn () => $dbForProject->getDocument('deployments', $deploymentId)); - if ($deployment->getAttribute('resourceId') !== $function->getId()) { + if ($deployment->getAttribute('resourceId') !== $resource->getId()) { throw new AppwriteException(AppwriteException::DEPLOYMENT_NOT_FOUND, 'Deployment not found. Create a deployment before trying to execute a function'); } @@ -190,30 +194,32 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo } //todo: figure out for sites/functions - if ($isFunction) { - $permissions = $function->getAttribute('execute'); + if ($type === 'function') { + $permissions = $resource->getAttribute('execute'); if (!(\in_array('any', $permissions)) && !(\in_array('guests', $permissions))) { throw new AppwriteException(AppwriteException::USER_UNAUTHORIZED, 'To execute function using domain, execute permissions must include "any" or "guests"'); } } - $jwtExpiry = $function->getAttribute('timeout', 900); - $jwtObj = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', $jwtExpiry, 0); - $apiKey = $jwtObj->encode([ - 'projectId' => $project->getId(), - 'scopes' => $function->getAttribute('scopes', []) - ]); - $headers = \array_merge([], $requestHeaders); - $headers['x-appwrite-key'] = API_KEY_DYNAMIC . '_' . $apiKey; - $headers['x-appwrite-trigger'] = 'http'; $headers['x-appwrite-user-id'] = ''; - $headers['x-appwrite-user-jwt'] = ''; $headers['x-appwrite-country-code'] = ''; $headers['x-appwrite-continent-code'] = ''; $headers['x-appwrite-continent-eu'] = 'false'; + if ($type === 'function') { + $jwtExpiry = $resource->getAttribute('timeout', 900); + $jwtObj = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', $jwtExpiry, 0); + $apiKey = $jwtObj->encode([ + 'projectId' => $project->getId(), + 'scopes' => $resource->getAttribute('scopes', []) + ]); + $headers['x-appwrite-key'] = API_KEY_DYNAMIC . '_' . $apiKey; + $headers['x-appwrite-trigger'] = 'http'; + $headers['x-appwrite-user-jwt'] = ''; + } + $ip = $headers['x-real-ip'] ?? ''; if (!empty($ip)) { $record = $geodb->get($ip); @@ -239,8 +245,8 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo $execution = new Document([ '$id' => $executionId, '$permissions' => [], - 'functionInternalId' => $function->getInternalId(), - 'functionId' => $function->getId(), + 'functionInternalId' => $resource->getInternalId(), + 'functionId' => $resource->getId(), 'deploymentInternalId' => $deployment->getInternalId(), 'deploymentId' => $deployment->getId(), 'trigger' => 'http', // http / schedule / event @@ -257,9 +263,9 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo ]); $queueForEvents - ->setParam('functionId', $function->getId()) + ->setParam('functionId', $resource->getId()) ->setParam('executionId', $execution->getId()) - ->setContext('function', $function); + ->setContext('function', $resource); $durationStart = \microtime(true); @@ -276,12 +282,12 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo } // Shared vars - foreach ($function->getAttribute('varsProject', []) as $var) { + foreach ($resource->getAttribute('varsProject', []) as $var) { $vars[$var->getAttribute('key')] = $var->getAttribute('value', ''); } // Function vars - foreach ($function->getAttribute('vars', []) as $var) { + foreach ($resource->getAttribute('vars', []) as $var) { $vars[$var->getAttribute('key')] = $var->getAttribute('value', ''); } @@ -293,7 +299,7 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo $vars = \array_merge($vars, [ 'APPWRITE_FUNCTION_API_ENDPOINT' => $endpoint, 'APPWRITE_FUNCTION_ID' => $resourceId, - 'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name'), + 'APPWRITE_FUNCTION_NAME' => $resource->getAttribute('name'), 'APPWRITE_FUNCTION_DEPLOYMENT' => $deployment->getId(), 'APPWRITE_FUNCTION_PROJECT_ID' => $project->getId(), 'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'] ?? '', @@ -320,21 +326,26 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo /** Execute function */ $executor = new Executor(System::getEnv('_APP_EXECUTOR_HOST')); try { - $version = $function->getAttribute('version', 'v2'); - $entrypoint = $deployment->getAttribute('entrypoint', ''); - // todo: figure out site specific settings - if ($isSite) { - $version = 'v4'; - $entrypoint = 'placeholder'; - } - $runtimeEntrypoint = $version === 'v2' ? '' : 'cp /tmp/code.tar.gz /mnt/code/code.tar.gz && nohup helpers/start.sh "' . $runtime['startCommand'] . '"'; + $version = match($type) { + 'function' => $resource->getAttribute('version', 'v2'), + 'site' => 'v4' + }; + $entrypoint = match($type) { + 'function' => $deployment->getAttribute('entrypoint', ''), + 'site' => 'placeholder' // entrypoint is required in api, but not needed with site + }; + $runtimeEntrypoint = match ($version) { + 'v2' => '', + default => 'cp /tmp/code.tar.gz /mnt/code/code.tar.gz && nohup helpers/start.sh "' . $runtime['startCommand'] . '"' + }; + $executionResponse = $executor->createExecution( projectId: $project->getId(), deploymentId: $deployment->getId(), body: \strlen($body) > 0 ? $body : null, variables: $vars, // todo: figure out timeouts for sites - timeout: $function->getAttribute('timeout', 30), + timeout: $resource->getAttribute('timeout', 30), image: $runtime['image'], source: $build->getAttribute('path', ''), entrypoint: $entrypoint, @@ -345,7 +356,7 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo runtimeEntrypoint: $runtimeEntrypoint, cpus: $spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT, memory: $spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT, - logging: $function->getAttribute('logging', true), + logging: $resource->getAttribute('logging', true), requestTimeout: 30 ); @@ -388,21 +399,22 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo ->addMetric(METRIC_NETWORK_REQUESTS, 1) ->addMetric(METRIC_NETWORK_INBOUND, $request->getSize() + $fileSize) ->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize()); - if ($isFunction) { + //todo: add metrics for sites + if ($type === 'function') { $queueForUsage ->addMetric(METRIC_EXECUTIONS, 1) - ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1) + ->addMetric(str_replace('{functionInternalId}', $resource->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1) ->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000)) // per project - ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000)) // per function + ->addMetric(str_replace('{functionInternalId}', $resource->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000)) // per function ->addMetric(METRIC_EXECUTIONS_MB_SECONDS, (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $execution->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT))) - ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_MB_SECONDS), (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $execution->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT))); + ->addMetric(str_replace('{functionInternalId}', $resource->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_MB_SECONDS), (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $execution->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT))); } $queueForUsage ->setProject($project) ->trigger(); - if ($isFunction) { + if ($type === 'function') { $queueForFunctions ->setType(Func::TYPE_ASYNC_WRITE) ->setExecution($execution) diff --git a/app/init.php b/app/init.php index 05824389e5..162acfa821 100644 --- a/app/init.php +++ b/app/init.php @@ -277,9 +277,17 @@ const METRIC_FUNCTION_ID_BUILDS_STORAGE = '{functionInternalId}.builds.storage'; const METRIC_FUNCTION_ID_BUILDS_COMPUTE = '{functionInternalId}.builds.compute'; const METRIC_FUNCTION_ID_BUILDS_COMPUTE_SUCCESS = '{functionInternalId}.builds.compute.success'; const METRIC_FUNCTION_ID_BUILDS_COMPUTE_FAILED = '{functionInternalId}.builds.compute.failed'; +const METRIC_FUNCTION_ID_BUILDS_MB_SECONDS = '{functionInternalId}.builds.mbSeconds'; +const METRIC_SITES_ID_BUILDS = 'sites.{siteInternalId}.builds'; +const METRIC_SITES_ID_BUILDS_SUCCESS = 'sites.{siteInternalId}.builds.success'; +const METRIC_SITES_ID_BUILDS_FAILED = 'sites.{siteInternalId}.builds.failed'; +const METRIC_SITES_ID_BUILDS_STORAGE = 'sites.{siteInternalId}.builds.storage'; +const METRIC_SITES_ID_BUILDS_COMPUTE = 'sites.{siteInternalId}.builds.compute'; +const METRIC_SITES_ID_BUILDS_COMPUTE_SUCCESS = 'sites.{siteInternalId}.builds.compute.success'; +const METRIC_SITES_ID_BUILDS_COMPUTE_FAILED = 'sites.{siteInternalId}.builds.compute.failed'; +const METRIC_SITES_ID_BUILDS_MB_SECONDS = 'sites.{siteInternalId}.builds.mbSeconds'; const METRIC_FUNCTION_ID_DEPLOYMENTS = '{resourceType}.{resourceInternalId}.deployments'; const METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE = '{resourceType}.{resourceInternalId}.deployments.storage'; -const METRIC_FUNCTION_ID_BUILDS_MB_SECONDS = '{functionInternalId}.builds.mbSeconds'; const METRIC_EXECUTIONS = 'executions'; const METRIC_EXECUTIONS_COMPUTE = 'executions.compute'; const METRIC_EXECUTIONS_MB_SECONDS = 'executions.mbSeconds'; diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index 312b9edcf3..4a884cd9ce 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -119,10 +119,11 @@ class Builds extends Action */ protected function buildDeployment(Device $deviceForFunctions, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Database $dbForConsole, Database $dbForProject, GitHub $github, Document $project, Document $resource, Document $deployment, Document $template, Log $log): void { - // todo: refactor - $isFunction = $resource->getCollection() === 'functions'; - $isSite = $resource->getCollection() === 'sites'; - $foreignKey = $isFunction ? 'functionId' : 'siteId'; + $foreignKey = match($resource->getCollection()) { + 'functions' => 'functionId', + 'sites' => 'siteId', + default => throw new \Exception('Invalid resource type') + }; $executor = new Executor(System::getEnv('_APP_EXECUTOR_HOST')); @@ -140,30 +141,14 @@ class Builds extends Action throw new \Exception('Deployment not found', 404); } - if ($isFunction && empty($deployment->getAttribute('entrypoint', ''))) { + // todo: figure out a better way, entrypoint is not required for sites + if ($resource->getCollection() === 'functions' && empty($deployment->getAttribute('entrypoint', ''))) { throw new \Exception('Entrypoint for your Appwrite Function is missing. Please specify it when making deployment or update the entrypoint under your function\'s "Settings" > "Configuration" > "Entrypoint".', 500); } - $version = $resource->getAttribute('version', 'v2'); - - // todo: fallback for sites - if ($isSite) { - $version = 'v4'; - } + $version = $this->getVersion($resource); + $runtime = $this->getRuntime($resource, $version); $spec = Config::getParam('runtime-specifications')[$resource->getAttribute('specifications', APP_FUNCTION_SPECIFICATION_DEFAULT)]; - $runtimes = Config::getParam($version === 'v2' ? 'runtimes-v2' : 'runtimes', []); - // todo: fix for sites using frameworks - $key = $resource->getAttribute('runtime'); - $runtime = $runtimes[$key] ?? null; - - // todo: fallback for sites - if ($isSite) { - $runtime = $runtimes['node-18.0']; - } - - if (\is_null($runtime)) { - throw new \Exception('Runtime "' . $resource->getAttribute('runtime', '') . '" is not supported'); - } // Realtime preparation $allEvents = Event::generateEvents("{$resource->getCollection()}.[{$foreignKey}].deployments.[deploymentId].update", [ @@ -503,36 +488,6 @@ class Builds extends Action $hostname = System::getEnv('_APP_DOMAIN'); $endpoint = $protocol . '://' . $hostname . "/v1"; - //todo: ugly, but works - if ($isFunction) { - $vars = [ - ...$vars, - 'APPWRITE_FUNCTION_API_ENDPOINT' => $endpoint, - 'APPWRITE_FUNCTION_API_KEY' => API_KEY_DYNAMIC . '_' . $apiKey, - 'APPWRITE_FUNCTION_ID' => $resource->getId(), - 'APPWRITE_FUNCTION_NAME' => $resource->getAttribute('name'), - 'APPWRITE_FUNCTION_DEPLOYMENT' => $deployment->getId(), - 'APPWRITE_FUNCTION_PROJECT_ID' => $project->getId(), - 'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'] ?? '', - 'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'] ?? '', - 'APPWRITE_FUNCTION_CPUS' => $cpus, - 'APPWRITE_FUNCTION_MEMORY' => $memory - ]; - } - if ($isSite) { - $vars = [ - ...$vars, - 'APPWRITE_SITE_ID' => $resource->getId(), - 'APPWRITE_SITE_NAME' => $resource->getAttribute('name'), - 'APPWRITE_SITE_DEPLOYMENT' => $deployment->getId(), - 'APPWRITE_SITE_PROJECT_ID' => $project->getId(), - 'APPWRITE_SITE_RUNTIME_NAME' => $runtime['name'] ?? '', - 'APPWRITE_SITE_RUNTIME_VERSION' => $runtime['version'] ?? '', - 'APPWRITE_SITE_CPUS' => $cpus, - 'APPWRITE_SITE_MEMORY' => $memory - ]; - } - // Appwrite vars $vars = \array_merge($vars, [ 'APPWRITE_VERSION' => APP_VERSION_STABLE, @@ -552,13 +507,42 @@ class Builds extends Action 'APPWRITE_VCS_ROOT_DIRECTORY' => $deployment->getAttribute('providerRootDirectory', ''), ]); - $command = $deployment->getAttribute('commands', ''); - - //todo: for sites use isntall and build command - if ($isSite) { - $command = 'npm ci && npm run build'; + switch ($resource->getCollection()) { + case 'functions': + $vars = [ + ...$vars, + 'APPWRITE_FUNCTION_API_ENDPOINT' => $endpoint, + 'APPWRITE_FUNCTION_API_KEY' => API_KEY_DYNAMIC . '_' . $apiKey, + 'APPWRITE_FUNCTION_ID' => $resource->getId(), + 'APPWRITE_FUNCTION_NAME' => $resource->getAttribute('name'), + 'APPWRITE_FUNCTION_DEPLOYMENT' => $deployment->getId(), + 'APPWRITE_FUNCTION_PROJECT_ID' => $project->getId(), + 'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'] ?? '', + 'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'] ?? '', + 'APPWRITE_FUNCTION_CPUS' => $cpus, + 'APPWRITE_FUNCTION_MEMORY' => $memory + ]; + break; + case 'sites': + $vars = [ + ...$vars, + 'APPWRITE_SITE_ID' => $resource->getId(), + 'APPWRITE_SITE_NAME' => $resource->getAttribute('name'), + 'APPWRITE_SITE_DEPLOYMENT' => $deployment->getId(), + 'APPWRITE_SITE_PROJECT_ID' => $project->getId(), + 'APPWRITE_SITE_RUNTIME_NAME' => $runtime['name'] ?? '', + 'APPWRITE_SITE_RUNTIME_VERSION' => $runtime['version'] ?? '', + 'APPWRITE_SITE_CPUS' => $cpus, + 'APPWRITE_SITE_MEMORY' => $memory + ]; + break; } + $command = $this->getCommand( + resource: $resource, + deployment: $deployment + ); + $response = null; $err = null; @@ -570,13 +554,8 @@ class Builds extends Action $isCanceled = false; Co::join([ - Co\go(function () use ($executor, &$response, $project, $deployment, $source, $resource, $runtime, $vars, $command, $cpus, $memory, &$err, $isSite) { + Co\go(function () use ($executor, &$response, $project, $deployment, $source, $resource, $runtime, $vars, $command, $cpus, $memory, &$err, $version) { try { - - $version = $resource->getAttribute('version', 'v2'); - if ($isSite) { - $version = 'v4'; - } $command = $version === 'v2' ? 'tar -zxf /tmp/code.tar.gz -C /usr/code && cd /usr/local/src/ && ./build.sh' : 'tar -zxf /tmp/code.tar.gz -C /mnt/code && helpers/build.sh "' . \trim(\escapeshellarg($command), "\'") . '"'; $response = $executor->createRuntime( @@ -691,14 +670,15 @@ class Builds extends Action if ($deployment->getAttribute('activate') === true) { $resource->setAttribute('deploymentInternalId', $deployment->getInternalId()); $resource->setAttribute('live', true); - // todo: fix here how clean this is - if ($isSite) { - $resource->setAttribute('deploymentId', $deployment->getId()); - $resource = $dbForProject->updateDocument('sites', $resource->getId(), $resource); - } - if ($isFunction) { - $resource->setAttribute('deployment', $deployment->getId()); - $resource = $dbForProject->updateDocument('functions', $resource->getId(), $resource); + switch ($resource->getCollection()) { + case 'functions': + $resource->setAttribute('deployment', $deployment->getId()); + $resource = $dbForProject->updateDocument('functions', $resource->getId(), $resource); + break; + case 'sites': + $resource->setAttribute('deploymentId', $deployment->getId()); + $resource = $dbForProject->updateDocument('sites', $resource->getId(), $resource); + break; } } @@ -710,7 +690,7 @@ class Builds extends Action /** Update function schedule */ // Inform scheduler if function is still active - if ($isFunction) { + if ($resource->getCollection() === 'functions') { $schedule = $dbForConsole->getDocument('schedules', $resource->getAttribute('scheduleId')); $schedule ->setAttribute('resourceUpdatedAt', DateTime::now()) @@ -753,41 +733,111 @@ class Builds extends Action channels: $target['channels'], roles: $target['roles'] ); - - /** Trigger usage queue */ - if ($build->getAttribute('status') === 'ready') { - if ($isFunction) { - $queueForUsage - ->addMetric(METRIC_BUILDS_SUCCESS, 1) // per project - ->addMetric(METRIC_BUILDS_COMPUTE_SUCCESS, (int)$build->getAttribute('duration', 0) * 1000) - ->addMetric(str_replace('{functionInternalId}', $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_SUCCESS), 1) // per function - ->addMetric(str_replace('{functionInternalId}', $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE_SUCCESS), (int)$build->getAttribute('duration', 0) * 1000); - } - } elseif ($build->getAttribute('status') === 'failed') { - if ($isFunction) { - $queueForUsage - ->addMetric(METRIC_BUILDS_FAILED, 1) // per project - ->addMetric(METRIC_BUILDS_COMPUTE_FAILED, (int)$build->getAttribute('duration', 0) * 1000) - ->addMetric(str_replace('{functionInternalId}', $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_FAILED), 1) // per function - ->addMetric(str_replace('{functionInternalId}', $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE_FAILED), (int)$build->getAttribute('duration', 0) * 1000); - } - } - if ($isFunction) { - $queueForUsage - ->addMetric(METRIC_BUILDS, 1) // per project - ->addMetric(METRIC_BUILDS_STORAGE, $build->getAttribute('size', 0)) - ->addMetric(METRIC_BUILDS_COMPUTE, (int)$build->getAttribute('duration', 0) * 1000) - ->addMetric(METRIC_BUILDS_MB_SECONDS, (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $build->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT))) - ->addMetric(str_replace('{functionInternalId}', $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS), 1) // per function - ->addMetric(str_replace('{functionInternalId}', $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE), $build->getAttribute('size', 0)) - ->addMetric(str_replace('{functionInternalId}', $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE), (int)$build->getAttribute('duration', 0) * 1000) - ->addMetric(str_replace('{functionInternalId}', $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_MB_SECONDS), (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $build->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT))) - ->setProject($project) - ->trigger(); - } + $this->sendUsage( + resource:$resource, + build: $build, + project: $project, + queue: $queueForUsage + ); } } + protected function sendUsage(Document $resource, Document $build, Document $project, Usage $queue): void + { + $key = match($resource->getCollection()) { + 'functions' => 'functionInternalId', + 'sites' => 'siteInternalId', + default => throw new \Exception('Invalid resource type') + }; + + $metrics = match($resource->getCollection()) { + 'functions' => [ + 'builds' => METRIC_FUNCTION_ID_BUILDS, + 'buildsSuccess' => METRIC_FUNCTION_ID_BUILDS_SUCCESS, + 'buildsFailed' => METRIC_FUNCTION_ID_BUILDS_FAILED, + 'buildsComputeSuccess' => METRIC_FUNCTION_ID_BUILDS_COMPUTE_SUCCESS, + 'buildsComputeFailed' => METRIC_FUNCTION_ID_BUILDS_COMPUTE_FAILED, + 'buildsStorage' => METRIC_FUNCTION_ID_BUILDS_STORAGE, + 'buildsCompute' => METRIC_FUNCTION_ID_BUILDS_COMPUTE, + 'buildsMbSeconds' => METRIC_FUNCTION_ID_BUILDS_MB_SECONDS + ], + 'sites' => [ + 'builds' => METRIC_SITES_ID_BUILDS, + 'buildsSuccess' => METRIC_SITES_ID_BUILDS_SUCCESS, + 'buildsFailed' => METRIC_SITES_ID_BUILDS_FAILED, + 'buildsComputeSuccess' => METRIC_SITES_ID_BUILDS_COMPUTE_SUCCESS, + 'buildsComputeFailed' => METRIC_SITES_ID_BUILDS_COMPUTE_FAILED, + 'buildsStorage' => METRIC_SITES_ID_BUILDS_STORAGE, + 'buildsCompute' => METRIC_SITES_ID_BUILDS_COMPUTE, + 'buildsMbSeconds' => METRIC_SITES_ID_BUILDS_MB_SECONDS + ] + }; + + switch ($build->getAttribute('status')) { + case 'ready': + $queue + ->addMetric(METRIC_BUILDS_SUCCESS, 1) // per project + ->addMetric(METRIC_BUILDS_COMPUTE_SUCCESS, (int)$build->getAttribute('duration', 0) * 1000) + ->addMetric(str_replace($key, $resource->getInternalId(), $metrics['buildsSuccess']), 1) // per function + ->addMetric(str_replace($key, $resource->getInternalId(), $metrics['buildsComputeSuccess']), (int)$build->getAttribute('duration', 0) * 1000); + break; + case 'failed': + $queue + ->addMetric(METRIC_BUILDS_FAILED, 1) // per project + ->addMetric(METRIC_BUILDS_COMPUTE_FAILED, (int)$build->getAttribute('duration', 0) * 1000) + ->addMetric(str_replace($key, $resource->getInternalId(), $metrics['buildsFailed']), 1) // per function + ->addMetric(str_replace($key, $resource->getInternalId(), $metrics['buildsComputeFailed']), (int)$build->getAttribute('duration', 0) * 1000); + break; + } + + $queue + ->addMetric(METRIC_BUILDS, 1) // per project + ->addMetric(METRIC_BUILDS_STORAGE, $build->getAttribute('size', 0)) + ->addMetric(METRIC_BUILDS_COMPUTE, (int)$build->getAttribute('duration', 0) * 1000) + ->addMetric(METRIC_BUILDS_MB_SECONDS, (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $build->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT))) + ->addMetric(str_replace($key, $resource->getInternalId(), $metrics['builds']), 1) // per function + ->addMetric(str_replace($key, $resource->getInternalId(), $metrics['buildsStorage']), $build->getAttribute('size', 0)) + ->addMetric(str_replace($key, $resource->getInternalId(), $metrics['buildsCompute']), (int)$build->getAttribute('duration', 0) * 1000) + ->addMetric(str_replace($key, $resource->getInternalId(), $metrics['buildsMbSeconds']), (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $build->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT))) + ->setProject($project) + ->trigger(); + } + + protected function getRuntime(Document $resource, string $version): array + { + $runtimes = Config::getParam($version === 'v2' ? 'runtimes-v2' : 'runtimes', []); + $key = $resource->getAttribute('runtime'); + $runtime = match ($resource->getCollection()) { + 'functions' => $runtimes[$key] ?? null, + 'sites' => $runtimes['node-18.0'] ?? null, + default => null + }; + if (\is_null($runtime)) { + throw new \Exception('Runtime "' . $resource->getAttribute('runtime', '') . '" is not supported'); + } + + return $runtime; + } + + protected function getVersion(Document $resource): string + { + return match ($resource->getCollection()) { + 'functions' => $resource->getAttribute('version', 'v2'), + 'sites' => 'v4', + }; + } + + protected function getCommand(Document $resource, Document $deployment): string + { + return match($resource->getCollection()) { + 'functions' => $deployment->getAttribute('command', ''), + 'sites' => implode(' && ', array_filter([ + $deployment->getAttribute('installCommand'), + $deployment->getAttribute('buildCommand') + ])) + }; + } + /** * @param string $status * @param GitHub $github From 8950f71b0addcef71393fbc810f34a6c8e5404b3 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Thu, 24 Oct 2024 19:24:24 +0200 Subject: [PATCH 16/25] revert: comments --- app/controllers/general.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index adc2c82f49..92c787f407 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -100,8 +100,6 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo $type = $rule->getAttribute('resourceType'); if ($type === 'function' || $type === 'site') { - // $isFunction = $type === 'function' ; - // $isSite = $type === 'site'; $resourceCollection = match($type) { 'function' => 'functions', 'site' => 'sites' From aba3a31663d1de73e3de7f9f3f9abb1a56d4c70a Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Thu, 24 Oct 2024 19:28:20 +0200 Subject: [PATCH 17/25] fix: deployment/deploymentId --- app/controllers/general.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index 92c787f407..d5463935f9 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -167,8 +167,8 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo } $deploymentId = match($type) { - 'function' => $resource->getAttribute('deploymentId', ''), - 'site' => $resource->getAttribute('deployment', '') + 'function' => $resource->getAttribute('deployment', ''), + 'site' => $resource->getAttribute('deploymentId', '') }; $deployment = Authorization::skip(fn () => $dbForProject->getDocument('deployments', $deploymentId)); From 5d6a8be66d425a1dfa1dcdaf439c077b7d02e8e4 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Thu, 24 Oct 2024 19:43:12 +0200 Subject: [PATCH 18/25] fix: function command --- src/Appwrite/Platform/Modules/Functions/Workers/Builds.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index 4a884cd9ce..0e9819c8e9 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -830,7 +830,7 @@ class Builds extends Action protected function getCommand(Document $resource, Document $deployment): string { return match($resource->getCollection()) { - 'functions' => $deployment->getAttribute('command', ''), + 'functions' => $deployment->getAttribute('commands', ''), 'sites' => implode(' && ', array_filter([ $deployment->getAttribute('installCommand'), $deployment->getAttribute('buildCommand') From 41484113a0b01fe880c53af34d1ae18262e3499d Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Fri, 25 Oct 2024 10:55:31 +0200 Subject: [PATCH 19/25] chore: add comments --- app/controllers/general.php | 2 ++ .../Platform/Modules/Functions/Workers/Builds.php | 14 +++++++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/app/controllers/general.php b/app/controllers/general.php index d5463935f9..378cfbfd84 100644 --- a/app/controllers/general.php +++ b/app/controllers/general.php @@ -206,6 +206,7 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo $headers['x-appwrite-continent-code'] = ''; $headers['x-appwrite-continent-eu'] = 'false'; + //todo: check if this would work for sites if ($type === 'function') { $jwtExpiry = $resource->getAttribute('timeout', 900); $jwtObj = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', $jwtExpiry, 0); @@ -330,6 +331,7 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo }; $entrypoint = match($type) { 'function' => $deployment->getAttribute('entrypoint', ''), + //todo: check if null works 'site' => 'placeholder' // entrypoint is required in api, but not needed with site }; $runtimeEntrypoint = match ($version) { diff --git a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php index 0e9819c8e9..4baca4f1bd 100644 --- a/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php +++ b/src/Appwrite/Platform/Modules/Functions/Workers/Builds.php @@ -119,7 +119,7 @@ class Builds extends Action */ protected function buildDeployment(Device $deviceForFunctions, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Database $dbForConsole, Database $dbForProject, GitHub $github, Document $project, Document $resource, Document $deployment, Document $template, Log $log): void { - $foreignKey = match($resource->getCollection()) { + $resourceKey = match($resource->getCollection()) { 'functions' => 'functionId', 'sites' => 'siteId', default => throw new \Exception('Invalid resource type') @@ -127,7 +127,7 @@ class Builds extends Action $executor = new Executor(System::getEnv('_APP_EXECUTOR_HOST')); - $log->addTag($foreignKey, $resource->getId()); + $log->addTag($resourceKey, $resource->getId()); $resource = $dbForProject->getDocument($resource->getCollection(), $resource->getId()); if ($resource->isEmpty()) { @@ -151,8 +151,8 @@ class Builds extends Action $spec = Config::getParam('runtime-specifications')[$resource->getAttribute('specifications', APP_FUNCTION_SPECIFICATION_DEFAULT)]; // Realtime preparation - $allEvents = Event::generateEvents("{$resource->getCollection()}.[{$foreignKey}].deployments.[deploymentId].update", [ - $foreignKey => $resource->getId(), + $allEvents = Event::generateEvents("{$resource->getCollection()}.[{$resourceKey}].deployments.[deploymentId].update", [ + $resourceKey => $resource->getId(), 'deploymentId' => $deployment->getId() ]); @@ -433,8 +433,8 @@ class Builds extends Action ->setQueue(Event::WEBHOOK_QUEUE_NAME) ->setClass(Event::WEBHOOK_CLASS_NAME) ->setProject($project) - ->setEvent("{$resource->getCollection()}.[{$foreignKey}].deployments.[deploymentId].update") - ->setParam($foreignKey, $resource->getId()) + ->setEvent("{$resource->getCollection()}.[{$resourceKey}].deployments.[deploymentId].update") + ->setParam($resourceKey, $resource->getId()) ->setParam('deploymentId', $deployment->getId()) ->setPayload($deployment->getArrayCopy(array_keys($deploymentModel->getRules()))); @@ -809,7 +809,7 @@ class Builds extends Action $key = $resource->getAttribute('runtime'); $runtime = match ($resource->getCollection()) { 'functions' => $runtimes[$key] ?? null, - 'sites' => $runtimes['node-18.0'] ?? null, + 'sites' => $runtimes['node-18.0'] ?? null, //todo: fix hardcode default => null }; if (\is_null($runtime)) { From 9d1d1015865f3535bf01abeb10e9b40abbaae59b Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Fri, 25 Oct 2024 14:41:14 +0200 Subject: [PATCH 20/25] Update VCS endpoin to work for both functions and sites --- app/controllers/api/vcs.php | 77 ++++++++++--------- .../Platform/Modules/Sites/Services/Http.php | 10 +++ 2 files changed, 51 insertions(+), 36 deletions(-) diff --git a/app/controllers/api/vcs.php b/app/controllers/api/vcs.php index e79eb67936..753fd043c8 100644 --- a/app/controllers/api/vcs.php +++ b/app/controllers/api/vcs.php @@ -44,29 +44,30 @@ use function Swoole\Coroutine\batch; $createGitDeployments = function (GitHub $github, string $providerInstallationId, array $repositories, string $providerBranch, string $providerBranchUrl, string $providerRepositoryName, string $providerRepositoryUrl, string $providerRepositoryOwner, string $providerCommitHash, string $providerCommitAuthor, string $providerCommitAuthorUrl, string $providerCommitMessage, string $providerCommitUrl, string $providerPullRequestId, bool $external, Database $dbForConsole, Build $queueForBuilds, callable $getProjectDB, Request $request) { $errors = []; - foreach ($repositories as $resource) { + foreach ($repositories as $repository) { try { - $resourceType = $resource->getAttribute('resourceType'); + $resourceType = $repository->getAttribute('resourceType'); - if ($resourceType !== "function") { + if ($resourceType !== "function" && $resourceType !== "site") { continue; } - $projectId = $resource->getAttribute('projectId'); + $projectId = $repository->getAttribute('projectId'); $project = Authorization::skip(fn () => $dbForConsole->getDocument('projects', $projectId)); $dbForProject = $getProjectDB($project); - $functionId = $resource->getAttribute('resourceId'); - $function = Authorization::skip(fn () => $dbForProject->getDocument('functions', $functionId)); - $functionInternalId = $function->getInternalId(); + $resourceCollection = $resourceType === "function" ? 'functions' : 'sites'; + $resourceId = $repository->getAttribute('resourceId'); + $resource = Authorization::skip(fn () => $dbForProject->getDocument($resourceCollection, $resourceId)); + $resourceInternalId = $resource->getInternalId(); $deploymentId = ID::unique(); - $repositoryId = $resource->getId(); - $repositoryInternalId = $resource->getInternalId(); - $providerRepositoryId = $resource->getAttribute('providerRepositoryId'); - $installationId = $resource->getAttribute('installationId'); - $installationInternalId = $resource->getAttribute('installationInternalId'); - $productionBranch = $function->getAttribute('providerBranch'); + $repositoryId = $repository->getId(); + $repositoryInternalId = $repository->getInternalId(); + $providerRepositoryId = $repository->getAttribute('providerRepositoryId'); + $installationId = $repository->getAttribute('installationId'); + $installationInternalId = $repository->getAttribute('installationInternalId'); + $productionBranch = $resource->getAttribute('providerBranch'); $activate = false; if ($providerBranch == $productionBranch && $external === false) { @@ -90,7 +91,7 @@ $createGitDeployments = function (GitHub $github, string $providerInstallationId $isAuthorized = !$external; if (!$isAuthorized && !empty($providerPullRequestId)) { - if (\in_array($providerPullRequestId, $resource->getAttribute('providerPullRequestIds', []))) { + if (\in_array($providerPullRequestId, $repository->getAttribute('providerPullRequestIds', []))) { $isAuthorized = true; } } @@ -103,7 +104,7 @@ $createGitDeployments = function (GitHub $github, string $providerInstallationId $latestCommentId = ''; - if (!empty($providerPullRequestId) && $function->getAttribute('providerSilentMode', false) === false) { + if (!empty($providerPullRequestId) && $resource->getAttribute('providerSilentMode', false) === false) { $latestComment = Authorization::skip(fn () => $dbForConsole->findOne('vcsComments', [ Query::equal('providerRepositoryId', [$providerRepositoryId]), Query::equal('providerPullRequestId', [$providerPullRequestId]), @@ -114,12 +115,12 @@ $createGitDeployments = function (GitHub $github, string $providerInstallationId $latestCommentId = $latestComment->getAttribute('providerCommentId', ''); $comment = new Comment(); $comment->parseComment($github->getComment($owner, $repositoryName, $latestCommentId)); - $comment->addBuild($project, $function, $commentStatus, $deploymentId, $action); + $comment->addBuild($project, $resource, $commentStatus, $deploymentId, $action); $latestCommentId = \strval($github->updateComment($owner, $repositoryName, $latestCommentId, $comment->generateComment())); } else { $comment = new Comment(); - $comment->addBuild($project, $function, $commentStatus, $deploymentId, $action); + $comment->addBuild($project, $resource, $commentStatus, $deploymentId, $action); $latestCommentId = \strval($github->createComment($owner, $repositoryName, $providerPullRequestId, $comment->generateComment())); if (!empty($latestCommentId)) { @@ -156,19 +157,19 @@ $createGitDeployments = function (GitHub $github, string $providerInstallationId $latestCommentId = $comment->getAttribute('providerCommentId', ''); $comment = new Comment(); $comment->parseComment($github->getComment($owner, $repositoryName, $latestCommentId)); - $comment->addBuild($project, $function, $commentStatus, $deploymentId, $action); + $comment->addBuild($project, $resource, $commentStatus, $deploymentId, $action); $latestCommentId = \strval($github->updateComment($owner, $repositoryName, $latestCommentId, $comment->generateComment())); } } if (!$isAuthorized) { - $functionName = $function->getAttribute('name'); + $resourceName = $resource->getAttribute('name'); $projectName = $project->getAttribute('name'); - $name = "{$functionName} ({$projectName})"; + $name = "{$resourceName} ({$projectName})"; $message = 'Authorization required for external contributor.'; - $providerRepositoryId = $resource->getAttribute('providerRepositoryId'); + $providerRepositoryId = $repository->getAttribute('providerRepositoryId'); try { $repositoryName = $github->getRepositoryName($providerRepositoryId) ?? ''; if (empty($repositoryName)) { @@ -195,11 +196,15 @@ $createGitDeployments = function (GitHub $github, string $providerInstallationId Permission::update(Role::any()), Permission::delete(Role::any()), ], - 'resourceId' => $functionId, - 'resourceInternalId' => $functionInternalId, - 'resourceType' => 'functions', - 'entrypoint' => $function->getAttribute('entrypoint'), - 'commands' => $function->getAttribute('commands'), + 'resourceId' => $resourceId, + 'resourceInternalId' => $resourceInternalId, + 'resourceType' => $resourceCollection, + 'entrypoint' => $resource->getAttribute('entrypoint', ''), + 'commands' => $resource->getAttribute('commands', []), + 'installCommand' => $resource->getAttribute('installCommand', ''), + 'buildCommand' => $resource->getAttribute('buildCommand', ''), + 'outputDirectory' => $resource->getAttribute('outputDirectory', ''), + 'fallbackRedirect' => $resource->getAttribute('fallbackRedirect', ''), 'type' => 'vcs', 'installationId' => $installationId, 'installationInternalId' => $installationInternalId, @@ -217,17 +222,17 @@ $createGitDeployments = function (GitHub $github, string $providerInstallationId 'providerCommitUrl' => $providerCommitUrl, 'providerCommentId' => \strval($latestCommentId), 'providerBranch' => $providerBranch, - 'search' => implode(' ', [$deploymentId, $function->getAttribute('entrypoint')]), + 'search' => implode(' ', [$deploymentId, $resource->getAttribute('entrypoint', '')]), 'activate' => $activate, ])); - if (!empty($providerCommitHash) && $function->getAttribute('providerSilentMode', false) === false) { - $functionName = $function->getAttribute('name'); + if (!empty($providerCommitHash) && $resource->getAttribute('providerSilentMode', false) === false) { + $resourceName = $resource->getAttribute('name'); $projectName = $project->getAttribute('name'); - $name = "{$functionName} ({$projectName})"; + $name = "{$resourceName} ({$projectName})"; $message = 'Starting...'; - $providerRepositoryId = $resource->getAttribute('providerRepositoryId'); + $providerRepositoryId = $repository->getAttribute('providerRepositoryId'); try { $repositoryName = $github->getRepositoryName($providerRepositoryId) ?? ''; if (empty($repositoryName)) { @@ -238,17 +243,17 @@ $createGitDeployments = function (GitHub $github, string $providerInstallationId } $owner = $github->getOwnerName($providerInstallationId); - $providerTargetUrl = $request->getProtocol() . '://' . $request->getHostname() . "/console/project-$projectId/functions/function-$functionId"; + $providerTargetUrl = $request->getProtocol() . '://' . $request->getHostname() . "/console/project-$projectId/$resourceCollection/$resourceType-$resourceId"; $github->updateCommitStatus($repositoryName, $providerCommitHash, $owner, 'pending', $message, $providerTargetUrl, $name); } $queueForBuilds ->setType(BUILD_TYPE_DEPLOYMENT) - ->setResource($function) + ->setResource($resource) ->setDeployment($deployment) ->setProject($project); // set the project because it won't be set for git deployments - $queueForBuilds->trigger(); // must trigger here so that we create a build for each function + $queueForBuilds->trigger(); // must trigger here so that we create a build for each function/site //TODO: Add event? } catch (Throwable $e) { @@ -936,7 +941,7 @@ App::post('/v1/vcs/github/events') $github->initializeVariables($providerInstallationId, $privateKey, $githubAppId); - //find functionId from functions table + //find resourceId from relevant resources table $repositories = Authorization::skip(fn () => $dbForConsole->find('repositories', [ Query::equal('providerRepositoryId', [$providerRepositoryId]), Query::limit(100), @@ -948,7 +953,7 @@ App::post('/v1/vcs/github/events') } } elseif ($event == $github::EVENT_INSTALLATION) { if ($parsedPayload["action"] == "deleted") { - // TODO: Use worker for this job instead (update function as well) + // TODO: Use worker for this job instead (update function/site as well) $providerInstallationId = $parsedPayload["installationId"]; $installations = $dbForConsole->find('installations', [ diff --git a/src/Appwrite/Platform/Modules/Sites/Services/Http.php b/src/Appwrite/Platform/Modules/Sites/Services/Http.php index 7552f16c4d..5adbdf3029 100644 --- a/src/Appwrite/Platform/Modules/Sites/Services/Http.php +++ b/src/Appwrite/Platform/Modules/Sites/Services/Http.php @@ -32,12 +32,18 @@ class Http extends Service public function __construct() { $this->type = Service::TYPE_HTTP; + // Sites $this->addAction(CreateSite::getName(), new CreateSite()); $this->addAction(GetSite::getName(), new GetSite()); $this->addAction(ListSites::getName(), new ListSites()); $this->addAction(UpdateSite::getName(), new UpdateSite()); $this->addAction(DeleteSite::getName(), new DeleteSite()); + + // Frameworks $this->addAction(ListFrameworks::getName(), new ListFrameworks()); + + + // Deployments $this->addAction(CreateDeployment::getName(), new CreateDeployment()); $this->addAction(GetDeployment::getName(), new GetDeployment()); $this->addAction(ListDeployments::getName(), new ListDeployments()); @@ -47,12 +53,16 @@ class Http extends Service $this->addAction(DownloadBuild::getName(), new DownloadBuild()); $this->addAction(RebuildDeployment::getName(), new RebuildDeployment()); $this->addAction(CancelDeployment::getName(), new CancelDeployment()); + + // Variables $this->addAction(CreateVariable::getName(), new CreateVariable()); $this->addAction(GetVariable::getName(), new GetVariable()); $this->addAction(ListVariables::getName(), new ListVariables()); $this->addAction(UpdateVariable::getName(), new UpdateVariable()); $this->addAction(DeleteVariable::getName(), new DeleteVariable()); $this->addAction(ListTemplates::getName(), new ListTemplates()); + + // Usage $this->addAction(GetSiteUsage::getName(), new GetSiteUsage()); $this->addAction(GetSitesUsage::getName(), new GetSitesUsage()); } From dd3ffbb391ab2c257cdb30727950cd014f6105fe Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Fri, 25 Oct 2024 14:55:09 +0200 Subject: [PATCH 21/25] Add getTemplate endpoint for sites --- app/config/errors.php | 5 ++ src/Appwrite/Extend/Exception.php | 1 + .../Modules/Sites/Http/Sites/GetTemplate.php | 56 +++++++++++++++++++ .../Platform/Modules/Sites/Services/Http.php | 4 ++ 4 files changed, 66 insertions(+) create mode 100644 src/Appwrite/Platform/Modules/Sites/Http/Sites/GetTemplate.php diff --git a/app/config/errors.php b/app/config/errors.php index df8cb45c98..d38f816136 100644 --- a/app/config/errors.php +++ b/app/config/errors.php @@ -546,6 +546,11 @@ return [ 'description' => 'The requested framework is either inactive or unsupported. Please check the value of the _APP_SITES_FRAMEWORKS environment variable.', 'code' => 404, ], + Exception::SITE_TEMPLATE_NOT_FOUND => [ + 'name' => Exception::SITE_TEMPLATE_NOT_FOUND, + 'description' => 'Site Template with the requested ID could not be found.', + 'code' => 404, + ], /** Builds */ Exception::BUILD_NOT_FOUND => [ diff --git a/src/Appwrite/Extend/Exception.php b/src/Appwrite/Extend/Exception.php index 4412505cae..86f02316bc 100644 --- a/src/Appwrite/Extend/Exception.php +++ b/src/Appwrite/Extend/Exception.php @@ -154,6 +154,7 @@ class Exception extends \Exception /** Sites */ public const SITE_NOT_FOUND = 'site_not_found'; public const SITE_FRAMEWORK_UNSUPPORTED = 'site_framework_unsupported'; + public const SITE_TEMPLATE_NOT_FOUND = 'site_template_not_found'; /** Functions */ public const FUNCTION_NOT_FOUND = 'function_not_found'; diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Sites/GetTemplate.php b/src/Appwrite/Platform/Modules/Sites/Http/Sites/GetTemplate.php new file mode 100644 index 0000000000..f9233ea4b5 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Sites/Http/Sites/GetTemplate.php @@ -0,0 +1,56 @@ +setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) + ->setHttpPath('/v1/sites/templates/:templateId') + ->desc('Get site template') + ->label('scope', 'public') + ->label('sdk.namespace', 'sites') + ->label('sdk.method', 'getTemplate') + ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) + ->label('sdk.description', '/docs/references/sites/get-template.md') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_TEMPLATE_SITE) + ->param('templateId', '', new Text(128), 'Template ID.') + ->inject('response') + ->callback([$this, 'action']); + } + + public function action(string $templateId, Response $response) + { + $templates = Config::getParam('site-templates', []); + + $template = array_shift(\array_filter($templates, function ($template) use ($templateId) { + return $template['id'] === $templateId; + })); + + if (empty($template)) { + throw new Exception(Exception::SITE_TEMPLATE_NOT_FOUND); + } + + $response->dynamic(new Document($template), Response::MODEL_TEMPLATE_SITE); + } +} diff --git a/src/Appwrite/Platform/Modules/Sites/Services/Http.php b/src/Appwrite/Platform/Modules/Sites/Services/Http.php index 5adbdf3029..b66866a2a6 100644 --- a/src/Appwrite/Platform/Modules/Sites/Services/Http.php +++ b/src/Appwrite/Platform/Modules/Sites/Services/Http.php @@ -16,6 +16,7 @@ use Appwrite\Platform\Modules\Sites\Http\Sites\DeleteSite; use Appwrite\Platform\Modules\Sites\Http\Sites\GetSite; use Appwrite\Platform\Modules\Sites\Http\Sites\GetSitesUsage; use Appwrite\Platform\Modules\Sites\Http\Sites\GetSiteUsage; +use Appwrite\Platform\Modules\Sites\Http\Sites\GetTemplate; use Appwrite\Platform\Modules\Sites\Http\Sites\ListFrameworks; use Appwrite\Platform\Modules\Sites\Http\Sites\ListSites; use Appwrite\Platform\Modules\Sites\Http\Sites\ListTemplates; @@ -60,7 +61,10 @@ class Http extends Service $this->addAction(ListVariables::getName(), new ListVariables()); $this->addAction(UpdateVariable::getName(), new UpdateVariable()); $this->addAction(DeleteVariable::getName(), new DeleteVariable()); + + // Templates $this->addAction(ListTemplates::getName(), new ListTemplates()); + $this->addAction(GetTemplate::getName(), new GetTemplate()); // Usage $this->addAction(GetSiteUsage::getName(), new GetSiteUsage()); From ac93e70dd27c3d8e99f0bd8825b8734c7f847922 Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Fri, 25 Oct 2024 15:56:28 +0200 Subject: [PATCH 22/25] Update listframeworks endpoint --- app/config/frameworks.php | 27 ++++++++++++++++++- .../Sites/Http/Sites/ListFrameworks.php | 5 ++-- src/Appwrite/Specification/Format.php | 11 ++++++++ .../Utopia/Response/Model/Framework.php | 12 ++++++--- 4 files changed, 48 insertions(+), 7 deletions(-) diff --git a/app/config/frameworks.php b/app/config/frameworks.php index 895c0cdf30..0d13c6a955 100644 --- a/app/config/frameworks.php +++ b/app/config/frameworks.php @@ -4,4 +4,29 @@ * List of Appwrite Sites supported frameworks */ -return ['sveltekit', 'nextjs']; +return [ + [ + '$id' => 'sveltekit', + 'key' => 'sveltekit', + 'name' => 'SvelteKit', + 'logo' => 'sveltekit.png', + 'defaultRuntime' => 'node-20.0', + 'runtimes' => [ + 'node-16.0', + 'node-18.0', + 'node-20.0' + ], + ], + [ + '$id' => 'nextjs', + 'key' => 'nextjs', + 'name' => 'Next.js', + 'logo' => 'nextjs.png', + 'defaultRuntime' => 'node-20.0', + 'runtimes' => [ + 'node-16.0', + 'node-18.0', + 'node-20.0' + ], + ] +]; diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Sites/ListFrameworks.php b/src/Appwrite/Platform/Modules/Sites/Http/Sites/ListFrameworks.php index e3e5aaa3e2..6620e1e17d 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Sites/ListFrameworks.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Sites/ListFrameworks.php @@ -12,7 +12,6 @@ use Utopia\System\System; class ListFrameworks extends Base { - // TODO: This won't work right now as the structure of frameworks is not properly defined, fix it later use HTTP; public static function getName() @@ -46,12 +45,12 @@ class ListFrameworks extends Base $allowList = \array_filter(\explode(',', System::getEnv('_APP_SITES_FRAMEWORKS', ''))); $allowed = []; - foreach ($frameworks as $id => $framework) { + foreach ($frameworks as $framework) { + $id = $framework['$id']; if (!empty($allowList) && !\in_array($id, $allowList)) { continue; } - $framework['$id'] = $id; $allowed[] = $framework; } diff --git a/src/Appwrite/Specification/Format.php b/src/Appwrite/Specification/Format.php index 30ce6470e1..018775e0b0 100644 --- a/src/Appwrite/Specification/Format.php +++ b/src/Appwrite/Specification/Format.php @@ -198,6 +198,17 @@ abstract class Format break; } break; + case 'sites': + switch ($method) { + case 'getUsage': + case 'getSiteUsage': + switch ($param) { + case 'range': + return 'SiteUsageRange'; + } + break; + } + // no break case 'messaging': switch ($method) { case 'getUsage': diff --git a/src/Appwrite/Utopia/Response/Model/Framework.php b/src/Appwrite/Utopia/Response/Model/Framework.php index dcdf31ad1f..ddd6322553 100644 --- a/src/Appwrite/Utopia/Response/Model/Framework.php +++ b/src/Appwrite/Utopia/Response/Model/Framework.php @@ -34,11 +34,17 @@ class Framework extends Model 'default' => '', 'example' => 'sveltekit.png', ]) - ->addRule('supports', [ + ->addRule('defaultRuntime', [ 'type' => self::TYPE_STRING, - 'description' => 'List of supported architectures.', + 'description' => 'Default runtime version.', 'default' => '', - 'example' => 'amd64', + 'example' => 'node-20.0', + ]) + ->addRule('runtimes', [ + 'type' => self::TYPE_STRING, + 'description' => 'List of supported runtime versions.', + 'default' => '', + 'example' => 'node-16.0', 'array' => true, ]) ; From 4d3d710211deaa34594889a7747b2d50b064caa7 Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Fri, 25 Oct 2024 16:05:45 +0200 Subject: [PATCH 23/25] Add missing group to templates API --- src/Appwrite/Platform/Modules/Sites/Http/Sites/GetTemplate.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Sites/GetTemplate.php b/src/Appwrite/Platform/Modules/Sites/Http/Sites/GetTemplate.php index f9233ea4b5..f8134f71c8 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Sites/GetTemplate.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Sites/GetTemplate.php @@ -26,6 +26,7 @@ class GetTemplate extends Base ->setHttpMethod(Action::HTTP_REQUEST_METHOD_GET) ->setHttpPath('/v1/sites/templates/:templateId') ->desc('Get site template') + ->groups(['api']) ->label('scope', 'public') ->label('sdk.namespace', 'sites') ->label('sdk.method', 'getTemplate') From 1a33a0c4d105579bf0598424de435a7ec6d0a225 Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Fri, 25 Oct 2024 16:23:29 +0200 Subject: [PATCH 24/25] Fix range and frameworks --- app/config/frameworks.php | 6 ++---- .../Platform/Modules/Sites/Http/Sites/CreateSite.php | 2 +- .../Modules/Sites/Http/Sites/ListFrameworks.php | 4 ++-- .../Platform/Modules/Sites/Http/Sites/UpdateSite.php | 2 +- src/Appwrite/Specification/Format.php | 10 +++++++++- 5 files changed, 15 insertions(+), 9 deletions(-) diff --git a/app/config/frameworks.php b/app/config/frameworks.php index 0d13c6a955..e8bf58286d 100644 --- a/app/config/frameworks.php +++ b/app/config/frameworks.php @@ -5,8 +5,7 @@ */ return [ - [ - '$id' => 'sveltekit', + "sveltekit" => [ 'key' => 'sveltekit', 'name' => 'SvelteKit', 'logo' => 'sveltekit.png', @@ -17,8 +16,7 @@ return [ 'node-20.0' ], ], - [ - '$id' => 'nextjs', + "nextjs" => [ 'key' => 'nextjs', 'name' => 'Next.js', 'logo' => 'nextjs.png', diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Sites/CreateSite.php b/src/Appwrite/Platform/Modules/Sites/Http/Sites/CreateSite.php index 476aad070e..c8e30a5ab9 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Sites/CreateSite.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Sites/CreateSite.php @@ -58,7 +58,7 @@ class CreateSite extends Base ->label('sdk.response.model', Response::MODEL_SITE) ->param('siteId', '', new CustomId(), 'Site ID. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') ->param('name', '', new Text(128), 'Site name. Max length: 128 chars.') - ->param('framework', '', new WhiteList(Config::getParam('frameworks'), true), 'Sites framework.') + ->param('framework', '', new WhiteList(array_keys(Config::getParam('frameworks')), true), 'Sites framework.') ->param('enabled', true, new Boolean(), 'Is site enabled? When set to \'disabled\', users cannot access the site but Server SDKs with and API key can still access the site. No data is lost when this is toggled.', true) // TODO: Add logging param later ->param('installCommand', '', new Text(8192, 0), 'Install Command.', true) ->param('buildCommand', '', new Text(8192, 0), 'Build Command.', true) diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Sites/ListFrameworks.php b/src/Appwrite/Platform/Modules/Sites/Http/Sites/ListFrameworks.php index 6620e1e17d..2d55045548 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Sites/ListFrameworks.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Sites/ListFrameworks.php @@ -45,12 +45,12 @@ class ListFrameworks extends Base $allowList = \array_filter(\explode(',', System::getEnv('_APP_SITES_FRAMEWORKS', ''))); $allowed = []; - foreach ($frameworks as $framework) { - $id = $framework['$id']; + foreach ($frameworks as $id => $framework) { if (!empty($allowList) && !\in_array($id, $allowList)) { continue; } + $framework['$id'] = $id; $allowed[] = $framework; } diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Sites/UpdateSite.php b/src/Appwrite/Platform/Modules/Sites/Http/Sites/UpdateSite.php index 67b47f00fc..b69eea7452 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Sites/UpdateSite.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Sites/UpdateSite.php @@ -55,7 +55,7 @@ class UpdateSite extends Base ->label('sdk.response.model', Response::MODEL_SITE) ->param('siteId', '', new UID(), 'Site ID.') ->param('name', '', new Text(128), 'Site name. Max length: 128 chars.') - ->param('framework', '', new WhiteList(Config::getParam('frameworks'), true), 'Sites framework.') + ->param('framework', '', new WhiteList(array_keys(Config::getParam('frameworks')), true), 'Sites framework.') ->param('enabled', true, new Boolean(), 'Is site enabled? When set to \'disabled\', users cannot access the site but Server SDKs with and API key can still access the site. No data is lost when this is toggled.', true) // TODO: Add logging param later ->param('installCommand', '', new Text(8192, 0), 'Install Command.', true) ->param('buildCommand', '', new Text(8192, 0), 'Build Command.', true) diff --git a/src/Appwrite/Specification/Format.php b/src/Appwrite/Specification/Format.php index 018775e0b0..eb284175eb 100644 --- a/src/Appwrite/Specification/Format.php +++ b/src/Appwrite/Specification/Format.php @@ -208,7 +208,7 @@ abstract class Format } break; } - // no break + break; case 'messaging': switch ($method) { case 'getUsage': @@ -397,6 +397,14 @@ abstract class Format return ['Twenty Four Hours', 'Thirty Days', 'Ninety Days']; } break; + case 'sites': + switch ($method) { + case 'getUsage': + case 'getSiteUsage': + // Range Enum Keys + return ['Twenty Four Hours', 'Thirty Days', 'Ninety Days']; + } + break; case 'users': switch ($method) { case 'getUsage': From 4ed08cefa7c97e2a236d47e3356b9604440cd6b0 Mon Sep 17 00:00:00 2001 From: Khushboo Verma <43381712+vermakhushboo@users.noreply.github.com> Date: Fri, 25 Oct 2024 18:04:32 +0200 Subject: [PATCH 25/25] Fix getTemplate endpoint --- app/config/site-templates.php | 40 +++++++++++++------ app/controllers/api/functions.php | 4 +- .../Modules/Sites/Http/Sites/GetTemplate.php | 4 +- 3 files changed, 32 insertions(+), 16 deletions(-) diff --git a/app/config/site-templates.php b/app/config/site-templates.php index baaf78469f..f1454433af 100644 --- a/app/config/site-templates.php +++ b/app/config/site-templates.php @@ -9,18 +9,16 @@ const TEMPLATE_FRAMEWORKS = [ ], ]; -function getFrameworks($framework, $installCommand, $buildCommand, $outputDirectory, $fallbackRedirect, $providerRootDirectory) +function getFramework($framework, $installCommand, $buildCommand, $outputDirectory, $fallbackRedirect, $providerRootDirectory) { - return array_map(function ($version) use ($framework, $installCommand, $buildCommand, $outputDirectory, $fallbackRedirect, $providerRootDirectory) { - return [ - 'name' => $framework['name'], - 'installCommand' => $installCommand, - 'buildCommand' => $buildCommand, - 'outputDirectory' => $outputDirectory, - 'fallbackRedirect' => $fallbackRedirect, - 'providerRootDirectory' => $providerRootDirectory - ]; - }, $framework); + return [ + 'name' => $framework['name'], + 'installCommand' => $installCommand, + 'buildCommand' => $buildCommand, + 'outputDirectory' => $outputDirectory, + 'fallbackRedirect' => $fallbackRedirect, + 'providerRootDirectory' => $providerRootDirectory + ]; } return [ @@ -32,7 +30,25 @@ return [ 'A simple site to get started. Edit this site to explore endless possibilities with Appwrite Sites.', 'useCases' => ['starter'], 'frameworks' => [ - ...getFrameworks(TEMPLATE_FRAMEWORKS['SVELTEKIT'], 'npm install', 'npm run build', 'build', 'index.html', 'node/starter') + ...getFramework(TEMPLATE_FRAMEWORKS['SVELTEKIT'], 'npm install', 'npm run build', 'build', 'index.html', 'node/starter') + ], + 'instructions' => 'For documentation and instructions check out file.', + 'vcsProvider' => 'github', + 'providerRepositoryId' => 'templates', + 'providerOwner' => 'appwrite', + 'providerVersion' => '0.2.*', + 'variables' => [], + 'scopes' => ['users.read'] + ], + [ + 'icon' => 'icon-lightning-bolt', + 'id' => 'starter1', + 'name' => 'Starter1 site', + 'tagline' => + 'A simple site to get started. Edit this site to explore endless possibilities with Appwrite Sites.', + 'useCases' => ['messaging'], + 'frameworks' => [ + ...getFramework(TEMPLATE_FRAMEWORKS['SVELTEKIT'], 'npm install', 'npm run build', 'build', 'index.html', 'node/starter1') ], 'instructions' => 'For documentation and instructions check out file.', 'vcsProvider' => 'github', diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index a942a18f35..8cfa9320f9 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -2615,8 +2615,8 @@ App::get('/v1/functions/templates/:templateId') ->action(function (string $templateId, Response $response) { $templates = Config::getParam('function-templates', []); - $template = array_shift(\array_filter($templates, function ($template) use ($templateId) { - return $template['id'] === $templateId; + $template = array_shift(array_filter($templates, function ($item) use ($templateId) { + return $item['id'] === $templateId; })); if (empty($template)) { diff --git a/src/Appwrite/Platform/Modules/Sites/Http/Sites/GetTemplate.php b/src/Appwrite/Platform/Modules/Sites/Http/Sites/GetTemplate.php index f8134f71c8..56277fb7d3 100644 --- a/src/Appwrite/Platform/Modules/Sites/Http/Sites/GetTemplate.php +++ b/src/Appwrite/Platform/Modules/Sites/Http/Sites/GetTemplate.php @@ -44,8 +44,8 @@ class GetTemplate extends Base { $templates = Config::getParam('site-templates', []); - $template = array_shift(\array_filter($templates, function ($template) use ($templateId) { - return $template['id'] === $templateId; + $template = array_shift(\array_filter($templates, function ($item) use ($templateId) { + return $item['id'] === $templateId; })); if (empty($template)) {