Elgato_dark/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite synced 2026-05-23 08:58:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Merge pull request #8863 from appwrite/feat-sites-endpoints

Browse source 
Add timeout and add sites scopes
This commit is contained in:
Khushboo Verma 2024-10-27 16:15:00 +01:00 committed by GitHub
commit da7ae6b953
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
38 changed files with 137 additions and 898 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.env
View file

@ -68,6 +68,10 @@ _APP_SMS_PROJECTS_DENY_LIST=
_APP_STORAGE_LIMIT=30000000
_APP_STORAGE_PREVIEW_LIMIT=20000000
_APP_SITES_SIZE_LIMIT=30000000
_APP_SITES_TIMEOUT=900
_APP_SITES_BUILD_TIMEOUT=900
_APP_SITES_CPUS=8
_APP_SITES_MEMORY=8192
_APP_FUNCTIONS_SIZE_LIMIT=30000000
_APP_FUNCTIONS_TIMEOUT=900
_APP_FUNCTIONS_BUILD_TIMEOUT=900

11
app/config/collections.php
View file

@ -3380,6 +3380,17 @@ $projectCollections = array_merge([
'array' => false,
'filters' => ['subQueryProjectVariables'],
],
[
'$id' => ID::custom('timeout'),
'type' => Database::VAR_INTEGER,
'format' => '',
'size' => 0,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => ID::custom('search'),
'type' => Database::VAR_STRING,

2
app/config/roles.php
View file

@ -58,6 +58,8 @@ $admins = [
'health.read',
'functions.read',
'functions.write',
'sites.read',
'sites.write',
'execution.read',
'execution.write',
'rules.read',

6
app/config/scopes.php
View file

@ -64,6 +64,12 @@ return [ // List of publicly visible scopes
'functions.write' => [
'description' => 'Access to create, update, and delete your project\'s functions and code deployments',
],
'sites.read' => [
'description' => 'Access to read your project\'s sites and deployments',
],
'sites.write' => [
'description' => 'Access to create, update, and delete your project\'s sites and deployments',
],
'execution.read' => [
'description' => 'Access to read your project\'s execution logs',
],

15
app/config/variables.php
View file

@ -714,6 +714,21 @@ return [
],
],
],
[
'category' => 'Sites',
'description' => '',
'variables' => [
[
'name' => '_APP_SITES_SIZE_LIMIT',
'description' => 'The maximum size of a site in bytes. The default value is 30MB.',
'introduction' => '0.13.0',
'default' => '30000000',
'required' => false,
'question' => '',
'filter' => ''
],
]
],
[
'category' => 'Functions',
'description' => '',

1
app/controllers/api/console.php
View file

@ -47,6 +47,7 @@ App::get('/v1/console/variables')
'_APP_DOMAIN_TARGET' => System::getEnv('_APP_DOMAIN_TARGET'),
'_APP_STORAGE_LIMIT' => +System::getEnv('_APP_STORAGE_LIMIT'),
'_APP_FUNCTIONS_SIZE_LIMIT' => +System::getEnv('_APP_FUNCTIONS_SIZE_LIMIT'),
'_APP_SITES_SIZE_LIMIT' => +System::getEnv('_APP_SITES_SIZE_LIMIT'),
'_APP_USAGE_STATS' => System::getEnv('_APP_USAGE_STATS'),
'_APP_VCS_ENABLED' => $isVcsEnabled,
'_APP_DOMAIN_ENABLED' => $isDomainEnabled,

4
app/controllers/general.php
View file

@ -325,8 +325,7 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo
};
$entrypoint = match($type) {
'function' => $deployment->getAttribute('entrypoint', ''),
//todo: check if null works
'site' => 'placeholder' // entrypoint is required in api, but not needed with site
'site' => ''
};
$runtimeEntrypoint = match ($version) {
'v2' => '',
@ -338,7 +337,6 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo
deploymentId: $deployment->getId(),
body: \strlen($body) > 0 ? $body : null,
variables: $vars,
// todo: figure out timeouts for sites
timeout: $resource->getAttribute('timeout', 30),
image: $runtime['image'],
source: $build->getAttribute('path', ''),

6
app/views/install/compose.phtml
View file

@ -137,6 +137,12 @@ $image = $this->getParam('image', '');
- _APP_FUNCTIONS_CPUS
- _APP_FUNCTIONS_MEMORY
- _APP_FUNCTIONS_RUNTIMES
_ _APP_SITES_SIZE_LIMIT
- _APP_SITES_TIMEOUT
- _APP_SITES_BUILD_TIMEOUT
- _APP_SITES_CPUS
- _APP_SITES_MEMORY
- _APP_SITES_FRAMEWORKS
- _APP_EXECUTOR_SECRET
- _APP_EXECUTOR_HOST
- _APP_LOGGING_CONFIG

7
docker-compose.yml
View file

@ -167,6 +167,8 @@ services:
- _APP_SITES_CPUS
- _APP_SITES_MEMORY
- _APP_SITES_SIZE_LIMIT
- _APP_SITES_TIMEOUT
- _APP_SITES_BUILD_TIMEOUT
- _APP_DOMAIN_SITES
- _APP_EXECUTOR_SECRET
- _APP_EXECUTOR_HOST
@ -459,6 +461,11 @@ services:
- _APP_FUNCTIONS_CPUS
- _APP_FUNCTIONS_MEMORY
- _APP_FUNCTIONS_SIZE_LIMIT
- _APP_SITES_TIMEOUT
- _APP_SITES_BUILD_TIMEOUT
- _APP_SITES_CPUS
- _APP_SITES_MEMORY
- _APP_SITES_SIZE_LIMIT
- _APP_OPTIONS_FORCE_HTTPS
- _APP_OPTIONS_FUNCTIONS_FORCE_HTTPS
- _APP_DOMAIN

52
src/Appwrite/Platform/Modules/Functions/Workers/Builds.php
View file

@ -115,6 +115,7 @@ class Builds extends Action
* @param Log $log
* @return void
* @throws \Utopia\Database\Exception
*
* @throws Exception
*/
protected function buildDeployment(Device $deviceForFunctions, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Database $dbForConsole, Database $dbForProject, GitHub $github, Document $project, Document $resource, Document $deployment, Document $template, Log $log): void
@ -393,9 +394,13 @@ class Builds extends Action
}
$directorySize = $localDevice->getDirectorySize($tmpDirectory);
$functionsSizeLimit = (int)System::getEnv('_APP_FUNCTIONS_SIZE_LIMIT', '30000000');
if ($directorySize > $functionsSizeLimit) {
throw new \Exception('Repository directory size should be less than ' . number_format($functionsSizeLimit / 1048576, 2) . ' MBs.');
$sizeLimit = match ($resource->getCollection()) {
'functions' => (int)System::getEnv('_APP_FUNCTIONS_SIZE_LIMIT', '30000000'),
'sites' => (int)System::getEnv('_APP_SITES_SIZE_LIMIT', '50000000')
};
if ($directorySize > $sizeLimit) {
throw new \Exception('Repository directory size should be less than ' . number_format($sizeLimit / 1048576, 2) . ' MBs.');
}
$tarParamDirectory = '/tmp/builds/' . $buildId . '/code' . (empty($rootDirectory) ? '' : '/' . $rootDirectory);
@ -473,16 +478,35 @@ class Builds extends Action
$vars[$var->getAttribute('key')] = $var->getAttribute('value', '');
}
$cpus = $spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT;
$memory = max($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT, 1024); // We have a minimum of 1024MB here because some runtimes can't compile with less memory than this.
$collection = $resource->getCollection();
$jwtExpiry = (int)System::getEnv('_APP_FUNCTIONS_BUILD_TIMEOUT', 900);
$jwtObj = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', $jwtExpiry, 0);
//todo: not needed for sites yet, might be useful as a build variable but too shortlived
$apiKey = $jwtObj->encode([
'projectId' => $project->getId(),
'scopes' => $resource->getAttribute('scopes', [])
]);
$cpus = match ($collection) {
'functions' => $spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT,
'sites' => $siteVars['cpus'] ?? APP_SITE_CPUS_DEFAULT
};
$memory = match ($collection) {
'functions' => max($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT, 1024), // We have a minimum of 1024MB here because some runtimes can't compile with less memory than this.
'sites' => max($siteVars['memory'] ?? APP_SITE_MEMORY_DEFAULT, 1024)
};
$timeout = match ($collection) {
'functions' => (int) System::getEnv('_APP_FUNCTIONS_BUILD_TIMEOUT', 900),
'sites' => (int) System::getEnv('_APP_SITES_BUILD_TIMEOUT', 900),
};
// JWT and API key generation for functions only
if ($collection === 'functions') {
$jwtExpiry = (int)System::getEnv('_APP_FUNCTIONS_BUILD_TIMEOUT', 900);
$jwtObj = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', $jwtExpiry, 0);
$apiKey = $jwtObj->encode([
'projectId' => $project->getId(),
'scopes' => $resource->getAttribute('scopes', [])
]);
$vars = array_merge($vars, ['APPWRITE_FUNCTION_API_KEY' => API_KEY_DYNAMIC . '_' . $apiKey]);
}
$protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') == 'disabled' ? 'http' : 'https';
$hostname = System::getEnv('_APP_DOMAIN');
@ -512,7 +536,6 @@ class Builds extends Action
$vars = [
...$vars,
'APPWRITE_FUNCTION_API_ENDPOINT' => $endpoint,
'APPWRITE_FUNCTION_API_KEY' => API_KEY_DYNAMIC . '_' . $apiKey,
'APPWRITE_FUNCTION_ID' => $resource->getId(),
'APPWRITE_FUNCTION_NAME' => $resource->getAttribute('name'),
'APPWRITE_FUNCTION_DEPLOYMENT' => $deployment->getId(),
@ -554,7 +577,7 @@ class Builds extends Action
$isCanceled = false;
Co::join([
Co\go(function () use ($executor, &$response, $project, $deployment, $source, $resource, $runtime, $vars, $command, $cpus, $memory, &$err, $version) {
Co\go(function () use ($executor, &$response, $project, $deployment, $source, $resource, $runtime, $vars, $command, $cpus, $memory, $timeout, &$err, $version) {
try {
$command = $version === 'v2' ? 'tar -zxf /tmp/code.tar.gz -C /usr/code && cd /usr/local/src/ && ./build.sh' : 'tar -zxf /tmp/code.tar.gz -C /mnt/code && helpers/build.sh "' . \trim(\escapeshellarg($command), "\'") . '"';
@ -566,6 +589,7 @@ class Builds extends Action
version: $version,
cpus: $cpus,
memory: $memory,
timeout: $timeout,
remove: true,
entrypoint: $deployment->getAttribute('entrypoint', 'package.json'), // TODO: change this later so that sites don't need to have an entrypoint
destination: APP_STORAGE_BUILDS . "/app-{$project->getId()}",

2
src/Appwrite/Platform/Modules/Sites/Http/Deployments/CancelDeployment.php
View file

@ -32,7 +32,7 @@ class CancelDeployment extends Action
->setHttpPath('/v1/sites/:siteId/deployments/:deploymentId/build')
->desc('Cancel deployment')
->groups(['api', 'sites'])
->label('scope', 'functions.write') //TODO: Update the scope to sites later
->label('scope', 'sites.write')
->label('audits.event', 'deployment.update')
->label('audits.resource', 'site/{request.siteId}')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])

2
src/Appwrite/Platform/Modules/Sites/Http/Deployments/CreateDeployment.php
View file

@ -41,7 +41,7 @@ class CreateDeployment extends Action
->setHttpPath('/v1/sites/:siteId/deployments')
->desc('Create deployment')
->groups(['api', 'sites'])
->label('scope', 'functions.write') //TODO: Update the scope to sites later
->label('scope', 'sites.write')
->label('event', 'sites.[siteId].deployments.[deploymentId].create')
->label('audits.event', 'deployment.create')
->label('audits.resource', 'site/{request.siteId}')

2
src/Appwrite/Platform/Modules/Sites/Http/Deployments/DeleteDeployment.php
View file

@ -29,7 +29,7 @@ class DeleteDeployment extends Action
->setHttpPath('/v1/sites/:siteId/deployments/:deploymentId')
->desc('Delete deployment')
->groups(['api', 'sites'])
->label('scope', 'functions.write') //TODO: Update the scope to sites later
->label('scope', 'sites.write')
->label('event', 'sites.[siteId].deployments.[deploymentId].delete')
->label('audits.event', 'deployment.delete')
->label('audits.resource', 'site/{request.siteId}')

2
src/Appwrite/Platform/Modules/Sites/Http/Deployments/DownloadBuild.php
View file

@ -27,7 +27,7 @@ class DownloadBuild extends Action
->setHttpPath('/v1/sites/:siteId/deployments/:deploymentId/build/download')
->desc('Download build')
->groups(['api', 'sites'])
->label('scope', 'functions.read') //TODO: Update the scope to sites later
->label('scope', 'sites.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'sites')
->label('sdk.method', 'getBuildDownload')

2
src/Appwrite/Platform/Modules/Sites/Http/Deployments/DownloadDeployment.php
View file

@ -27,7 +27,7 @@ class DownloadDeployment extends Action
->setHttpPath('/v1/sites/:siteId/deployments/:deploymentId/download')
->desc('Download deployment')
->groups(['api', 'sites'])
->label('scope', 'functions.read') //TODO: Update the scope to sites later
->label('scope', 'sites.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
->label('sdk.namespace', 'sites')
->label('sdk.method', 'getDeploymentDownload')

2
src/Appwrite/Platform/Modules/Sites/Http/Deployments/GetDeployment.php
View file

@ -25,7 +25,7 @@ class GetDeployment extends Action
->setHttpPath('/v1/sites/:siteId/deployments/:deploymentId')
->desc('Get deployment')
->groups(['api', 'sites'])
->label('scope', 'functions.read') //TODO: Update the scope to sites later
->label('scope', 'sites.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'sites')
->label('sdk.method', 'getDeployment')

2
src/Appwrite/Platform/Modules/Sites/Http/Deployments/ListDeployments.php
View file

@ -31,7 +31,7 @@ class ListDeployments extends Action
->setHttpPath('/v1/sites/:siteId/deployments')
->desc('List deployments')
->groups(['api', 'sites'])
->label('scope', 'functions.read') //TODO: Update the scope to sites later
->label('scope', 'sites.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'sites')
->label('sdk.method', 'listDeployments')

2
src/Appwrite/Platform/Modules/Sites/Http/Deployments/RebuildDeployment.php
View file

@ -29,7 +29,7 @@ class RebuildDeployment extends Action
->setHttpPath('/v1/sites/:siteId/deployments/:deploymentId/build')
->desc('Rebuild deployment')
->groups(['api', 'sites'])
->label('scope', 'functions.write') //TODO: Update the scope to sites later
->label('scope', 'sites.write')
->label('event', 'sites.[siteId].deployments.[deploymentId].update')
->label('audits.event', 'deployment.update')
->label('audits.resource', 'site/{request.siteId}')

2
src/Appwrite/Platform/Modules/Sites/Http/Deployments/UpdateDeployment.php
View file

@ -27,7 +27,7 @@ class UpdateDeployment extends Action
->setHttpPath('/v1/sites/:siteId/deployments/:deploymentId')
->desc('Update deployment')
->groups(['api', 'sites'])
->label('scope', 'functions.write') //TODO: Update the scope to sites later
->label('scope', 'sites.write')
->label('event', 'sites.[siteId].deployments.[deploymentId].update')
->label('audits.event', 'deployment.update')
->label('audits.resource', 'site/{request.siteId}')

7
src/Appwrite/Platform/Modules/Sites/Http/Sites/CreateSite.php
View file

@ -25,6 +25,7 @@ use Utopia\Platform\Scope\HTTP;
use Utopia\Swoole\Request;
use Utopia\System\System;
use Utopia\Validator\Boolean;
use Utopia\Validator\Range;
use Utopia\Validator\Text;
use Utopia\Validator\WhiteList;
use Utopia\VCS\Adapter\Git\GitHub;
@ -45,7 +46,7 @@ class CreateSite extends Base
->setHttpPath('/v1/sites')
->desc('Create site')
->groups(['api', 'sites'])
->label('scope', 'functions.write') // TODO: Update scope to sites.write
->label('scope', 'sites.write')
->label('event', 'sites.[siteId].create')
->label('audits.event', 'site.create')
->label('audits.resource', 'site/{response.$id}')
@ -60,6 +61,7 @@ class CreateSite extends Base
->param('name', '', new Text(128), 'Site name. Max length: 128 chars.')
->param('framework', '', new WhiteList(array_keys(Config::getParam('frameworks')), true), 'Sites framework.')
->param('enabled', true, new Boolean(), 'Is site enabled? When set to \'disabled\', users cannot access the site but Server SDKs with and API key can still access the site. No data is lost when this is toggled.', true) // TODO: Add logging param later
->param('timeout', 15, new Range(1, (int) System::getEnv('_APP_SITES_TIMEOUT', 900)), 'Maximum request time in seconds.', true)
->param('installCommand', '', new Text(8192, 0), 'Install Command.', true)
->param('buildCommand', '', new Text(8192, 0), 'Build Command.', true)
->param('outputDirectory', '', new Text(8192, 0), 'Output Directory for site.', true)
@ -92,7 +94,7 @@ class CreateSite extends Base
->callback([$this, 'action']);
}
public function action(string $siteId, string $name, string $framework, bool $enabled, string $installCommand, string $buildCommand, string $outputDirectory, string $fallbackRedirect, string $subdomain, string $installationId, string $providerRepositoryId, string $providerBranch, bool $providerSilentMode, string $providerRootDirectory, string $templateRepository, string $templateOwner, string $templateRootDirectory, string $templateVersion, string $specification, Request $request, Response $response, Database $dbForProject, Document $project, Document $user, Event $queueForEvents, Build $queueForBuilds, Database $dbForConsole, GitHub $github)
public function action(string $siteId, string $name, string $framework, bool $enabled, int $timeout, string $installCommand, string $buildCommand, string $outputDirectory, string $fallbackRedirect, string $subdomain, string $installationId, string $providerRepositoryId, string $providerBranch, bool $providerSilentMode, string $providerRootDirectory, string $templateRepository, string $templateOwner, string $templateRootDirectory, string $templateVersion, string $specification, Request $request, Response $response, Database $dbForProject, Document $project, Document $user, Event $queueForEvents, Build $queueForBuilds, Database $dbForConsole, GitHub $github)
{
$sitesDomain = System::getEnv('_APP_DOMAIN_SITES', '');
$ruleId = '';
@ -153,6 +155,7 @@ class CreateSite extends Base
'framework' => $framework,
'deploymentInternalId' => '',
'deploymentId' => '',
'timeout' => $timeout,
'installCommand' => $installCommand,
'buildCommand' => $buildCommand,
'outputDirectory' => $outputDirectory,

2
src/Appwrite/Platform/Modules/Sites/Http/Sites/DeleteSite.php
View file

@ -28,7 +28,7 @@ class DeleteSite extends Base
->setHttpPath('/v1/sites/:siteId')
->desc('Delete site')
->groups(['api', 'sites'])
->label('scope', 'functions.write') // TODO: Update scope to sites.write
->label('scope', 'sites.write')
->label('event', 'sites.[siteId].delete')
->label('audits.event', 'site.delete')
->label('audits.resource', 'site/{request.siteId}')

2
src/Appwrite/Platform/Modules/Sites/Http/Sites/GetSite.php
View file

@ -26,7 +26,7 @@ class GetSite extends Base
->setHttpPath('/v1/sites/:siteId')
->desc('Get site')
->groups(['api', 'sites'])
->label('scope', 'functions.read') // TODO: Update scope to sites.read
->label('scope', 'sites.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'sites')
->label('sdk.method', 'get')

2
src/Appwrite/Platform/Modules/Sites/Http/Sites/GetSiteUsage.php
View file

@ -31,7 +31,7 @@ class GetSiteUsage extends Base
->setHttpPath('/v1/sites/:siteId/usage')
->desc('Get site usage')
->groups(['api', 'sites', 'usage'])
->label('scope', 'functions.read') // TODO: Update scope to sites.read
->label('scope', 'sites.read')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'sites')
->label('sdk.method', 'getSiteUsage')

2
src/Appwrite/Platform/Modules/Sites/Http/Sites/GetSitesUsage.php
View file

@ -29,7 +29,7 @@ class GetSitesUsage extends Base
->setHttpPath('/v1/sites/usage')
->desc('Get sites usage')
->groups(['api', 'sites', 'usage'])
->label('scope', 'functions.read') // TODO: Update scope to sites.read
->label('scope', 'sites.read')
->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
->label('sdk.namespace', 'sites')
->label('sdk.method', 'getUsage')

2
src/Appwrite/Platform/Modules/Sites/Http/Sites/ListFrameworks.php
View file

@ -26,7 +26,7 @@ class ListFrameworks extends Base
->setHttpPath('/v1/sites/frameworks')
->desc('List frameworks')
->groups(['api', 'sites'])
->label('scope', 'functions.read') // TODO: Update scope to sites.read
->label('scope', 'sites.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'sites')
->label('sdk.method', 'listFrameworks')

2
src/Appwrite/Platform/Modules/Sites/Http/Sites/ListSites.php
View file

@ -31,7 +31,7 @@ class ListSites extends Base
->setHttpPath('/v1/sites')
->desc('List sites')
->groups(['api', 'sites'])
->label('scope', 'functions.write') // TODO: Update scope to sites.write
->label('scope', 'sites.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'sites')
->label('sdk.method', 'list')

13
src/Appwrite/Platform/Modules/Sites/Http/Sites/UpdateSite.php
View file

@ -21,8 +21,9 @@ use Utopia\Database\Validator\UID;
use Utopia\Platform\Action;
use Utopia\Platform\Scope\HTTP;
use Utopia\Swoole\Request;
use Utopia\Validator\ArrayList;
use Utopia\System\System;
use Utopia\Validator\Boolean;
use Utopia\Validator\Range;
use Utopia\Validator\Text;
use Utopia\Validator\WhiteList;
use Utopia\VCS\Adapter\Git\GitHub;
@ -42,7 +43,7 @@ class UpdateSite extends Base
->setHttpPath('/v1/sites/:siteId')
->desc('Update site')
->groups(['api', 'sites'])
->label('scope', 'functions.write') // TODO: update it to sites.write later
->label('scope', 'sites.write')
->label('event', 'sites.[siteId].update')
->label('audits.event', 'sites.update')
->label('audits.resource', 'site/{response.$id}')
@ -57,11 +58,11 @@ class UpdateSite extends Base
->param('name', '', new Text(128), 'Site name. Max length: 128 chars.')
->param('framework', '', new WhiteList(array_keys(Config::getParam('frameworks')), true), 'Sites framework.')
->param('enabled', true, new Boolean(), 'Is site enabled? When set to \'disabled\', users cannot access the site but Server SDKs with and API key can still access the site. No data is lost when this is toggled.', true) // TODO: Add logging param later
->param('timeout', 15, new Range(1, (int) System::getEnv('_APP_SITES_TIMEOUT', 900)), 'Maximum request time in seconds.', true)
->param('installCommand', '', new Text(8192, 0), 'Install Command.', true)
->param('buildCommand', '', new Text(8192, 0), 'Build Command.', true)
->param('outputDirectory', '', new Text(8192, 0), 'Output Directory for site.', true)
->param('fallbackRedirect', '', new Text(8192, 0), 'Fallback Redirect URL for site in case a route is not found.', true)
->param('scopes', [], new ArrayList(new WhiteList(array_keys(Config::getParam('scopes')), true), APP_LIMIT_ARRAY_PARAMS_SIZE), 'List of scopes allowed for API key auto-generated for every execution. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' scopes are allowed.', true) //TODO: Update description of scopes
->param('installationId', '', new Text(128, 0), 'Appwrite Installation ID for VCS (Version Control System) deployment.', true)
->param('providerRepositoryId', '', new Text(128, 0), 'Repository ID of the repo linked to the site.', true)
->param('providerBranch', '', new Text(128, 0), 'Production branch for the repo linked to the site.', true)
@ -84,7 +85,7 @@ class UpdateSite extends Base
->callback([$this, 'action']);
}
public function action(string $siteId, string $name, string $framework, bool $enabled, string $installCommand, string $buildCommand, string $outputDirectory, string $fallbackRedirect, array $scopes, string $installationId, ?string $providerRepositoryId, string $providerBranch, bool $providerSilentMode, string $providerRootDirectory, string $specification, Request $request, Response $response, Database $dbForProject, Document $project, Event $queueForEvents, Build $queueForBuilds, Database $dbForConsole, GitHub $github)
public function action(string $siteId, string $name, string $framework, bool $enabled, int $timeout, string $installCommand, string $buildCommand, string $outputDirectory, string $fallbackRedirect, string $installationId, ?string $providerRepositoryId, string $providerBranch, bool $providerSilentMode, string $providerRootDirectory, string $specification, Request $request, Response $response, Database $dbForProject, Document $project, Event $queueForEvents, Build $queueForBuilds, Database $dbForConsole, GitHub $github)
{
// TODO: If only branch changes, re-deploy
$site = $dbForProject->getDocument('sites', $siteId);
@ -202,11 +203,11 @@ class UpdateSite extends Base
'framework' => $framework,
'enabled' => $enabled,
'live' => $live,
'buildCommand' => $buildCommand,
'timeout' => $timeout,
'installCommand' => $installCommand,
'buildCommand' => $buildCommand,
'outputDirectory' => $outputDirectory,
'fallbackRedirect' => $fallbackRedirect,
'scopes' => $scopes,
'installationId' => $installation->getId(),
'installationInternalId' => $installation->getInternalId(),
'providerRepositoryId' => $providerRepositoryId,

2
src/Appwrite/Platform/Modules/Sites/Http/Variables/CreateVariable.php
View file

@ -33,7 +33,7 @@ class CreateVariable extends Base
->setHttpPath('/v1/sites/:siteId/variables')
->desc('Create variable')
->groups(['api', 'sites'])
->label('scope', 'functions.write') // TODO: Update scope to sites.write
->label('scope', 'sites.write')
->label('audits.event', 'variable.create')
->label('audits.resource', 'site/{request.siteId}')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])

2
src/Appwrite/Platform/Modules/Sites/Http/Variables/DeleteVariable.php
View file

@ -26,7 +26,7 @@ class DeleteVariable extends Base
->setHttpPath('/v1/sites/:siteId/variables/:variableId')
->desc('Delete variable')
->groups(['api', 'sites'])
->label('scope', 'functions.write') // TODO: Update scope to sites
->label('scope', 'sites.write')
->label('audits.event', 'variable.delete')
->label('audits.resource', 'site/{request.siteId}')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])

2
src/Appwrite/Platform/Modules/Sites/Http/Variables/GetVariable.php
View file

@ -26,7 +26,7 @@ class GetVariable extends Base
->setHttpPath('/v1/sites/:siteId/variables/:variableId')
->desc('Get variable')
->groups(['api', 'sites'])
->label('scope', 'functions.read') // TODO: Update scope to sites
->label('scope', 'sites.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'sites')
->label('sdk.method', 'getVariable')

2
src/Appwrite/Platform/Modules/Sites/Http/Variables/ListVariables.php
View file

@ -27,7 +27,7 @@ class ListVariables extends Base
->setHttpPath('/v1/sites/:siteId/variables')
->desc('List variables')
->groups(['api', 'sites'])
->label('scope', 'functions.read') // TODO: Update scope to sites
->label('scope', 'sites.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'sites')
->label('sdk.method', 'listVariables')

2
src/Appwrite/Platform/Modules/Sites/Http/Variables/UpdateVariable.php
View file

@ -28,7 +28,7 @@ class UpdateVariable extends Base
->setHttpPath('/v1/sites/:siteId/variables/:variableId')
->desc('Update variable')
->groups(['api', 'sites'])
->label('scope', 'functions.write') // TODO: Update scope to sites
->label('scope', 'sites.write')
->label('audits.event', 'variable.update')
->label('audits.resource', 'site/{request.siteId}')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])

2
src/Appwrite/Platform/Services/Workers.php
View file

@ -3,7 +3,6 @@
namespace Appwrite\Platform\Services;
use Appwrite\Platform\Workers\Audits;
use Appwrite\Platform\Workers\Builds;
use Appwrite\Platform\Workers\Certificates;
use Appwrite\Platform\Workers\Databases;
use Appwrite\Platform\Workers\Deletes;
@ -23,7 +22,6 @@ class Workers extends Service
$this->type = Service::TYPE_WORKER;
$this
->addAction(Audits::getName(), new Audits())
->addAction(Builds::getName(), new Builds())
->addAction(Certificates::getName(), new Certificates())
->addAction(Databases::getName(), new Databases())
->addAction(Deletes::getName(), new Deletes())

850
src/Appwrite/Platform/Workers/Builds.php
View file

@ -1,850 +0,0 @@
<?php
namespace Appwrite\Platform\Workers;
use Ahc\Jwt\JWT;
use Appwrite\Event\Event;
use Appwrite\Event\Func;
use Appwrite\Event\Usage;
use Appwrite\Messaging\Adapter\Realtime;
use Appwrite\Utopia\Response\Model\Deployment;
use Appwrite\Vcs\Comment;
use Exception;
use Executor\Executor;
use Swoole\Coroutine as Co;
use Utopia\Cache\Cache;
use Utopia\CLI\Console;
use Utopia\Config\Config;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Database\Exception\Conflict;
use Utopia\Database\Exception\Restricted;
use Utopia\Database\Exception\Structure;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Validator\Authorization;
use Utopia\Logger\Log;
use Utopia\Platform\Action;
use Utopia\Queue\Message;
use Utopia\Storage\Device;
use Utopia\Storage\Device\Local;
use Utopia\System\System;
use Utopia\VCS\Adapter\Git\GitHub;
class Builds extends Action
{
public static function getName(): string
{
return 'builds';
}
/**
* @throws Exception
*/
public function __construct()
{
$this
->desc('Builds worker')
->inject('message')
->inject('dbForConsole')
->inject('queueForEvents')
->inject('queueForFunctions')
->inject('queueForUsage')
->inject('cache')
->inject('dbForProject')
->inject('deviceForFunctions')
->inject('log')
->callback(fn ($message, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions, Usage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log) => $this->action($message, $dbForConsole, $queueForEvents, $queueForFunctions, $usage, $cache, $dbForProject, $deviceForFunctions, $log));
}
/**
* @param Message $message
* @param Database $dbForConsole
* @param Event $queueForEvents
* @param Func $queueForFunctions
* @param Usage $queueForUsage
* @param Cache $cache
* @param Database $dbForProject
* @param Device $deviceForFunctions
* @param Log $log
* @return void
* @throws \Utopia\Database\Exception
*/
public function action(Message $message, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions, Usage $queueForUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Log $log): void
{
$payload = $message->getPayload() ?? [];
if (empty($payload)) {
throw new \Exception('Missing payload');
}
$type = $payload['type'] ?? '';
$project = new Document($payload['project'] ?? []);
$resource = new Document($payload['resource'] ?? []);
$deployment = new Document($payload['deployment'] ?? []);
$template = new Document($payload['template'] ?? []);
$log->addTag('projectId', $project->getId());
$log->addTag('type', $type);
switch ($type) {
case BUILD_TYPE_DEPLOYMENT:
case BUILD_TYPE_RETRY:
Console::info('Creating build for deployment: ' . $deployment->getId());
$github = new GitHub($cache);
$this->buildDeployment($deviceForFunctions, $queueForFunctions, $queueForEvents, $queueForUsage, $dbForConsole, $dbForProject, $github, $project, $resource, $deployment, $template, $log);
break;
default:
throw new \Exception('Invalid build type');
}
}
/**
* @param Device $deviceForFunctions
* @param Func $queueForFunctions
* @param Event $queueForEvents
* @param Usage $queueForUsage
* @param Database $dbForConsole
* @param Database $dbForProject
* @param GitHub $github
* @param Document $project
* @param Document $resource
* @param Document $deployment
* @param Document $template
* @param Log $log
* @return void
* @throws \Utopia\Database\Exception
* @throws Exception
*/
protected function buildDeployment(Device $deviceForFunctions, Func $queueForFunctions, Event $queueForEvents, Usage $queueForUsage, Database $dbForConsole, Database $dbForProject, GitHub $github, Document $project, Document $resource, Document $deployment, Document $template, Log $log): void
{
// todo: refactor
$isFunction = $resource->getCollection() === 'functions';
$isSite = $resource->getCollection() === 'sites';
$foreignKey = $isFunction ? 'functionId' : 'siteId';
$executor = new Executor(System::getEnv('_APP_EXECUTOR_HOST'));
$log->addTag($foreignKey, $resource->getId());
$resource = $dbForProject->getDocument($resource->getCollection(), $resource->getId());
if ($resource->isEmpty()) {
throw new \Exception('Function not found', 404);
}
$log->addTag('deploymentId', $deployment->getId());
$deployment = $dbForProject->getDocument('deployments', $deployment->getId());
if ($deployment->isEmpty()) {
throw new \Exception('Deployment not found', 404);
}
if (empty($deployment->getAttribute('entrypoint', ''))) {
throw new \Exception('Entrypoint for your Appwrite Function is missing. Please specify it when making deployment or update the entrypoint under your function\'s "Settings" > "Configuration" > "Entrypoint".', 500);
}
$version = $resource->getAttribute('version', 'v2');
$spec = Config::getParam('runtime-specifications')[$resource->getAttribute('specifications', APP_FUNCTION_SPECIFICATION_DEFAULT)];
$runtimes = Config::getParam($version === 'v2' ? 'runtimes-v2' : 'runtimes', []);
// todo: fix for sites using frameworks
$key = $resource->getAttribute('runtime');
$runtime = $runtimes[$key] ?? null;
if (\is_null($runtime)) {
throw new \Exception('Runtime "' . $resource->getAttribute('runtime', '') . '" is not supported');
}
// Realtime preparation
$allEvents = Event::generateEvents("{$resource->getCollection()}.[{$foreignKey}].deployments.[deploymentId].update", [
$foreignKey => $resource->getId(),
'deploymentId' => $deployment->getId()
]);
$startTime = DateTime::now();
$durationStart = \microtime(true);
$buildId = $deployment->getAttribute('buildId', '');
$build = $dbForProject->getDocument('builds', $buildId);
$isNewBuild = empty($buildId);
if ($build->isEmpty()) {
$buildId = ID::unique();
$build = $dbForProject->createDocument('builds', new Document([
'$id' => $buildId,
'$permissions' => [],
'startTime' => $startTime,
'deploymentInternalId' => $deployment->getInternalId(),
'deploymentId' => $deployment->getId(),
'status' => 'processing',
'path' => '',
'runtime' => $resource->getAttribute('runtime'),
'source' => $deployment->getAttribute('path', ''),
'sourceType' => strtolower($deviceForFunctions->getType()),
'logs' => '',
'endTime' => null,
'duration' => 0,
'size' => 0
]));
$deployment->setAttribute('buildId', $build->getId());
$deployment->setAttribute('buildInternalId', $build->getInternalId());
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment);
} elseif ($build->getAttribute('status') === 'canceled') {
Console::info('Build has been canceled');
return;
} else {
$build = $dbForProject->getDocument('builds', $buildId);
}
$source = $deployment->getAttribute('path', '');
$installationId = $deployment->getAttribute('installationId', '');
$providerRepositoryId = $deployment->getAttribute('providerRepositoryId', '');
$providerCommitHash = $deployment->getAttribute('providerCommitHash', '');
$isVcsEnabled = !empty($providerRepositoryId);
$owner = '';
$repositoryName = '';
if ($isVcsEnabled) {
$installation = $dbForConsole->getDocument('installations', $installationId);
$providerInstallationId = $installation->getAttribute('providerInstallationId');
$privateKey = System::getEnv('_APP_VCS_GITHUB_PRIVATE_KEY');
$githubAppId = System::getEnv('_APP_VCS_GITHUB_APP_ID');
$github->initializeVariables($providerInstallationId, $privateKey, $githubAppId);
}
try {
if ($isNewBuild && !$isVcsEnabled) {
// Non-vcs+Template
$templateRepositoryName = $template->getAttribute('repositoryName', '');
$templateOwnerName = $template->getAttribute('ownerName', '');
$templateVersion = $template->getAttribute('version', '');
$templateRootDirectory = $template->getAttribute('rootDirectory', '');
$templateRootDirectory = \rtrim($templateRootDirectory, '/');
$templateRootDirectory = \ltrim($templateRootDirectory, '.');
$templateRootDirectory = \ltrim($templateRootDirectory, '/');
if (!empty($templateRepositoryName) && !empty($templateOwnerName) && !empty($templateVersion)) {
$stdout = '';
$stderr = '';
// Clone template repo
$tmpTemplateDirectory = '/tmp/builds/' . $buildId . '-template';
$gitCloneCommandForTemplate = $github->generateCloneCommand($templateOwnerName, $templateRepositoryName, $templateVersion, GitHub::CLONE_TYPE_TAG, $tmpTemplateDirectory, $templateRootDirectory);
$exit = Console::execute($gitCloneCommandForTemplate, '', $stdout, $stderr);
if ($exit !== 0) {
throw new \Exception('Unable to clone code repository: ' . $stderr);
}
// Ensure directories
Console::execute('mkdir -p ' . \escapeshellarg($tmpTemplateDirectory . '/' . $templateRootDirectory), '', $stdout, $stderr);
$tmpPathFile = $tmpTemplateDirectory . '/code.tar.gz';
$localDevice = new Local();
if (substr($tmpTemplateDirectory, -1) !== '/') {
$tmpTemplateDirectory .= '/';
}
$tarParamDirectory = \escapeshellarg($tmpTemplateDirectory . (empty($templateRootDirectory) ? '' : '/' . $templateRootDirectory));
Console::execute('tar --exclude code.tar.gz -czf ' . \escapeshellarg($tmpPathFile) . ' -C ' . \escapeshellcmd($tarParamDirectory) . ' .', '', $stdout, $stderr); // TODO: Replace escapeshellcmd with escapeshellarg if we find a way that doesnt break syntax
$source = $deviceForFunctions->getPath($deployment->getId() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION));
$result = $localDevice->transfer($tmpPathFile, $source, $deviceForFunctions);
if (!$result) {
throw new \Exception("Unable to move file");
}
Console::execute('rm -rf ' . \escapeshellarg($tmpTemplateDirectory), '', $stdout, $stderr);
$directorySize = $deviceForFunctions->getFileSize($source);
$build = $dbForProject->updateDocument('builds', $build->getId(), $build->setAttribute('source', $source));
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment->setAttribute('path', $source)->setAttribute('size', $directorySize));
}
} elseif ($isNewBuild && $isVcsEnabled) {
// VCS and VCS+Temaplte
$tmpDirectory = '/tmp/builds/' . $buildId . '/code';
$rootDirectory = $resource->getAttribute('providerRootDirectory', '');
$rootDirectory = \rtrim($rootDirectory, '/');
$rootDirectory = \ltrim($rootDirectory, '.');
$rootDirectory = \ltrim($rootDirectory, '/');
$owner = $github->getOwnerName($providerInstallationId);
$repositoryName = $github->getRepositoryName($providerRepositoryId);
$cloneOwner = $deployment->getAttribute('providerRepositoryOwner', $owner);
$cloneRepository = $deployment->getAttribute('providerRepositoryName', $repositoryName);
$branchName = $deployment->getAttribute('providerBranch');
$commitHash = $deployment->getAttribute('providerCommitHash', '');
$cloneVersion = $branchName;
$cloneType = GitHub::CLONE_TYPE_BRANCH;
if (!empty($commitHash)) {
$cloneVersion = $commitHash;
$cloneType = GitHub::CLONE_TYPE_COMMIT;
}
$gitCloneCommand = $github->generateCloneCommand($cloneOwner, $cloneRepository, $cloneVersion, $cloneType, $tmpDirectory, $rootDirectory);
$stdout = '';
$stderr = '';
Console::execute('mkdir -p ' . \escapeshellarg('/tmp/builds/' . $buildId), '', $stdout, $stderr);
if ($dbForProject->getDocument('builds', $buildId)->getAttribute('status') === 'canceled') {
Console::info('Build has been canceled');
return;
}
$exit = Console::execute($gitCloneCommand, '', $stdout, $stderr);
if ($exit !== 0) {
throw new \Exception('Unable to clone code repository: ' . $stderr);
}
// Local refactoring for function folder with spaces
if (str_contains($rootDirectory, ' ')) {
$rootDirectoryWithoutSpaces = str_replace(' ', '', $rootDirectory);
$from = $tmpDirectory . '/' . $rootDirectory;
$to = $tmpDirectory . '/' . $rootDirectoryWithoutSpaces;
$exit = Console::execute('mv "' . \escapeshellarg($from) . '" "' . \escapeshellarg($to) . '"', '', $stdout, $stderr);
if ($exit !== 0) {
throw new \Exception('Unable to move function with spaces' . $stderr);
}
$rootDirectory = $rootDirectoryWithoutSpaces;
}
// Build from template
$templateRepositoryName = $template->getAttribute('repositoryName', '');
$templateOwnerName = $template->getAttribute('ownerName', '');
$templateVersion = $template->getAttribute('version', '');
$templateRootDirectory = $template->getAttribute('rootDirectory', '');
$templateRootDirectory = \rtrim($templateRootDirectory, '/');
$templateRootDirectory = \ltrim($templateRootDirectory, '.');
$templateRootDirectory = \ltrim($templateRootDirectory, '/');
if (!empty($templateRepositoryName) && !empty($templateOwnerName) && !empty($templateVersion)) {
// Clone template repo
$tmpTemplateDirectory = '/tmp/builds/' . $buildId . '/template';
$gitCloneCommandForTemplate = $github->generateCloneCommand($templateOwnerName, $templateRepositoryName, $templateVersion, GitHub::CLONE_TYPE_TAG, $tmpTemplateDirectory, $templateRootDirectory);
$exit = Console::execute($gitCloneCommandForTemplate, '', $stdout, $stderr);
if ($exit !== 0) {
throw new \Exception('Unable to clone code repository: ' . $stderr);
}
// Ensure directories
Console::execute('mkdir -p ' . \escapeshellarg($tmpTemplateDirectory . '/' . $templateRootDirectory), '', $stdout, $stderr);
Console::execute('mkdir -p ' . \escapeshellarg($tmpDirectory . '/' . $rootDirectory), '', $stdout, $stderr);
// Merge template into user repo
Console::execute('rsync -av --exclude \'.git\' ' . \escapeshellarg($tmpTemplateDirectory . '/' . $templateRootDirectory . '/') . ' ' . \escapeshellarg($tmpDirectory . '/' . $rootDirectory), '', $stdout, $stderr);
// Commit and push
$exit = Console::execute('git config --global user.email "team@appwrite.io" && git config --global user.name "Appwrite" && cd ' . \escapeshellarg($tmpDirectory) . ' && git add . && git commit -m "Create ' . \escapeshellarg($resource->getAttribute('name', '')) . ' function" && git push origin ' . \escapeshellarg($branchName), '', $stdout, $stderr);
if ($exit !== 0) {
throw new \Exception('Unable to push code repository: ' . $stderr);
}
$exit = Console::execute('cd ' . \escapeshellarg($tmpDirectory) . ' && git rev-parse HEAD', '', $stdout, $stderr);
if ($exit !== 0) {
throw new \Exception('Unable to get vcs commit SHA: ' . $stderr);
}
$providerCommitHash = \trim($stdout);
$authorUrl = "https://github.com/$cloneOwner";
$deployment->setAttribute('providerCommitHash', $providerCommitHash ?? '');
$deployment->setAttribute('providerCommitAuthorUrl', $authorUrl);
$deployment->setAttribute('providerCommitAuthor', 'Appwrite');
$deployment->setAttribute('providerCommitMessage', "Create '" . $resource->getAttribute('name', '') . "' function");
$deployment->setAttribute('providerCommitUrl', "https://github.com/$cloneOwner/$cloneRepository/commit/$providerCommitHash");
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment);
/**
* Send realtime Event
*/
$target = Realtime::fromPayload(
// Pass first, most verbose event pattern
event: $allEvents[0],
payload: $build,
project: $project
);
Realtime::send(
projectId: 'console',
payload: $build->getArrayCopy(),
events: $allEvents,
channels: $target['channels'],
roles: $target['roles']
);
}
$tmpPath = '/tmp/builds/' . $buildId;
$tmpPathFile = $tmpPath . '/code.tar.gz';
$localDevice = new Local();
if (substr($tmpDirectory, -1) !== '/') {
$tmpDirectory .= '/';
}
$directorySize = $localDevice->getDirectorySize($tmpDirectory);
$functionsSizeLimit = (int)System::getEnv('_APP_FUNCTIONS_SIZE_LIMIT', '30000000');
if ($directorySize > $functionsSizeLimit) {
throw new \Exception('Repository directory size should be less than ' . number_format($functionsSizeLimit / 1048576, 2) . ' MBs.');
}
$tarParamDirectory = '/tmp/builds/' . $buildId . '/code' . (empty($rootDirectory) ? '' : '/' . $rootDirectory);
Console::execute('tar --exclude code.tar.gz -czf ' . \escapeshellarg($tmpPathFile) . ' -C ' . \escapeshellcmd($tarParamDirectory) . ' .', '', $stdout, $stderr); // TODO: Replace escapeshellcmd with escapeshellarg if we find a way that doesnt break syntax
$source = $deviceForFunctions->getPath($deployment->getId() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION));
$result = $localDevice->transfer($tmpPathFile, $source, $deviceForFunctions);
if (!$result) {
throw new \Exception("Unable to move file");
}
Console::execute('rm -rf ' . \escapeshellarg($tmpPath), '', $stdout, $stderr);
$build = $dbForProject->updateDocument('builds', $build->getId(), $build->setAttribute('source', $source));
$directorySize = $deviceForFunctions->getFileSize($source);
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment->setAttribute('path', $source)->setAttribute('size', $directorySize));
$this->runGitAction('processing', $github, $providerCommitHash, $owner, $repositoryName, $project, $resource, $deployment->getId(), $dbForProject, $dbForConsole);
}
/** Request the executor to build the code... */
$build->setAttribute('status', 'building');
$build = $dbForProject->updateDocument('builds', $buildId, $build);
if ($isVcsEnabled) {
$this->runGitAction('building', $github, $providerCommitHash, $owner, $repositoryName, $project, $resource, $deployment->getId(), $dbForProject, $dbForConsole);
}
/** Trigger Webhook */
$deploymentModel = new Deployment();
$deploymentUpdate =
$queueForEvents
->setQueue(Event::WEBHOOK_QUEUE_NAME)
->setClass(Event::WEBHOOK_CLASS_NAME)
->setProject($project)
->setEvent("{$resource->getCollection()}.[{$foreignKey}].deployments.[deploymentId].update")
->setParam($foreignKey, $resource->getId())
->setParam('deploymentId', $deployment->getId())
->setPayload($deployment->getArrayCopy(array_keys($deploymentModel->getRules())));
$deploymentUpdate->trigger();
/** Trigger Functions */
$queueForFunctions
->from($deploymentUpdate)
->trigger();
/** Trigger Realtime */
$target = Realtime::fromPayload(
// Pass first, most verbose event pattern
event: $allEvents[0],
payload: $build,
project: $project
);
Realtime::send(
projectId: 'console',
payload: $build->getArrayCopy(),
events: $allEvents,
channels: $target['channels'],
roles: $target['roles']
);
$vars = [];
// Shared vars
foreach ($resource->getAttribute('varsProject', []) as $var) {
$vars[$var->getAttribute('key')] = $var->getAttribute('value', '');
}
// Function vars
foreach ($resource->getAttribute('vars', []) as $var) {
$vars[$var->getAttribute('key')] = $var->getAttribute('value', '');
}
$cpus = $spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT;
$memory = max($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT, 1024); // We have a minimum of 1024MB here because some runtimes can't compile with less memory than this.
$jwtExpiry = (int)System::getEnv('_APP_FUNCTIONS_BUILD_TIMEOUT', 900);
$jwtObj = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', $jwtExpiry, 0);
//todo: not needed for sites yet, might be useful as a build variable but too shortlived
$apiKey = $jwtObj->encode([
'projectId' => $project->getId(),
'scopes' => $resource->getAttribute('scopes', [])
]);
$protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') == 'disabled' ? 'http' : 'https';
$hostname = System::getEnv('_APP_DOMAIN');
$endpoint = $protocol . '://' . $hostname . "/v1";
//todo: ugly, but works
if ($isFunction) {
$vars = [
...$vars,
'APPWRITE_FUNCTION_API_ENDPOINT' => $endpoint,
'APPWRITE_FUNCTION_API_KEY' => API_KEY_DYNAMIC . '_' . $apiKey,
'APPWRITE_FUNCTION_ID' => $resource->getId(),
'APPWRITE_FUNCTION_NAME' => $resource->getAttribute('name'),
'APPWRITE_FUNCTION_DEPLOYMENT' => $deployment->getId(),
'APPWRITE_FUNCTION_PROJECT_ID' => $project->getId(),
'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'] ?? '',
'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'] ?? '',
'APPWRITE_FUNCTION_CPUS' => $cpus,
'APPWRITE_FUNCTION_MEMORY' => $memory
];
}
if ($isSite) {
$vars = [
...$vars,
'APPWRITE_SITE_ID' => $resource->getId(),
'APPWRITE_SITE_NAME' => $resource->getAttribute('name'),
'APPWRITE_SITE_DEPLOYMENT' => $deployment->getId(),
'APPWRITE_SITE_PROJECT_ID' => $project->getId(),
'APPWRITE_SITE_RUNTIME_NAME' => $runtime['name'] ?? '',
'APPWRITE_SITE_RUNTIME_VERSION' => $runtime['version'] ?? '',
'APPWRITE_SITE_CPUS' => $cpus,
'APPWRITE_SITE_MEMORY' => $memory
];
}
// Appwrite vars
$vars = \array_merge($vars, [
'APPWRITE_VERSION' => APP_VERSION_STABLE,
'APPWRITE_REGION' => $project->getAttribute('region'),
'APPWRITE_DEPLOYMENT_TYPE' => $deployment->getAttribute('type', ''),
'APPWRITE_VCS_REPOSITORY_ID' => $deployment->getAttribute('providerRepositoryId', ''),
'APPWRITE_VCS_REPOSITORY_NAME' => $deployment->getAttribute('providerRepositoryName', ''),
'APPWRITE_VCS_REPOSITORY_OWNER' => $deployment->getAttribute('providerRepositoryOwner', ''),
'APPWRITE_VCS_REPOSITORY_URL' => $deployment->getAttribute('providerRepositoryUrl', ''),
'APPWRITE_VCS_REPOSITORY_BRANCH' => $deployment->getAttribute('providerBranch', ''),
'APPWRITE_VCS_REPOSITORY_BRANCH_URL' => $deployment->getAttribute('providerBranchUrl', ''),
'APPWRITE_VCS_COMMIT_HASH' => $deployment->getAttribute('providerCommitHash', ''),
'APPWRITE_VCS_COMMIT_MESSAGE' => $deployment->getAttribute('providerCommitMessage', ''),
'APPWRITE_VCS_COMMIT_URL' => $deployment->getAttribute('providerCommitUrl', ''),
'APPWRITE_VCS_COMMIT_AUTHOR_NAME' => $deployment->getAttribute('providerCommitAuthor', ''),
'APPWRITE_VCS_COMMIT_AUTHOR_URL' => $deployment->getAttribute('providerCommitAuthorUrl', ''),
'APPWRITE_VCS_ROOT_DIRECTORY' => $deployment->getAttribute('providerRootDirectory', ''),
]);
//todo: for sites use isntall and build command
$command = $deployment->getAttribute('commands', '');
$response = null;
$err = null;
if ($dbForProject->getDocument('builds', $buildId)->getAttribute('status') === 'canceled') {
Console::info('Build has been canceled');
return;
}
$isCanceled = false;
Co::join([
Co\go(function () use ($executor, &$response, $project, $deployment, $source, $resource, $runtime, $vars, $command, $cpus, $memory, &$err) {
try {
$version = $resource->getAttribute('version', 'v2');
$command = $version === 'v2' ? 'tar -zxf /tmp/code.tar.gz -C /usr/code && cd /usr/local/src/ && ./build.sh' : 'tar -zxf /tmp/code.tar.gz -C /mnt/code && helpers/build.sh "' . \trim(\escapeshellarg($command), "\'") . '"';
$response = $executor->createRuntime(
deploymentId: $deployment->getId(),
projectId: $project->getId(),
source: $source,
image: $runtime['image'],
version: $version,
cpus: $cpus,
memory: $memory,
remove: true,
entrypoint: $deployment->getAttribute('entrypoint'),
destination: APP_STORAGE_BUILDS . "/app-{$project->getId()}",
variables: $vars,
command: $command
);
} catch (\Throwable $error) {
$err = $error;
}
}),
Co\go(function () use ($executor, $project, $deployment, &$response, &$build, $dbForProject, $allEvents, &$err, &$isCanceled) {
try {
$executor->getLogs(
deploymentId: $deployment->getId(),
projectId: $project->getId(),
callback: function ($logs) use (&$response, &$err, &$build, $dbForProject, $allEvents, $project, &$isCanceled) {
if ($isCanceled) {
return;
}
// If we have response or error from concurrent coroutine, we already have latest logs
if ($response === null && $err === null) {
$build = $dbForProject->getDocument('builds', $build->getId());
if ($build->isEmpty()) {
throw new \Exception('Build not found', 404);
}
if ($build->getAttribute('status') === 'canceled') {
$isCanceled = true;
Console::info('Ignoring realtime logs because build has been canceled');
return;
}
$logs = \mb_substr($logs, 0, null, 'UTF-8'); // Get only valid UTF8 part - removes leftover half-multibytes causing SQL errors
$build = $build->setAttribute('logs', $build->getAttribute('logs', '') . $logs);
$build = $dbForProject->updateDocument('builds', $build->getId(), $build);
/**
* Send realtime Event
*/
$target = Realtime::fromPayload(
// Pass first, most verbose event pattern
event: $allEvents[0],
payload: $build,
project: $project
);
Realtime::send(
projectId: 'console',
payload: $build->getArrayCopy(),
events: $allEvents,
channels: $target['channels'],
roles: $target['roles']
);
}
}
);
} catch (\Throwable $error) {
if (empty($err)) {
$err = $error;
}
}
}),
]);
if ($dbForProject->getDocument('builds', $buildId)->getAttribute('status') === 'canceled') {
Console::info('Build has been canceled');
return;
}
if ($err) {
throw $err;
}
$endTime = DateTime::now();
$durationEnd = \microtime(true);
$buildSizeLimit = (int)System::getEnv('_APP_FUNCTIONS_BUILD_SIZE_LIMIT', '2000000000');
if ($response['size'] > $buildSizeLimit) {
throw new \Exception('Build size should be less than ' . number_format($buildSizeLimit / 1048576, 2) . ' MBs.');
}
/** Update the build document */
$build->setAttribute('startTime', DateTime::format((new \DateTime())->setTimestamp(floor($response['startTime']))));
$build->setAttribute('endTime', $endTime);
$build->setAttribute('duration', \intval(\ceil($durationEnd - $durationStart)));
$build->setAttribute('status', 'ready');
$build->setAttribute('path', $response['path']);
$build->setAttribute('size', $response['size']);
$build->setAttribute('logs', $response['output']);
$build = $dbForProject->updateDocument('builds', $buildId, $build);
if ($isVcsEnabled) {
$this->runGitAction('ready', $github, $providerCommitHash, $owner, $repositoryName, $project, $resource, $deployment->getId(), $dbForProject, $dbForConsole);
}
Console::success("Build id: $buildId created");
/** Set auto deploy */
if ($deployment->getAttribute('activate') === true) {
$resource->setAttribute('deploymentInternalId', $deployment->getInternalId());
$resource->setAttribute('deployment', $deployment->getId());
$resource->setAttribute('live', true);
$resource = $dbForProject->updateDocument('functions', $resource->getId(), $resource);
}
if ($dbForProject->getDocument('builds', $buildId)->getAttribute('status') === 'canceled') {
Console::info('Build has been canceled');
return;
}
/** Update function schedule */
// Inform scheduler if function is still active
if ($isFunction) {
$schedule = $dbForConsole->getDocument('schedules', $resource->getAttribute('scheduleId'));
$schedule
->setAttribute('resourceUpdatedAt', DateTime::now())
->setAttribute('schedule', $resource->getAttribute('schedule'))
->setAttribute('active', !empty($resource->getAttribute('schedule')) && !empty($resource->getAttribute('deployment')));
Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule));
}
} catch (\Throwable $th) {
if ($dbForProject->getDocument('builds', $buildId)->getAttribute('status') === 'canceled') {
Console::info('Build has been canceled');
return;
}
$endTime = DateTime::now();
$durationEnd = \microtime(true);
$build->setAttribute('endTime', $endTime);
$build->setAttribute('duration', \intval(\ceil($durationEnd - $durationStart)));
$build->setAttribute('status', 'failed');
$build->setAttribute('logs', $th->getMessage());
$build = $dbForProject->updateDocument('builds', $buildId, $build);
if ($isVcsEnabled) {
$this->runGitAction('failed', $github, $providerCommitHash, $owner, $repositoryName, $project, $resource, $deployment->getId(), $dbForProject, $dbForConsole);
}
} finally {
/**
* Send realtime Event
*/
$target = Realtime::fromPayload(
// Pass first, most verbose event pattern
event: $allEvents[0],
payload: $build,
project: $project
);
Realtime::send(
projectId: 'console',
payload: $build->getArrayCopy(),
events: $allEvents,
channels: $target['channels'],
roles: $target['roles']
);
/** Trigger usage queue */
if ($build->getAttribute('status') === 'ready') {
if ($isFunction) {
$queueForUsage
->addMetric(METRIC_BUILDS_SUCCESS, 1) // per project
->addMetric(METRIC_BUILDS_COMPUTE_SUCCESS, (int)$build->getAttribute('duration', 0) * 1000)
->addMetric(str_replace('{functionInternalId}', $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_SUCCESS), 1) // per function
->addMetric(str_replace('{functionInternalId}', $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE_SUCCESS), (int)$build->getAttribute('duration', 0) * 1000);
}
} elseif ($build->getAttribute('status') === 'failed') {
if ($isFunction) {
$queueForUsage
->addMetric(METRIC_BUILDS_FAILED, 1) // per project
->addMetric(METRIC_BUILDS_COMPUTE_FAILED, (int)$build->getAttribute('duration', 0) * 1000)
->addMetric(str_replace('{functionInternalId}', $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_FAILED), 1) // per function
->addMetric(str_replace('{functionInternalId}', $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE_FAILED), (int)$build->getAttribute('duration', 0) * 1000);
}
}
if ($isFunction) {
$queueForUsage
->addMetric(METRIC_BUILDS, 1) // per project
->addMetric(METRIC_BUILDS_STORAGE, $build->getAttribute('size', 0))
->addMetric(METRIC_BUILDS_COMPUTE, (int)$build->getAttribute('duration', 0) * 1000)
->addMetric(METRIC_BUILDS_MB_SECONDS, (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $build->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT)))
->addMetric(str_replace('{functionInternalId}', $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS), 1) // per function
->addMetric(str_replace('{functionInternalId}', $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE), $build->getAttribute('size', 0))
->addMetric(str_replace('{functionInternalId}', $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE), (int)$build->getAttribute('duration', 0) * 1000)
->addMetric(str_replace('{functionInternalId}', $resource->getInternalId(), METRIC_FUNCTION_ID_BUILDS_MB_SECONDS), (int)(($spec['memory'] ?? APP_FUNCTION_MEMORY_DEFAULT) * $build->getAttribute('duration', 0) * ($spec['cpus'] ?? APP_FUNCTION_CPUS_DEFAULT)))
->setProject($project)
->trigger();
}
}
}
/**
* @param string $status
* @param GitHub $github
* @param string $providerCommitHash
* @param string $owner
* @param string $repositoryName
* @param Document $project
* @param Document $resource
* @param string $deploymentId
* @param Database $dbForProject
* @param Database $dbForConsole
* @return void
* @throws Structure
* @throws \Utopia\Database\Exception
* @throws Authorization
* @throws Conflict
* @throws Restricted
*/
protected function runGitAction(string $status, GitHub $github, string $providerCommitHash, string $owner, string $repositoryName, Document $project, Document $resource, string $deploymentId, Database $dbForProject, Database $dbForConsole): void
{
if ($resource->getAttribute('providerSilentMode', false) === true) {
return;
}
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
$commentId = $deployment->getAttribute('providerCommentId', '');
if (!empty($providerCommitHash)) {
$message = match ($status) {
'ready' => 'Build succeeded.',
'failed' => 'Build failed.',
'processing' => 'Building...',
default => $status
};
$state = match ($status) {
'ready' => 'success',
'failed' => 'failure',
'processing' => 'pending',
default => $status
};
$resourceName = $resource->getAttribute('name');
$projectName = $project->getAttribute('name');
$name = "{$resourceName} ({$projectName})";
$protocol = System::getEnv('_APP_OPTIONS_FORCE_HTTPS') == 'disabled' ? 'http' : 'https';
$hostname = System::getEnv('_APP_DOMAIN');
$providerTargetUrl = "{$protocol}://{$hostname}/console/project-{$project->getId()}/functions/function-{$resource->getId()}";
$github->updateCommitStatus($repositoryName, $providerCommitHash, $owner, $state, $message, $providerTargetUrl, $name);
}
if (!empty($commentId)) {
$retries = 0;
while (true) {
$retries++;
try {
$dbForConsole->createDocument('vcsCommentLocks', new Document([
'$id' => $commentId
]));
break;
} catch (\Throwable $err) {
if ($retries >= 9) {
throw $err;
}
\sleep(1);
}
}
// Wrap in try/finally to ensure lock file gets deleted
try {
$comment = new Comment();
$comment->parseComment($github->getComment($owner, $repositoryName, $commentId));
$comment->addBuild($project, $resource, $status, $deployment->getId(), ['type' => 'logs']);
$github->updateComment($owner, $repositoryName, $commentId, $comment->generateComment());
} finally {
$dbForConsole->deleteDocument('vcsCommentLocks', $commentId);
}
}
}
}

6
src/Appwrite/Utopia/Response/Model/ConsoleVariables.php
View file

@ -28,6 +28,12 @@ class ConsoleVariables extends Model
'default' => '',
'example' => '30000000',
])
->addRule('_APP_SITES_SIZE_LIMIT', [
'type' => self::TYPE_INTEGER,
'description' => 'Maximum file size allowed for site deployment in bytes.',
'default' => '',
'example' => '30000000',
])
->addRule('_APP_USAGE_STATS', [
'type' => self::TYPE_STRING,
'description' => 'Defines if usage stats are enabled. This value is set to \'enabled\' by default, to disable the usage stats set the value to \'disabled\'.',

6
src/Appwrite/Utopia/Response/Model/Site.php
View file

@ -65,6 +65,12 @@ class Site extends Model
'example' => [],
'array' => true
])
->addRule('timeout', [
'type' => self::TYPE_INTEGER,
'description' => 'Site request timeout in seconds.',
'default' => 15,
'example' => 300,
])
->addRule('installCommand', [
'type' => self::TYPE_STRING,
'description' => 'The install command used to install the site dependencies.',

2
src/Executor/Executor.php
View file

@ -62,6 +62,7 @@ class Executor
string $version,
float $cpus,
int $memory,
int $timeout,
bool $remove = false,
string $entrypoint = '',
string $destination = '',
@ -71,7 +72,6 @@ class Executor
) {
$runtimeId = "$projectId-$deploymentId-build";
$route = "/runtimes";
$timeout = (int) System::getEnv('_APP_FUNCTIONS_BUILD_TIMEOUT', 900);
// Remove after migration
if ($version == 'v3') {

1
tests/e2e/Services/Console/ConsoleConsoleClientTest.php
View file

@ -28,6 +28,7 @@ class ConsoleConsoleClientTest extends Scope
$this->assertIsString($response['body']['_APP_DOMAIN_TARGET']);
$this->assertIsInt($response['body']['_APP_STORAGE_LIMIT']);
$this->assertIsInt($response['body']['_APP_FUNCTIONS_SIZE_LIMIT']);
$this->assertIsInt($response['body']['_APP_SITES_SIZE_LIMIT']);
$this->assertIsString($response['body']['_APP_DOMAIN_TARGET']);
$this->assertIsBool($response['body']['_APP_DOMAIN_ENABLED']);
$this->assertIsBool($response['body']['_APP_VCS_ENABLED']);