Elgato_dark/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite synced 2026-05-23 08:58:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Merge pull request #9407 from appwrite/fix-sites-device

Browse source 
Fix: deviceForSites
This commit is contained in:
Matej Bačo 2025-02-25 17:55:49 +01:00 committed by GitHub
commit 1752e8ed83
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 56 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
app/controllers/api/health.php
View file

@ -857,9 +857,10 @@ App::get('/v1/health/storage')
->inject('response')
->inject('deviceForFiles')
->inject('deviceForFunctions')
->inject('deviceForSites')
->inject('deviceForBuilds')
->action(function (Response $response, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForBuilds) {
$devices = [$deviceForFiles, $deviceForFunctions, $deviceForBuilds];
->action(function (Response $response, Device $deviceForFiles, Device $deviceForFunctions, Device $deviceForSites, Device $deviceForBuilds) {
$devices = [$deviceForFiles, $deviceForFunctions, $deviceForSites, $deviceForBuilds];
$checkStart = \microtime(true);
foreach ($devices as $device) {

5
docker-compose.yml
View file

@ -76,6 +76,7 @@ services:
- appwrite-config:/storage/config:rw
- appwrite-certificates:/storage/certificates:rw
- appwrite-functions:/storage/functions:rw
- appwrite-sites:/storage/sites:rw
- appwrite-builds:/storage/builds:rw
- ./phpunit.xml:/usr/src/code/phpunit.xml
- ./tests:/usr/src/code/tests
@ -348,6 +349,7 @@ services:
- appwrite-uploads:/storage/uploads:rw
- appwrite-cache:/storage/cache:rw
- appwrite-functions:/storage/functions:rw
- appwrite-sites:/storage/sites:rw
- appwrite-builds:/storage/builds:rw
- appwrite-certificates:/storage/certificates:rw
- ./app:/usr/src/code/app
@ -434,6 +436,7 @@ services:
- appwrite
volumes:
- appwrite-functions:/storage/functions:rw
- appwrite-sites:/storage/sites:rw
- appwrite-builds:/storage/builds:rw
- appwrite-uploads:/storage/uploads:rw
- ./app:/usr/src/code/app
@ -968,6 +971,7 @@ services:
- /var/run/docker.sock:/var/run/docker.sock
- appwrite-builds:/storage/builds:rw
- appwrite-functions:/storage/functions:rw
- appwrite-sites:/storage/sites:rw
# Host mount nessessary to share files between executor and runtimes.
# It's not possible to share mount file between 2 containers without host mount (copying is too slow)
- /tmp:/tmp:rw
@ -1139,5 +1143,6 @@ volumes:
appwrite-uploads:
appwrite-certificates:
appwrite-functions:
appwrite-sites:
appwrite-builds:
appwrite-config:

33
src/Appwrite/Platform/Modules/Functions/Workers/Builds.php
View file

@ -59,11 +59,12 @@ class Builds extends Action
->inject('cache')
->inject('dbForProject')
->inject('deviceForFunctions')
->inject('deviceForSites')
->inject('isResourceBlocked')
->inject('deviceForFiles')
->inject('log')
->callback(fn ($message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, StatsUsage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Device $deviceForFiles, Log $log) =>
$this->action($message, $project, $dbForPlatform, $queueForEvents, $queueForFunctions, $usage, $cache, $dbForProject, $deviceForFunctions, $isResourceBlocked, $deviceForFiles, $log));
->callback(fn ($message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, StatsUsage $usage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Device $deviceForSites, callable $isResourceBlocked, Device $deviceForFiles, Log $log) =>
$this->action($message, $project, $dbForPlatform, $queueForEvents, $queueForFunctions, $usage, $cache, $dbForProject, $deviceForFunctions, $deviceForSites, $isResourceBlocked, $deviceForFiles, $log));
}
/**
@ -76,12 +77,13 @@ class Builds extends Action
* @param Cache $cache
* @param Database $dbForProject
* @param Device $deviceForFunctions
* @param Device $deviceForSites
* @param Device $deviceForFiles
* @param Log $log
* @return void
* @throws \Utopia\Database\Exception
*/
public function action(Message $message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, StatsUsage $queueForStatsUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, callable $isResourceBlocked, Device $deviceForFiles, Log $log): void
public function action(Message $message, Document $project, Database $dbForPlatform, Event $queueForEvents, Func $queueForFunctions, StatsUsage $queueForStatsUsage, Cache $cache, Database $dbForProject, Device $deviceForFunctions, Device $deviceForSites, callable $isResourceBlocked, Device $deviceForFiles, Log $log): void
{
$payload = $message->getPayload() ?? [];
@ -102,7 +104,7 @@ class Builds extends Action
case BUILD_TYPE_RETRY:
Console::info('Creating build for deployment: ' . $deployment->getId());
$github = new GitHub($cache);
$this->buildDeployment($deviceForFunctions, $deviceForFiles, $queueForFunctions, $queueForEvents, $queueForStatsUsage, $dbForPlatform, $dbForProject, $github, $project, $resource, $deployment, $template, $isResourceBlocked, $log);
$this->buildDeployment($deviceForFunctions, $deviceForSites, $deviceForFiles, $queueForFunctions, $queueForEvents, $queueForStatsUsage, $dbForPlatform, $dbForProject, $github, $project, $resource, $deployment, $template, $isResourceBlocked, $log);
break;
default:
@ -112,6 +114,7 @@ class Builds extends Action
/**
* @param Device $deviceForFunctions
* @param Device $deviceForSites
* @param Device $deviceForFiles
* @param Func $queueForFunctions
* @param Event $queueForEvents
@ -129,7 +132,7 @@ class Builds extends Action
*
* @throws Exception
*/
protected function buildDeployment(Device $deviceForFunctions, Device $deviceForFiles, Func $queueForFunctions, Event $queueForEvents, StatsUsage $queueForStatsUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, Document $project, Document $resource, Document $deployment, Document $template, callable $isResourceBlocked, Log $log): void
protected function buildDeployment(Device $deviceForFunctions, Device $deviceForSites, Device $deviceForFiles, Func $queueForFunctions, Event $queueForEvents, StatsUsage $queueForStatsUsage, Database $dbForPlatform, Database $dbForProject, GitHub $github, Document $project, Document $resource, Document $deployment, Document $template, callable $isResourceBlocked, Log $log): void
{
$resourceKey = match($resource->getCollection()) {
'functions' => 'functionId',
@ -137,6 +140,11 @@ class Builds extends Action
default => throw new \Exception('Invalid resource type')
};
$device = match ($resource->getCollection()) {
'sites' => $deviceForSites,
'functions' => $deviceForFunctions,
};
$executor = new Executor(System::getEnv('_APP_EXECUTOR_HOST'));
$log->addTag($resourceKey, $resource->getId());
@ -146,8 +154,7 @@ class Builds extends Action
throw new \Exception('Resource not found');
}
// TODO: Sites support
if ($isResourceBlocked($project, RESOURCE_TYPE_FUNCTIONS, $resource->getId())) {
if ($isResourceBlocked($project, $resourceKey === 'functions' ? RESOURCE_TYPE_FUNCTIONS : RESOURCE_TYPE_SITES, $resource->getId())) {
throw new \Exception('Resource is blocked');
}
@ -269,8 +276,8 @@ class Builds extends Action
$tarParamDirectory = \escapeshellarg($tmpTemplateDirectory . (empty($templateRootDirectory) ? '' : '/' . $templateRootDirectory));
Console::execute('tar --exclude code.tar.gz -czf ' . \escapeshellarg($tmpPathFile) . ' -C ' . \escapeshellcmd($tarParamDirectory) . ' .', '', $stdout, $stderr); // TODO: Replace escapeshellcmd with escapeshellarg if we find a way that doesnt break syntax
$source = $deviceForFunctions->getPath($deployment->getId() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION));
$result = $localDevice->transfer($tmpPathFile, $source, $deviceForFunctions);
$source = $device->getPath($deployment->getId() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION));
$result = $localDevice->transfer($tmpPathFile, $source, $device);
if (!$result) {
throw new \Exception("Unable to move file");
@ -278,7 +285,7 @@ class Builds extends Action
Console::execute('rm -rf ' . \escapeshellarg($tmpTemplateDirectory), '', $stdout, $stderr);
$directorySize = $deviceForFunctions->getFileSize($source);
$directorySize = $device->getFileSize($source);
$build = $dbForProject->updateDocument('builds', $build->getId(), $build->setAttribute('source', $source));
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment->setAttribute('path', $source)->setAttribute('size', $directorySize));
}
@ -426,8 +433,8 @@ class Builds extends Action
$tarParamDirectory = '/tmp/builds/' . $buildId . '/code' . (empty($rootDirectory) ? '' : '/' . $rootDirectory);
Console::execute('tar --exclude code.tar.gz -czf ' . \escapeshellarg($tmpPathFile) . ' -C ' . \escapeshellcmd($tarParamDirectory) . ' .', '', $stdout, $stderr); // TODO: Replace escapeshellcmd with escapeshellarg if we find a way that doesnt break syntax
$source = $deviceForFunctions->getPath($deployment->getId() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION));
$result = $localDevice->transfer($tmpPathFile, $source, $deviceForFunctions);
$source = $device->getPath($deployment->getId() . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION));
$result = $localDevice->transfer($tmpPathFile, $source, $device);
if (!$result) {
throw new \Exception("Unable to move file");
@ -437,7 +444,7 @@ class Builds extends Action
$build = $dbForProject->updateDocument('builds', $build->getId(), $build->setAttribute('source', $source));
$directorySize = $deviceForFunctions->getFileSize($source);
$directorySize = $device->getFileSize($source);
$deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment->setAttribute('path', $source)->setAttribute('size', $directorySize));
$this->runGitAction('processing', $github, $providerCommitHash, $owner, $repositoryName, $project, $resource, $deployment->getId(), $dbForProject, $dbForPlatform);

9
src/Appwrite/Platform/Modules/Sites/Http/Deployments/Builds/Create.php
View file

@ -62,11 +62,10 @@ class Create extends Action
->inject('queueForEvents')
->inject('queueForBuilds')
->inject('deviceForSites')
->inject('deviceForFunctions') //TODO: remove it later
->callback([$this, 'action']);
}
public function action(string $siteId, string $deploymentId, Response $response, Document $project, Database $dbForProject, Database $dbForPlatform, Event $queueForEvents, Build $queueForBuilds, Device $deviceForSites, Device $deviceForFunctions)
public function action(string $siteId, string $deploymentId, Response $response, Document $project, Database $dbForProject, Database $dbForPlatform, Event $queueForEvents, Build $queueForBuilds, Device $deviceForSites)
{
$site = $dbForProject->getDocument('sites', $siteId);
@ -80,14 +79,14 @@ class Create extends Action
}
$path = $deployment->getAttribute('path');
if (empty($path) || !$deviceForFunctions->exists($path)) {
if (empty($path) || !$deviceForSites->exists($path)) {
throw new Exception(Exception::DEPLOYMENT_NOT_FOUND);
}
$deploymentId = ID::unique();
$destination = $deviceForFunctions->getPath($deploymentId . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION));
$deviceForFunctions->transfer($path, $destination, $deviceForFunctions);
$destination = $deviceForSites->getPath($deploymentId . '.' . \pathinfo('code.tar.gz', PATHINFO_EXTENSION));
$deviceForSites->transfer($path, $destination, $deviceForSites);
$deployment->removeAttribute('$internalId');
$deployment = $dbForProject->createDocument('deployments', $deployment->setAttributes([

9
src/Appwrite/Platform/Modules/Sites/Http/Deployments/Create.php
View file

@ -80,13 +80,12 @@ class Create extends Action
->inject('project')
->inject('queueForEvents')
->inject('deviceForSites')
->inject('deviceForFunctions') // TODO: Remove this later once volume is added to executor
->inject('deviceForLocal')
->inject('queueForBuilds')
->callback([$this, 'action']);
}
public function action(string $siteId, ?string $installCommand, ?string $buildCommand, ?string $outputDirectory, mixed $code, mixed $activate, Request $request, Response $response, Database $dbForProject, Database $dbForPlatform, Document $project, Event $queueForEvents, Device $deviceForSites, Device $deviceForFunctions, Device $deviceForLocal, Build $queueForBuilds)
public function action(string $siteId, ?string $installCommand, ?string $buildCommand, ?string $outputDirectory, mixed $code, mixed $activate, Request $request, Response $response, Database $dbForProject, Database $dbForPlatform, Document $project, Event $queueForEvents, Device $deviceForSites, Device $deviceForLocal, Build $queueForBuilds)
{
$activate = \strval($activate) === 'true' || \strval($activate) === '1';
@ -168,7 +167,7 @@ class Create extends Action
// Save to storage
$fileSize ??= $deviceForLocal->getFileSize($fileTmpName);
$path = $deviceForFunctions->getPath($deploymentId . '.' . \pathinfo($fileName, PATHINFO_EXTENSION));
$path = $deviceForSites->getPath($deploymentId . '.' . \pathinfo($fileName, PATHINFO_EXTENSION));
$deployment = $dbForProject->getDocument('deployments', $deploymentId);
$metadata = ['content_type' => $deviceForLocal->getFileMimeType($fileTmpName)];
@ -180,7 +179,7 @@ class Create extends Action
}
}
$chunksUploaded = $deviceForFunctions->upload($fileTmpName, $path, $chunk, $chunks, $metadata);
$chunksUploaded = $deviceForSites->upload($fileTmpName, $path, $chunk, $chunks, $metadata);
if (empty($chunksUploaded)) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed moving file');
@ -203,7 +202,7 @@ class Create extends Action
}
}
$fileSize = $deviceForFunctions->getFileSize($path);
$fileSize = $deviceForSites->getFileSize($path);
if ($deployment->isEmpty()) {
$deployment = $dbForProject->createDocument('deployments', new Document([

5
src/Appwrite/Platform/Modules/Sites/Http/Deployments/Delete.php
View file

@ -59,11 +59,10 @@ class Delete extends Action
->inject('queueForDeletes')
->inject('queueForEvents')
->inject('deviceForSites')
->inject('deviceForFunctions') //TODO: remove it later
->callback([$this, 'action']);
}
public function action(string $siteId, string $deploymentId, Response $response, Database $dbForProject, DeleteEvent $queueForDeletes, Event $queueForEvents, Device $deviceForSites, Device $deviceForFunctions)
public function action(string $siteId, string $deploymentId, Response $response, Database $dbForProject, DeleteEvent $queueForDeletes, Event $queueForEvents, Device $deviceForSites)
{
$site = $dbForProject->getDocument('sites', $siteId);
if ($site->isEmpty()) {
@ -84,7 +83,7 @@ class Delete extends Action
}
if (!empty($deployment->getAttribute('path', ''))) {
if (!($deviceForFunctions->delete($deployment->getAttribute('path', '')))) {
if (!($deviceForSites->delete($deployment->getAttribute('path', '')))) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove deployment from storage');
}
}

13
src/Appwrite/Platform/Modules/Sites/Http/Deployments/Download/Get.php
View file

@ -55,11 +55,10 @@ class Get extends Action
->inject('request')
->inject('dbForProject')
->inject('deviceForSites')
->inject('deviceForFunctions') //TODO: Remove this later
->callback([$this, 'action']);
}
public function action(string $siteId, string $deploymentId, Response $response, Request $request, Database $dbForProject, Device $deviceForSites, Device $deviceForFunctions)
public function action(string $siteId, string $deploymentId, Response $response, Request $request, Database $dbForProject, Device $deviceForSites)
{
$site = $dbForProject->getDocument('sites', $siteId);
if ($site->isEmpty()) {
@ -76,7 +75,7 @@ class Get extends Action
}
$path = $deployment->getAttribute('path', '');
if (!$deviceForFunctions->exists($path)) {
if (!$deviceForSites->exists($path)) {
throw new Exception(Exception::DEPLOYMENT_NOT_FOUND);
}
@ -86,7 +85,7 @@ class Get extends Action
->addHeader('X-Peak', \memory_get_peak_usage())
->addHeader('Content-Disposition', 'attachment; filename="' . $deploymentId . '.tar.gz"');
$size = $deviceForFunctions->getFileSize($path);
$size = $deviceForSites->getFileSize($path);
$rangeHeader = $request->getHeader('range');
if (!empty($rangeHeader)) {
@ -108,13 +107,13 @@ class Get extends Action
->addHeader('Content-Length', $end - $start + 1)
->setStatusCode(Response::STATUS_CODE_PARTIALCONTENT);
$response->send($deviceForFunctions->read($path, $start, ($end - $start + 1)));
$response->send($deviceForSites->read($path, $start, ($end - $start + 1)));
}
if ($size > APP_STORAGE_READ_BUFFER) {
for ($i = 0; $i < ceil($size / MAX_OUTPUT_CHUNK_SIZE); $i++) {
$response->chunk(
$deviceForFunctions->read(
$deviceForSites->read(
$path,
($i * MAX_OUTPUT_CHUNK_SIZE),
min(MAX_OUTPUT_CHUNK_SIZE, $size - ($i * MAX_OUTPUT_CHUNK_SIZE))
@ -123,7 +122,7 @@ class Get extends Action
);
}
} else {
$response->send($deviceForFunctions->read($path));
$response->send($deviceForSites->read($path));
}
}
}

20
src/Appwrite/Platform/Workers/Deletes.php
View file

@ -93,13 +93,13 @@ class Deletes extends Action
$this->deleteProject($dbForPlatform, $getProjectDB, $deviceForFiles, $deviceForSites, $deviceForFunctions, $deviceForBuilds, $deviceForCache, $certificates, $document);
break;
case DELETE_TYPE_SITES:
$this->deleteSite($dbForPlatform, $getProjectDB, $deviceForSites, $deviceForFunctions, $deviceForBuilds, $deviceForFiles, $document, $certificates, $project);
$this->deleteSite($dbForPlatform, $getProjectDB, $deviceForSites, $deviceForBuilds, $deviceForFiles, $document, $certificates, $project);
break;
case DELETE_TYPE_FUNCTIONS:
$this->deleteFunction($dbForPlatform, $getProjectDB, $deviceForFunctions, $deviceForBuilds, $certificates, $document, $project);
break;
case DELETE_TYPE_DEPLOYMENTS:
$this->deleteDeployment($dbForPlatform, $getProjectDB, $deviceForFunctions, $deviceForBuilds, $deviceForFiles, $document, $certificates, $project);
$this->deleteDeployment($dbForPlatform, $getProjectDB, $deviceForFunctions, $deviceForSites, $deviceForBuilds, $deviceForFiles, $document, $certificates, $project);
break;
case DELETE_TYPE_USERS:
$this->deleteUser($getProjectDB, $document, $project);
@ -743,14 +743,13 @@ class Deletes extends Action
/**
* @param callable $getProjectDB
* @param Device $deviceForSites
* @param Device $deviceForFunctions
* @param Device $deviceForBuilds
* @param Document $document function document
* @param Document $project
* @return void
* @throws Exception
*/
private function deleteSite(Database $dbForPlatform, callable $getProjectDB, Device $deviceForSites, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForFiles, Document $document, CertificatesAdapter $certificates, Document $project): void
private function deleteSite(Database $dbForPlatform, callable $getProjectDB, Device $deviceForSites, Device $deviceForBuilds, Device $deviceForFiles, Document $document, CertificatesAdapter $certificates, Document $project): void
{
$dbForProject = $getProjectDB($project);
$siteId = $document->getId();
@ -785,10 +784,10 @@ class Deletes extends Action
$deploymentIds = [];
$this->deleteByGroup('deployments', [
Query::equal('resourceInternalId', [$siteInternalId])
], $dbForProject, function (Document $document) use ($project, $certificates, $deviceForFunctions, $deviceForFiles, $dbForPlatform, &$deploymentInternalIds) {
], $dbForProject, function (Document $document) use ($project, $certificates, $deviceForSites, $deviceForFiles, $dbForPlatform, &$deploymentInternalIds) {
$deploymentInternalIds[] = $document->getInternalId();
$deploymentIds[] = $document->getId();
$this->deleteDeploymentFiles($deviceForFunctions, $document);
$this->deleteDeploymentFiles($deviceForSites, $document);
$this->deleteDeploymentScreenshots($deviceForFiles, $dbForPlatform, $document);
$this->deleteDeploymentRules($dbForPlatform, $document, $project, $certificates);
});
@ -1057,13 +1056,14 @@ class Deletes extends Action
/**
* @param callable $getProjectDB
* @param Device $deviceForFunctions
* @param Device $deviceForSites
* @param Device $deviceForBuilds
* @param Document $document
* @param Document $project
* @return void
* @throws Exception
*/
private function deleteDeployment(Database $dbForPlatform, callable $getProjectDB, Device $deviceForFunctions, Device $deviceForBuilds, Device $deviceForFiles, Document $document, CertificatesAdapter $certificates, Document $project): void
private function deleteDeployment(Database $dbForPlatform, callable $getProjectDB, Device $deviceForFunctions, Device $deviceForSites, Device $deviceForBuilds, Device $deviceForFiles, Document $document, CertificatesAdapter $certificates, Document $project): void
{
$projectId = $project->getId();
$dbForProject = $getProjectDB($project);
@ -1073,7 +1073,11 @@ class Deletes extends Action
/**
* Delete deployment files
*/
$this->deleteDeploymentFiles($deviceForFunctions, $document); //TODO: For sites, this should be deviceForSites
match ($document->getAttribute('resourceType')) {
'functions' => $this->deleteDeploymentFiles($deviceForFunctions, $document),
'sites' => $this->deleteDeploymentFiles($deviceForSites, $document),
default => throw new Exception('Invalid resource type')
};
/**
* Delete deployment screenshots