Merge pull request #9047 from appwrite/feat-site-logs

Logs for sites
This commit is contained in:
Matej Bačo 2024-12-02 15:31:21 +01:00 committed by GitHub
commit f0b3c95035
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
16 changed files with 361 additions and 24 deletions

View file

@ -4097,7 +4097,7 @@ $projectCollections = array_merge([
'name' => 'Executions',
'attributes' => [
[
'$id' => ID::custom('functionInternalId'),
'$id' => ID::custom('resourceInternalId'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => Database::LENGTH_KEY,
@ -4108,7 +4108,18 @@ $projectCollections = array_merge([
'filters' => [],
],
[
'$id' => ID::custom('functionId'),
'$id' => ID::custom('resourceId'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => Database::LENGTH_KEY,
'signed' => true,
'required' => false,
'default' => null,
'array' => false,
'filters' => [],
],
[
'$id' => ID::custom('resourceType'),
'type' => Database::VAR_STRING,
'format' => '',
'size' => Database::LENGTH_KEY,
@ -4297,9 +4308,9 @@ $projectCollections = array_merge([
],
'indexes' => [
[
'$id' => ID::custom('_key_function'),
'$id' => ID::custom('_key_resource'),
'type' => Database::INDEX_KEY,
'attributes' => ['functionId'],
'attributes' => ['resourceId'],
'lengths' => [Database::LENGTH_KEY],
'orders' => [Database::ORDER_ASC],
],

View file

@ -599,6 +599,13 @@ return [
'code' => 400,
],
/** Logs */
Exception::LOG_NOT_FOUND => [
'name' => Exception::LOG_NOT_FOUND,
'description' => 'Log with the requested ID could not be found.',
'code' => 404,
],
/** Databases */
Exception::DATABASE_NOT_FOUND => [
'name' => Exception::DATABASE_NOT_FOUND,

View file

@ -60,6 +60,8 @@ $admins = [
'functions.write',
'sites.read',
'sites.write',
'log.read',
'log.write',
'execution.read',
'execution.write',
'rules.read',

View file

@ -70,6 +70,12 @@ return [ // List of publicly visible scopes
'sites.write' => [
'description' => 'Access to create, update, and delete your project\'s sites and deployments',
],
'log.read' => [
'description' => 'Access to read your site\'s logs',
],
'log.write' => [
'description' => 'Access to update, and delete your site\'s logs',
],
'execution.read' => [
'description' => 'Access to read your project\'s execution logs',
],

View file

@ -1904,8 +1904,9 @@ App::post('/v1/functions/:functionId/executions')
$execution = new Document([
'$id' => $executionId,
'$permissions' => !$user->isEmpty() ? [Permission::read(Role::user($user->getId()))] : [],
'functionInternalId' => $function->getInternalId(),
'functionId' => $function->getId(),
'resourceInternalId' => $function->getInternalId(),
'resourceId' => $function->getId(),
'resourceType' => 'functions',
'deploymentInternalId' => $deployment->getInternalId(),
'deploymentId' => $deployment->getId(),
'trigger' => (!is_null($scheduledAt)) ? 'schedule' : 'http',
@ -2172,7 +2173,8 @@ App::get('/v1/functions/:functionId/executions')
}
// Set internal queries
$queries[] = Query::equal('functionId', [$function->getId()]);
$queries[] = Query::equal('resourceInternalId', [$function->getInternalId()]);
$queries[] = Query::equal('resourceType', ['functions']);
/**
* Get cursor document if there was a cursor query, we use array_filter and reset for reference $cursor to $queries
@ -2250,7 +2252,7 @@ App::get('/v1/functions/:functionId/executions/:executionId')
$execution = $dbForProject->getDocument('executions', $executionId);
if ($execution->getAttribute('functionId') !== $function->getId()) {
if ($execution->getAttribute('resourceType') !== 'functions' && $execution->getAttribute('resourceInternalId') !== $function->getInternalId()) {
throw new Exception(Exception::EXECUTION_NOT_FOUND);
}
@ -2301,7 +2303,7 @@ App::delete('/v1/functions/:functionId/executions/:executionId')
throw new Exception(Exception::EXECUTION_NOT_FOUND);
}
if ($execution->getAttribute('functionId') !== $function->getId()) {
if ($execution->getAttribute('resourceType') !== 'functions' && $execution->getAttribute('resourceInternalId') !== $function->getInternalId()) {
throw new Exception(Exception::EXECUTION_NOT_FOUND);
}
$status = $execution->getAttribute('status');

View file

@ -245,12 +245,10 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo
$execution = new Document([
'$id' => $executionId,
'$permissions' => [],
'functionInternalId' => $resource->getInternalId(),
'functionId' => $resource->getId(),
'resourceInternalId' => $resource->getInternalId(),
'resourceId' => $resource->getId(),
'deploymentInternalId' => $deployment->getInternalId(),
'deploymentId' => $deployment->getId(),
'trigger' => 'http', // http / schedule / event
'status' => 'processing', // waiting / processing / completed / failed
'responseStatusCode' => 0,
'responseHeaders' => [],
'requestPath' => $path,
@ -262,6 +260,14 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo
'search' => implode(' ', [$resourceId, $executionId]),
]);
if ($type === 'function') {
$execution->setAttribute('resourceType', 'functions');
$execution->setAttribute('trigger', 'http'); // http / schedule / event
$execution->setAttribute('status', 'processing'); // waiting / processing / completed / failed
} elseif ($type === 'site') {
$execution->setAttribute('resourceType', 'sites');
}
$queueForEvents
->setParam('functionId', $resource->getId())
->setParam('executionId', $execution->getId())
@ -370,20 +376,26 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo
/** Update execution status */
$status = $executionResponse['statusCode'] >= 500 ? 'failed' : 'completed';
$execution->setAttribute('status', $status);
$execution->setAttribute('responseStatusCode', $executionResponse['statusCode']);
$execution->setAttribute('responseHeaders', $headersFiltered);
if ($type === 'function') {
$execution->setAttribute('status', $status);
}
$execution->setAttribute('logs', $executionResponse['logs']);
$execution->setAttribute('errors', $executionResponse['errors']);
$execution->setAttribute('responseStatusCode', $executionResponse['statusCode']);
$execution->setAttribute('responseHeaders', $headersFiltered);
$execution->setAttribute('duration', $executionResponse['duration']);
} catch (\Throwable $th) {
$durationEnd = \microtime(true);
$execution
->setAttribute('duration', $durationEnd - $durationStart)
->setAttribute('responseStatusCode', 500);
if ($type === 'function') {
$execution
->setAttribute('status', 'failed')
->setAttribute('responseStatusCode', 500)
->setAttribute('errors', $th->getMessage() . '\nError Code: ' . $th->getCode());
}
Console::error($th->getMessage());
if ($th instanceof AppwriteException) {
@ -421,6 +433,8 @@ function router(App $utopia, Database $dbForConsole, callable $getProjectDB, Swo
->setExecution($execution)
->setProject($project)
->trigger();
} elseif ($type === 'site') { // TODO: Move it to logs worker later
$dbForProject->createDocument('executions', $execution);
}
}

View file

@ -309,6 +309,7 @@ const METRIC_NETWORK_OUTBOUND = 'network.outbound';
const RESOURCE_TYPE_PROJECTS = 'projects';
const RESOURCE_TYPE_FUNCTIONS = 'functions';
const RESOURCE_TYPE_SITES = 'sites';
const RESOURCE_TYPE_DATABASES = 'databases';
const RESOURCE_TYPE_BUCKETS = 'buckets';
const RESOURCE_TYPE_PROVIDERS = 'providers';

View file

@ -177,6 +177,9 @@ class Exception extends \Exception
public const EXECUTION_NOT_FOUND = 'execution_not_found';
public const EXECUTION_IN_PROGRESS = 'execution_in_progress';
/** Log */
public const LOG_NOT_FOUND = 'log_not_found';
/** Databases */
public const DATABASE_NOT_FOUND = 'database_not_found';
public const DATABASE_ALREADY_EXISTS = 'database_already_exists';

View file

@ -0,0 +1,77 @@
<?php
namespace Appwrite\Platform\Modules\Sites\Http\Logs;
use Appwrite\Event\Event;
use Appwrite\Extend\Exception;
use Appwrite\Platform\Modules\Compute\Base;
use Appwrite\Utopia\Response;
use Utopia\Database\Database;
use Utopia\Database\Validator\UID;
use Utopia\Platform\Action;
use Utopia\Platform\Scope\HTTP;
class DeleteLog extends Base
{
use HTTP;
public static function getName()
{
return 'deleteLog';
}
public function __construct()
{
$this
->setHttpMethod(Action::HTTP_REQUEST_METHOD_DELETE)
->setHttpPath('/v1/sites/:siteId/logs/:logId')
->desc('Delete log')
->groups(['api', 'sites'])
->label('scope', 'log.write')
->label('resourceType', RESOURCE_TYPE_SITES)
->label('event', 'sites.[siteId].logs.[logId].delete')
->label('audits.event', 'logs.delete')
->label('audits.resource', 'site/{request.siteId}')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'sites')
->label('sdk.method', 'deleteLog')
->label('sdk.description', '/docs/references/sites/delete-log.md') // TODO: add this file
->label('sdk.response.code', Response::STATUS_CODE_NOCONTENT)
->label('sdk.response.model', Response::MODEL_NONE)
->param('siteId', '', new UID(), 'Site ID.')
->param('logId', '', new UID(), 'Log ID.')
->inject('response')
->inject('dbForProject')
->inject('queueForEvents')
->callback([$this, 'action']);
}
public function action(string $siteId, string $logId, Response $response, Database $dbForProject, Event $queueForEvents)
{
$site = $dbForProject->getDocument('sites', $siteId);
if ($site->isEmpty()) {
throw new Exception(Exception::SITE_NOT_FOUND);
}
$log = $dbForProject->getDocument('executions', $logId);
if ($log->isEmpty()) {
throw new Exception(Exception::LOG_NOT_FOUND);
}
if ($log->getAttribute('resourceType') !== 'sites' && $log->getAttribute('resourceInternalId') !== $site->getInternalId()) {
throw new Exception(Exception::LOG_NOT_FOUND);
}
if (!$dbForProject->deleteDocument('executions', $log->getId())) {
throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove log from DB');
}
$queueForEvents
->setParam('siteId', $site->getId())
->setParam('logId', $log->getId())
->setPayload($response->output($log, Response::MODEL_EXECUTION)); // TODO: Update model
$response->noContent();
}
}

View file

@ -0,0 +1,64 @@
<?php
namespace Appwrite\Platform\Modules\Sites\Http\Logs;
use Appwrite\Extend\Exception;
use Appwrite\Platform\Modules\Compute\Base;
use Appwrite\Utopia\Response;
use Utopia\Database\Database;
use Utopia\Database\Validator\UID;
use Utopia\Platform\Action;
use Utopia\Platform\Scope\HTTP;
class GetLog extends Base
{
use HTTP;
public static function getName()
{
return 'getLog';
}
public function __construct()
{
$this
->setHttpMethod(Action::HTTP_REQUEST_METHOD_GET)
->setHttpPath('/v1/sites/:siteId/logs/:logId')
->desc('Get log')
->groups(['api', 'sites'])
->label('scope', 'log.read')
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'sites')
->label('sdk.method', 'getLog')
->label('sdk.description', '/docs/references/sites/get-log.md') // TODO: add this file
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_EXECUTION)
->param('siteId', '', new UID(), 'Site ID.')
->param('logId', '', new UID(), 'Log ID.')
->inject('response')
->inject('dbForProject')
->callback([$this, 'action']);
}
public function action(string $siteId, string $logId, Response $response, Database $dbForProject)
{
$site = $dbForProject->getDocument('sites', $siteId);
if ($site->isEmpty() || !$site->getAttribute('enabled')) {
throw new Exception(Exception::SITE_NOT_FOUND);
}
$log = $dbForProject->getDocument('executions', $logId);
if ($log->getAttribute('resourceType') !== 'sites' && $log->getAttribute('resourceInternalId') !== $site->getInternalId()) {
throw new Exception(Exception::LOG_NOT_FOUND);
}
if ($log->isEmpty()) {
throw new Exception(Exception::LOG_NOT_FOUND);
}
$response->dynamic($log, Response::MODEL_EXECUTION); //TODO: Change to model log, but model log already exists - decide what to do
}
}

View file

@ -0,0 +1,110 @@
<?php
namespace Appwrite\Platform\Modules\Sites\Http\Logs;
use Appwrite\Extend\Exception;
use Appwrite\Platform\Modules\Compute\Base;
use Appwrite\Utopia\Database\Validator\Queries\Executions;
use Appwrite\Utopia\Database\Validator\Queries\Logs;
use Appwrite\Utopia\Response;
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Exception\Query as QueryException;
use Utopia\Database\Query;
use Utopia\Database\Validator\Query\Cursor;
use Utopia\Database\Validator\UID;
use Utopia\Platform\Action;
use Utopia\Platform\Scope\HTTP;
use Utopia\Validator\Text;
class ListLogs extends Base
{
use HTTP;
public static function getName()
{
return 'listLogs';
}
public function __construct()
{
$this
->setHttpMethod(Action::HTTP_REQUEST_METHOD_GET)
->setHttpPath('/v1/sites/:siteId/logs')
->desc('List logs')
->groups(['api', 'sites'])
->label('scope', 'log.read')
->label('resourceType', RESOURCE_TYPE_SITES)
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'sites')
->label('sdk.method', 'listLogs')
->label('sdk.description', '/docs/references/sites/list-logs.md') // TODO: add this file
->label('sdk.response.code', Response::STATUS_CODE_OK)
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_EXECUTION_LIST) // TODO: Update this later
->param('siteId', '', new UID(), 'Site ID.')
->param('queries', [], new Logs(), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/queries). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long. You may filter on the following attributes: ' . implode(', ', Executions::ALLOWED_ATTRIBUTES), true)
->param('search', '', new Text(256), 'Search term to filter your list results. Max length: 256 chars.', true)
->inject('response')
->inject('dbForProject')
->callback([$this, 'action']);
}
public function action(string $siteId, array $queries, string $search, Response $response, Database $dbForProject)
{
$site = $dbForProject->getDocument('sites', $siteId);
if ($site->isEmpty() || !$site->getAttribute('enabled')) {
throw new Exception(Exception::SITE_NOT_FOUND);
}
try {
$queries = Query::parseQueries($queries);
} catch (QueryException $e) {
throw new Exception(Exception::GENERAL_QUERY_INVALID, $e->getMessage());
}
if (!empty($search)) {
$queries[] = Query::search('search', $search);
}
// Set internal queries
$queries[] = Query::equal('resourceInternalId', [$site->getInternalId()]);
$queries[] = Query::equal('resourceType', ['sites']);
/**
* Get cursor document if there was a cursor query, we use array_filter and reset for reference $cursor to $queries
*/
$cursor = \array_filter($queries, function ($query) {
return \in_array($query->getMethod(), [Query::TYPE_CURSOR_AFTER, Query::TYPE_CURSOR_BEFORE]);
});
$cursor = reset($cursor);
if ($cursor) {
/** @var Query $cursor */
$validator = new Cursor();
if (!$validator->isValid($cursor)) {
throw new Exception(Exception::GENERAL_QUERY_INVALID, $validator->getDescription());
}
$logId = $cursor->getValue();
$cursorDocument = $dbForProject->getDocument('executions', $logId);
if ($cursorDocument->isEmpty() || $cursorDocument->getAttribute('resourceType') !== 'sites') {
throw new Exception(Exception::GENERAL_CURSOR_NOT_FOUND, "Log '{$logId}' for the 'cursor' value not found.");
}
$cursor->setValue($cursorDocument);
}
$filterQueries = Query::groupByType($queries)['filters'];
$results = $dbForProject->find('executions', $queries);
$total = $dbForProject->count('executions', $filterQueries, APP_LIMIT_COUNT);
$response->dynamic(new Document([
'executions' => $results,
'total' => $total,
]), Response::MODEL_EXECUTION_LIST); // TODO: Update response model
}
}

View file

@ -27,6 +27,7 @@ class GetSite extends Base
->desc('Get site')
->groups(['api', 'sites'])
->label('scope', 'sites.read')
->label('resourceType', RESOURCE_TYPE_SITES)
->label('sdk.auth', [APP_AUTH_TYPE_KEY])
->label('sdk.namespace', 'sites')
->label('sdk.method', 'get')

View file

@ -11,6 +11,9 @@ use Appwrite\Platform\Modules\Sites\Http\Deployments\GetDeployment;
use Appwrite\Platform\Modules\Sites\Http\Deployments\ListDeployments;
use Appwrite\Platform\Modules\Sites\Http\Deployments\RebuildDeployment;
use Appwrite\Platform\Modules\Sites\Http\Deployments\UpdateDeployment;
use Appwrite\Platform\Modules\Sites\Http\Logs\DeleteLog;
use Appwrite\Platform\Modules\Sites\Http\Logs\GetLog;
use Appwrite\Platform\Modules\Sites\Http\Logs\ListLogs;
use Appwrite\Platform\Modules\Sites\Http\Sites\CreateSite;
use Appwrite\Platform\Modules\Sites\Http\Sites\DeleteSite;
use Appwrite\Platform\Modules\Sites\Http\Sites\GetSite;
@ -55,6 +58,11 @@ class Http extends Service
$this->addAction(RebuildDeployment::getName(), new RebuildDeployment());
$this->addAction(CancelDeployment::getName(), new CancelDeployment());
// Logs
$this->addAction(GetLog::getName(), new GetLog());
$this->addAction(ListLogs::getName(), new ListLogs());
$this->addAction(DeleteLog::getName(), new DeleteLog());
// Variables
$this->addAction(CreateVariable::getName(), new CreateVariable());
$this->addAction(GetVariable::getName(), new GetVariable());

View file

@ -256,8 +256,9 @@ class Functions extends Action
$execution = new Document([
'$id' => $executionId,
'$permissions' => $user->isEmpty() ? [] : [Permission::read(Role::user($user->getId()))],
'functionInternalId' => $function->getInternalId(),
'functionId' => $function->getId(),
'resourceInternalId' => $function->getInternalId(),
'resourceId' => $function->getId(),
'resourceType' => 'functions',
'deploymentInternalId' => '',
'deploymentId' => '',
'trigger' => $trigger,
@ -403,8 +404,9 @@ class Functions extends Action
$execution = new Document([
'$id' => $executionId,
'$permissions' => $user->isEmpty() ? [] : [Permission::read(Role::user($user->getId()))],
'functionInternalId' => $function->getInternalId(),
'functionId' => $function->getId(),
'resourceInternalId' => $function->getInternalId(),
'resourceId' => $function->getId(),
'resourceType' => 'functions',
'deploymentInternalId' => $deployment->getInternalId(),
'deploymentId' => $deployment->getId(),
'trigger' => $trigger,

View file

@ -0,0 +1,23 @@
<?php
namespace Appwrite\Utopia\Database\Validator\Queries;
class Logs extends Base
{
public const ALLOWED_ATTRIBUTES = [
'responseStatusCode',
'duration',
'requestMethod',
'requestPath',
'deploymentId'
];
/**
* Expression constructor
*
*/
public function __construct()
{
parent::__construct('executions', self::ALLOWED_ATTRIBUTES); //TODO: Update this later
}
}

View file

@ -37,12 +37,18 @@ class Execution extends Model
'example' => [Role::any()->toString()],
'array' => true,
])
->addRule('functionId', [
->addRule('resourceId', [
'type' => self::TYPE_STRING,
'description' => 'Function ID.',
'description' => 'Resource ID.',
'default' => '',
'example' => '5e5ea6g16897e',
])
->addRule('resourceType', [
'type' => self::TYPE_STRING,
'description' => 'Resource type.',
'default' => '',
'example' => 'sites',
])
->addRule('trigger', [
'type' => self::TYPE_STRING,
'description' => 'The trigger that caused the function to execute. Possible values can be: `http`, `schedule`, or `event`.',
@ -106,7 +112,7 @@ class Execution extends Model
])
->addRule('duration', [
'type' => self::TYPE_FLOAT,
'description' => 'Function execution duration in seconds.',
'description' => 'Resource(function/site) execution duration in seconds.',
'default' => 0,
'example' => 0.400,
])