From e87435dd617abdab00ac373685e1094a4a5391d4 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 10 Aug 2021 14:29:31 +0545 Subject: [PATCH 001/206] wip usage tasks --- app/cli.php | 1 + app/tasks/usage.php | 74 +++++++++++++++++++++++++++++++++++++++++++++ bin/usage | 3 ++ 3 files changed, 78 insertions(+) create mode 100644 app/tasks/usage.php create mode 100755 bin/usage diff --git a/app/cli.php b/app/cli.php index ee6fe1a801..b48c853a24 100644 --- a/app/cli.php +++ b/app/cli.php @@ -15,6 +15,7 @@ include 'tasks/migrate.php'; include 'tasks/sdks.php'; include 'tasks/ssl.php'; include 'tasks/vars.php'; +include 'tasks/usage.php'; $cli ->task('version') diff --git a/app/tasks/usage.php b/app/tasks/usage.php new file mode 100644 index 0000000000..769d241042 --- /dev/null +++ b/app/tasks/usage.php @@ -0,0 +1,74 @@ +task('usage') + ->desc('Schedules syncing data from influxdb to Appwrite console db') + ->action(function () use ($register, $dbForConsole) { + Console::title('Usage Sync V1'); + Console::success(APP_NAME . ' usage sync process v1 has started'); + + $interval = (int) App::getEnv('_APP_USAGE_SYNC_INTERVAL', '10'); //10 seconds + + // Console::loop(function () use ($interval, $register, $dbForConsole) { + $time = date('d-m-Y H:i:s', time()); + Console::info("[{$time}] Syncing usage data from influxdb to Appwrite Console DB every {$interval} seconds"); + + $client = $register->get('influxdb'); + + $client = $register->get('influxdb'); + + $requests = []; + $network = []; + $functions = []; + + if ($client) { + // $start30 = DateTime::createFromFormat('U', \strtotime('-30 minutes'))->format(DateTime::RFC3339); + $start = DateTime::createFromFormat('U', \strtotime('-30 days'))->format(DateTime::RFC3339); + $end = DateTime::createFromFormat('U', \strtotime('now'))->format(DateTime::RFC3339); + $database = $client->selectDB('telegraf'); + + // Requests + $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_requests_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' GROUP BY time(' . '1d' . ') FILL(null)'); + $points = $result->getPoints(); + + foreach ($points as $point) { + $requests[] = [ + 'value' => (!empty($point['value'])) ? $point['value'] : 0, + 'date' => \strtotime($point['time']), + ]; + } + + // Network + $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_network_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' GROUP BY time(' . '1d' . ') FILL(null)'); + $points = $result->getPoints(); + + foreach ($points as $point) { + $network[] = [ + 'value' => (!empty($point['value'])) ? $point['value'] : 0, + 'date' => \strtotime($point['time']), + ]; + } + + // Functions + $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_executions_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' GROUP BY time(' . '1d' . ') FILL(null)'); + $points = $result->getPoints(); + + foreach ($points as $point) { + $functions[] = [ + 'value' => (!empty($point['value'])) ? $point['value'] : 0, + 'date' => \strtotime($point['time']), + ]; + } + + var_dump($requests, $network, $functions); + } + + // }, $interval); + }); diff --git a/bin/usage b/bin/usage new file mode 100755 index 0000000000..2709200ae4 --- /dev/null +++ b/bin/usage @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php usage $@ \ No newline at end of file From 0f69216d66d247a33bc82f639f8fac979de0de9e Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 11 Aug 2021 10:59:27 +0545 Subject: [PATCH 002/206] collection for usage stats --- app/config/collections2.php | 78 +++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/app/config/collections2.php b/app/config/collections2.php index ae9a4feeb1..ce9f4374ec 100644 --- a/app/config/collections2.php +++ b/app/config/collections2.php @@ -1403,6 +1403,84 @@ $collections = [ ], ], ], + 'stats' => [ + '$collection' => Database::COLLECTIONS, + '$id' => 'stats', + 'name' => 'Stats', + 'attributes' => [ + [ + '$id' => 'metric', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 255, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'value', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'time', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'period', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 4, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'type', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 1, + 'signed' => false, + 'required' => true, + 'default' => 0, // 0 -> count, 1 -> sum + 'array' => false, + 'filters' => [], + ], + ], + 'indexes' => [ + [ + '$id' => '_key_time', + 'type' => Database::INDEX_KEY, + 'attributes' => ['time'], + 'lengths' => [100], + 'orders' => [Database::ORDER_DESC], + ], + [ + '$id' => '_key_time_period', + 'type' => Database::INDEX_KEY, + 'attributes' => ['time', 'period'], + 'lengths' => [100, 4], + 'orders' => [Database::ORDER_DESC], + ], + ], + ] ]; return $collections; From d77c36dcca35703c039f1fd27cbac5eb9b077ea3 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 11 Aug 2021 11:29:08 +0545 Subject: [PATCH 003/206] usage container definition --- Dockerfile | 1 + docker-compose.yml | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/Dockerfile b/Dockerfile index 68b53747be..dd0dde11d2 100755 --- a/Dockerfile +++ b/Dockerfile @@ -225,6 +225,7 @@ RUN mkdir -p /storage/uploads && \ # Executables RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/maintenance && \ + chmod +x /usr/local/bin/usage && \ chmod +x /usr/local/bin/install && \ chmod +x /usr/local/bin/migrate && \ chmod +x /usr/local/bin/schedule && \ diff --git a/docker-compose.yml b/docker-compose.yml index 42c909ced5..a1880dd434 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -344,6 +344,30 @@ services: - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT + appwrite-usage: + entrypoint: usage + container_name: appwrite-usage + build: + context: . + networks: + - appwrite + volumes: + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src + depends_on: + - influxdb + - mariadb + environment: + - _APP_ENV + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_INFLUXDB_HOST + - _APP_INFLUXDB_PORT + - _APP_USAGE_SYNC_INTERVAL + appwrite-schedule: entrypoint: schedule container_name: appwrite-schedule From a29847e7fa8e89de68b1a13360af7ffcb7f14fa5 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 11 Aug 2021 14:12:37 +0545 Subject: [PATCH 004/206] debugger support for appwrite-usage container --- docker-compose.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index a1880dd434..707057d324 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -345,15 +345,22 @@ services: - _APP_MAINTENANCE_RETENTION_AUDIT appwrite-usage: - entrypoint: usage + entrypoint: + - php + - -e + - /usr/src/code/app/cli.php + - usage container_name: appwrite-usage build: context: . + args: + - DEBUG=false networks: - appwrite volumes: - ./app:/usr/src/code/app - ./src:/usr/src/code/src + - ./dev:/usr/local/dev depends_on: - influxdb - mariadb From 949abf876b8df51078a1d8ced29ed46b9032e8e6 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 11 Aug 2021 16:00:00 +0545 Subject: [PATCH 005/206] basic setup for syncing usage stats to appwrite db --- app/config/collections2.php | 4 +- app/tasks/usage.php | 206 ++++++++++++++++++++++++++---------- app/workers.php | 13 +++ 3 files changed, 166 insertions(+), 57 deletions(-) diff --git a/app/config/collections2.php b/app/config/collections2.php index ce9f4374ec..2b2419204e 100644 --- a/app/config/collections2.php +++ b/app/config/collections2.php @@ -1469,14 +1469,14 @@ $collections = [ '$id' => '_key_time', 'type' => Database::INDEX_KEY, 'attributes' => ['time'], - 'lengths' => [100], + 'lengths' => [], 'orders' => [Database::ORDER_DESC], ], [ '$id' => '_key_time_period', 'type' => Database::INDEX_KEY, 'attributes' => ['time', 'period'], - 'lengths' => [100, 4], + 'lengths' => [], 'orders' => [Database::ORDER_DESC], ], ], diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 769d241042..857c71de40 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -1,74 +1,170 @@ task('usage') ->desc('Schedules syncing data from influxdb to Appwrite console db') - ->action(function () use ($register, $dbForConsole) { + ->action(function () use ($register) { Console::title('Usage Sync V1'); Console::success(APP_NAME . ' usage sync process v1 has started'); - $interval = (int) App::getEnv('_APP_USAGE_SYNC_INTERVAL', '10'); //10 seconds + $interval = (int) App::getEnv('_APP_USAGE_SYNC_INTERVAL', '30'); //30 seconds - // Console::loop(function () use ($interval, $register, $dbForConsole) { - $time = date('d-m-Y H:i:s', time()); - Console::info("[{$time}] Syncing usage data from influxdb to Appwrite Console DB every {$interval} seconds"); + $cacheAdapter = new Cache(new Redis($register->get('cache'))); + $dbForConsole = new Database(new MariaDB($register->get('db')), $cacheAdapter); + $dbForConsole->setNamespace('project_console_internal'); - $client = $register->get('influxdb'); + Authorization::disable(); + $projects = $dbForConsole->find('projects'); + $projectIds = []; + foreach ($projects as $project) { + $projectIds[] = $project['$id']; + } + $projects = null; - $client = $register->get('influxdb'); - - $requests = []; - $network = []; - $functions = []; - - if ($client) { - // $start30 = DateTime::createFromFormat('U', \strtotime('-30 minutes'))->format(DateTime::RFC3339); - $start = DateTime::createFromFormat('U', \strtotime('-30 days'))->format(DateTime::RFC3339); - $end = DateTime::createFromFormat('U', \strtotime('now'))->format(DateTime::RFC3339); - $database = $client->selectDB('telegraf'); - - // Requests - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_requests_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' GROUP BY time(' . '1d' . ') FILL(null)'); - $points = $result->getPoints(); - - foreach ($points as $point) { - $requests[] = [ - 'value' => (!empty($point['value'])) ? $point['value'] : 0, - 'date' => \strtotime($point['time']), - ]; - } - - // Network - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_network_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' GROUP BY time(' . '1d' . ') FILL(null)'); - $points = $result->getPoints(); - - foreach ($points as $point) { - $network[] = [ - 'value' => (!empty($point['value'])) ? $point['value'] : 0, - 'date' => \strtotime($point['time']), - ]; - } - - // Functions - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_executions_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' GROUP BY time(' . '1d' . ') FILL(null)'); - $points = $result->getPoints(); - - foreach ($points as $point) { - $functions[] = [ - 'value' => (!empty($point['value'])) ? $point['value'] : 0, - 'date' => \strtotime($point['time']), - ]; - } - - var_dump($requests, $network, $functions); + $latestData = []; + $dbForProject = new Database(new MariaDB($register->get('db')), $cacheAdapter); + foreach ($projectIds as $id) { + $dbForProject->setNamespace("project_{$id}_internal"); + $doc = $dbForProject->find('stats', [new Query("period", Query::TYPE_EQUAL, ["1d"])], 1, 0, ['time'], [Database::ORDER_DESC]); + $doc = reset($doc); + $latestData[$id]["1d"] = $doc ? $doc->getAttribute('time') : null; + $doc = $dbForProject->find('stats', [new Query("period", Query::TYPE_EQUAL, ["30m"])], 1, 0, ['time'], [Database::ORDER_DESC]); + $doc = reset($doc); + $latestData[$id]["30m"] = $doc ? $doc->getAttribute('time') : null; } - // }, $interval); + $firstRun = true; + Console::loop(function () use ($interval, $register, $projectIds, $latestData, $dbForProject, &$firstRun) { + $time = date('d-m-Y H:i:s', time()); + Console::info("[{$time}] Syncing usage data from influxdb to Appwrite Console DB every {$interval} seconds"); + + $client = $register->get('influxdb'); + + $requests = []; + $network = []; + $functions = []; + + if ($client) { + foreach ($projectIds as $id) { + if ($firstRun) { + $start = DateTime::createFromFormat('U', \strtotime('-1 days'))->format(DateTime::RFC3339); + if (!empty($latestData[$id]["30m"])) { + $start = DateTime::createFromFormat('U', $latestData[$id]['30m'])->format(DateTime::RFC3339); + } + } else { + $start = DateTime::createFromFormat('U', \strtotime("-{$interval} seconds"))->format(DateTime::RFC3339); + } + $end = DateTime::createFromFormat('U', \strtotime('now'))->format(DateTime::RFC3339); + $database = $client->selectDB('telegraf'); + + // Requests + $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_requests_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $id . '\' GROUP BY time(' . '30m' . ') FILL(null)'); + $points = $result->getPoints(); + + $dbForProject->setNamespace("project_{$id}_internal"); + foreach ($points as $point) { + $requests[] = [ + 'value' => (!empty($point['value'])) ? $point['value'] : 0, + 'date' => \strtotime($point['time']), + 'dateStr' => $point['time'], + 'point' => $point, + ]; + $time = \strtotime($point['time']); + $id = \md5($time . '_' . '30m' . '_requests'); + $value = (!empty($point['value'])) ? $point['value'] : 0; + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'period' => '30m', + 'time' => $time, + 'metric' => 'requests', + 'value' => $value, + 'type' => 0, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $firstRun ? $value : $document->getAttribute('value') + $value)); + } + + } + + // Network + $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_network_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $id . '\'GROUP BY time(' . '30m' . ') FILL(null)'); + $points = $result->getPoints(); + + foreach ($points as $point) { + $network[] = [ + 'value' => (!empty($point['value'])) ? $point['value'] : 0, + 'date' => \strtotime($point['time']), + 'dateStr' => $point['time'], + 'point' => $point, + ]; + $time = \strtotime($point['time']); + $id = \md5($time . '_' . '30m' . '_network'); + $value = (!empty($point['value'])) ? $point['value'] : 0; + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'period' => '30m', + 'time' => $time, + 'metric' => 'network', + 'value' => $value, + 'type' => 0, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $firstRun ? $value : $document->getAttribute('value') + $value)); + } + } + + // Functions + $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_executions_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $id . '\'GROUP BY time(' . '30m' . ') FILL(null)'); + $points = $result->getPoints(); + + foreach ($points as $point) { + $functions[] = [ + 'value' => (!empty($point['value'])) ? $point['value'] : 0, + 'date' => \strtotime($point['time']), + 'dateStr' => $point['time'], + 'point' => $point, + ]; + $time = \strtotime($point['time']); + $id = \md5($time . '_' . '30m' . '_functions'); + $value = (!empty($point['value'])) ? $point['value'] : 0; + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'period' => '30m', + 'time' => $time, + 'metric' => 'functions', + 'value' => $value, + 'type' => 0, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $firstRun ? $value : $document->getAttribute('value') + $value)); + } + } + + } + } + $firstRun = false; + }, $interval); }); diff --git a/app/workers.php b/app/workers.php index 7dd48eaff8..56dc862fc6 100644 --- a/app/workers.php +++ b/app/workers.php @@ -32,3 +32,16 @@ $register->set('cache', function () { // Register cache connection return $redis; }); +$register->set('influxdb', function () { // Register DB connection + $host = App::getEnv('_APP_INFLUXDB_HOST', ''); + $port = App::getEnv('_APP_INFLUXDB_PORT', ''); + + if (empty($host) || empty($port)) { + return; + } + $driver = new InfluxDB\Driver\Curl("http://{$host}:{$port}"); + $client = new InfluxDB\Client($host, $port, '', '', false, false, 5); + $client->setDriver($driver); + + return $client; +}); \ No newline at end of file From dfde2b970f5a6defa0a60af8abe0b22a6e51c6aa Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 12 Aug 2021 11:38:02 +0545 Subject: [PATCH 006/206] refactor to sync usage for 1d and 30m periods --- app/tasks/usage.php | 247 ++++++++++++++++++++++++-------------------- 1 file changed, 134 insertions(+), 113 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 857c71de40..6154f5c969 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -5,7 +5,7 @@ global $cli, $register; require_once __DIR__ . '/../workers.php'; use Utopia\App; -use Utopia\Cache\Adapter\Redis; +use Utopia\Cache\Adapter\None; use Utopia\Cache\Cache; use Utopia\CLI\Console; use Utopia\Database\Adapter\MariaDB; @@ -23,7 +23,7 @@ $cli $interval = (int) App::getEnv('_APP_USAGE_SYNC_INTERVAL', '30'); //30 seconds - $cacheAdapter = new Cache(new Redis($register->get('cache'))); + $cacheAdapter = new Cache(new None()); $dbForConsole = new Database(new MariaDB($register->get('db')), $cacheAdapter); $dbForConsole->setNamespace('project_console_internal'); @@ -31,13 +31,13 @@ $cli $projects = $dbForConsole->find('projects'); $projectIds = []; foreach ($projects as $project) { - $projectIds[] = $project['$id']; + $projectIds[$project['$id']] = true; } $projects = null; $latestData = []; $dbForProject = new Database(new MariaDB($register->get('db')), $cacheAdapter); - foreach ($projectIds as $id) { + foreach ($projectIds as $id => $value) { $dbForProject->setNamespace("project_{$id}_internal"); $doc = $dbForProject->find('stats', [new Query("period", Query::TYPE_EQUAL, ["1d"])], 1, 0, ['time'], [Database::ORDER_DESC]); $doc = reset($doc); @@ -48,123 +48,144 @@ $cli } $firstRun = true; - Console::loop(function () use ($interval, $register, $projectIds, $latestData, $dbForProject, &$firstRun) { + Console::loop(function () use ($interval, $register, &$projectIds, &$latestData, $dbForProject, $dbForConsole, &$firstRun) { $time = date('d-m-Y H:i:s', time()); Console::info("[{$time}] Syncing usage data from influxdb to Appwrite Console DB every {$interval} seconds"); $client = $register->get('influxdb'); - - $requests = []; - $network = []; - $functions = []; + //fetch delta projects + if (!$firstRun) { + $projects = $dbForConsole->find('projects'); + foreach ($projects as $project) { + $id = $project['$id']; + if (!$projectIds[$id]) { + $projectIds[$id] = true; + if ($latestData[$id] == null) { + $dbForProject->setNamespace("project_{$id}_internal"); + $doc = $dbForProject->find('stats', [new Query("period", Query::TYPE_EQUAL, ["1d"])], 1, 0, ['time'], [Database::ORDER_DESC]); + $doc = reset($doc); + $latestData[$id]["1d"] = $doc ? $doc->getAttribute('time') : null; + $doc = $dbForProject->find('stats', [new Query("period", Query::TYPE_EQUAL, ["30m"])], 1, 0, ['time'], [Database::ORDER_DESC]); + $doc = reset($doc); + $latestData[$id]["30m"] = $doc ? $doc->getAttribute('time') : null; + } + } + } + } if ($client) { - foreach ($projectIds as $id) { - if ($firstRun) { - $start = DateTime::createFromFormat('U', \strtotime('-1 days'))->format(DateTime::RFC3339); - if (!empty($latestData[$id]["30m"])) { - $start = DateTime::createFromFormat('U', $latestData[$id]['30m'])->format(DateTime::RFC3339); - } - } else { - $start = DateTime::createFromFormat('U', \strtotime("-{$interval} seconds"))->format(DateTime::RFC3339); - } - $end = DateTime::createFromFormat('U', \strtotime('now'))->format(DateTime::RFC3339); - $database = $client->selectDB('telegraf'); - - // Requests - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_requests_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $id . '\' GROUP BY time(' . '30m' . ') FILL(null)'); - $points = $result->getPoints(); - - $dbForProject->setNamespace("project_{$id}_internal"); - foreach ($points as $point) { - $requests[] = [ - 'value' => (!empty($point['value'])) ? $point['value'] : 0, - 'date' => \strtotime($point['time']), - 'dateStr' => $point['time'], - 'point' => $point, - ]; - $time = \strtotime($point['time']); - $id = \md5($time . '_' . '30m' . '_requests'); - $value = (!empty($point['value'])) ? $point['value'] : 0; - $document = $dbForProject->getDocument('stats', $id); - if ($document->isEmpty()) { - $dbForProject->createDocument('stats', new Document([ - '$id' => $id, - 'period' => '30m', - 'time' => $time, - 'metric' => 'requests', - 'value' => $value, - 'type' => 0, - ])); - } else { - $dbForProject->updateDocument('stats', $document->getId(), - $document->setAttribute('value', $firstRun ? $value : $document->getAttribute('value') + $value)); - } - - } - - // Network - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_network_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $id . '\'GROUP BY time(' . '30m' . ') FILL(null)'); - $points = $result->getPoints(); - - foreach ($points as $point) { - $network[] = [ - 'value' => (!empty($point['value'])) ? $point['value'] : 0, - 'date' => \strtotime($point['time']), - 'dateStr' => $point['time'], - 'point' => $point, - ]; - $time = \strtotime($point['time']); - $id = \md5($time . '_' . '30m' . '_network'); - $value = (!empty($point['value'])) ? $point['value'] : 0; - $document = $dbForProject->getDocument('stats', $id); - if ($document->isEmpty()) { - $dbForProject->createDocument('stats', new Document([ - '$id' => $id, - 'period' => '30m', - 'time' => $time, - 'metric' => 'network', - 'value' => $value, - 'type' => 0, - ])); - } else { - $dbForProject->updateDocument('stats', $document->getId(), - $document->setAttribute('value', $firstRun ? $value : $document->getAttribute('value') + $value)); - } - } - - // Functions - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_executions_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $id . '\'GROUP BY time(' . '30m' . ') FILL(null)'); - $points = $result->getPoints(); - - foreach ($points as $point) { - $functions[] = [ - 'value' => (!empty($point['value'])) ? $point['value'] : 0, - 'date' => \strtotime($point['time']), - 'dateStr' => $point['time'], - 'point' => $point, - ]; - $time = \strtotime($point['time']); - $id = \md5($time . '_' . '30m' . '_functions'); - $value = (!empty($point['value'])) ? $point['value'] : 0; - $document = $dbForProject->getDocument('stats', $id); - if ($document->isEmpty()) { - $dbForProject->createDocument('stats', new Document([ - '$id' => $id, - 'period' => '30m', - 'time' => $time, - 'metric' => 'functions', - 'value' => $value, - 'type' => 0, - ])); - } else { - $dbForProject->updateDocument('stats', $document->getId(), - $document->setAttribute('value', $firstRun ? $value : $document->getAttribute('value') + $value)); - } - } - + foreach ($projectIds as $id => $value) { + syncData($client, $id, '30m', $latestData, $dbForProject); + syncData($client, $id, '1d', $latestData, $dbForProject); } } $firstRun = false; }, $interval); }); + +function syncData($client, $projectId, $period, &$latestData, $dbForProject) +{ + $requests = []; + $network = []; + $functions = []; + $start = DateTime::createFromFormat('U', \strtotime($period == '1d' ? '-90 days' : '-1 days'))->format(DateTime::RFC3339); + if (!empty($latestData[$projectId][$period])) { + $start = DateTime::createFromFormat('U', $latestData[$projectId][$period])->format(DateTime::RFC3339); + } + $end = DateTime::createFromFormat('U', \strtotime('now'))->format(DateTime::RFC3339); + $database = $client->selectDB('telegraf'); + + // Requests + $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_requests_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $projectId . '\' GROUP BY time(' . $period . ') FILL(null)'); + $points = $result->getPoints(); + + $dbForProject->setNamespace("project_{$projectId}_internal"); + foreach ($points as $point) { + $requests[] = [ + 'value' => (!empty($point['value'])) ? $point['value'] : 0, + 'date' => \strtotime($point['time']), + 'dateStr' => $point['time'], + 'point' => $point, + ]; + $time = \strtotime($point['time']); + $id = \md5($time . '_' . $period . '_requests'); + $value = (!empty($point['value'])) ? $point['value'] : 0; + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'period' => $period, + 'time' => $time, + 'metric' => 'requests', + 'value' => $value, + 'type' => 0, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $value)); + } + $latestData[$id]["30m"] = $time; + } + + // Network + $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_network_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $projectId . '\'GROUP BY time(' . $period . ') FILL(null)'); + $points = $result->getPoints(); + + foreach ($points as $point) { + $network[] = [ + 'value' => (!empty($point['value'])) ? $point['value'] : 0, + 'date' => \strtotime($point['time']), + 'dateStr' => $point['time'], + 'point' => $point, + ]; + $time = \strtotime($point['time']); + $id = \md5($time . '_' . $period . '_network'); + $value = (!empty($point['value'])) ? $point['value'] : 0; + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'period' => $period, + 'time' => $time, + 'metric' => 'network', + 'value' => $value, + 'type' => 0, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $value)); + } + $latestData[$id]["30m"] = $time; + } + + // Functions + $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_executions_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $id . '\'GROUP BY time(' . $period . ') FILL(null)'); + $points = $result->getPoints(); + + foreach ($points as $point) { + $functions[] = [ + 'value' => (!empty($point['value'])) ? $point['value'] : 0, + 'date' => \strtotime($point['time']), + 'dateStr' => $point['time'], + 'point' => $point, + ]; + $time = \strtotime($point['time']); + $id = \md5($time . '_' . $period . '_functions'); + $value = (!empty($point['value'])) ? $point['value'] : 0; + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'period' => $period, + 'time' => $time, + 'metric' => 'functions', + 'value' => $value, + 'type' => 0, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $value)); + } + $latestData[$id]["30m"] = $time; + } +} From 9ce9c32335402ad56fae01d47f33ff9ed5487d00 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 12 Aug 2021 16:56:33 +0545 Subject: [PATCH 007/206] refactor --- app/tasks/usage.php | 64 +++++++++++++++++++++------------------------ 1 file changed, 30 insertions(+), 34 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 6154f5c969..4d85535097 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -26,51 +26,32 @@ $cli $cacheAdapter = new Cache(new None()); $dbForConsole = new Database(new MariaDB($register->get('db')), $cacheAdapter); $dbForConsole->setNamespace('project_console_internal'); + $dbForProject = new Database(new MariaDB($register->get('db')), $cacheAdapter); Authorization::disable(); - $projects = $dbForConsole->find('projects'); $projectIds = []; - foreach ($projects as $project) { - $projectIds[$project['$id']] = true; - } + $latestProject = null; + $latestData = []; + do { + $projects = $dbForConsole->find('projects', [], 100, orderAfter:$latestProject); + if (!empty($projects)) { + $latestProject = $projects[array_key_last($projects)]; + $latestData = getLatestData($projects, $latestData, $dbForProject); + } + } while (!empty($projects)); + $projects = null; - $latestData = []; - $dbForProject = new Database(new MariaDB($register->get('db')), $cacheAdapter); - foreach ($projectIds as $id => $value) { - $dbForProject->setNamespace("project_{$id}_internal"); - $doc = $dbForProject->find('stats', [new Query("period", Query::TYPE_EQUAL, ["1d"])], 1, 0, ['time'], [Database::ORDER_DESC]); - $doc = reset($doc); - $latestData[$id]["1d"] = $doc ? $doc->getAttribute('time') : null; - $doc = $dbForProject->find('stats', [new Query("period", Query::TYPE_EQUAL, ["30m"])], 1, 0, ['time'], [Database::ORDER_DESC]); - $doc = reset($doc); - $latestData[$id]["30m"] = $doc ? $doc->getAttribute('time') : null; - } - $firstRun = true; - Console::loop(function () use ($interval, $register, &$projectIds, &$latestData, $dbForProject, $dbForConsole, &$firstRun) { + Console::loop(function () use ($interval, $register, &$projectIds, &$latestData, $dbForProject, $dbForConsole, &$firstRun, &$latestProject) { $time = date('d-m-Y H:i:s', time()); Console::info("[{$time}] Syncing usage data from influxdb to Appwrite Console DB every {$interval} seconds"); $client = $register->get('influxdb'); - //fetch delta projects if (!$firstRun) { - $projects = $dbForConsole->find('projects'); - foreach ($projects as $project) { - $id = $project['$id']; - if (!$projectIds[$id]) { - $projectIds[$id] = true; - if ($latestData[$id] == null) { - $dbForProject->setNamespace("project_{$id}_internal"); - $doc = $dbForProject->find('stats', [new Query("period", Query::TYPE_EQUAL, ["1d"])], 1, 0, ['time'], [Database::ORDER_DESC]); - $doc = reset($doc); - $latestData[$id]["1d"] = $doc ? $doc->getAttribute('time') : null; - $doc = $dbForProject->find('stats', [new Query("period", Query::TYPE_EQUAL, ["30m"])], 1, 0, ['time'], [Database::ORDER_DESC]); - $doc = reset($doc); - $latestData[$id]["30m"] = $doc ? $doc->getAttribute('time') : null; - } - } - } + $projects = $dbForConsole->find('projects', limit:100, orderAfter:$latestProject); + $latestProject = $projects[array_key_last($projects)]; + $latestData = getLatestData($projects, $latestData, $dbForProject); } if ($client) { @@ -83,6 +64,21 @@ $cli }, $interval); }); +function getLatestData(&$projects, &$latestData, $dbForProject) +{ + foreach ($projects as $project) { + $id = $project->getId(); + $projectIds[$id] = true; + $dbForProject->setNamespace("project_{$id}_internal"); + $doc = $dbForProject->findOne('stats', [new Query("period", Query::TYPE_EQUAL, ["1d"])], 0, ['time'], [Database::ORDER_DESC]); + $latestData[$id]["1d"] = $doc ? $doc->getAttribute('time') : null; + $doc = $dbForProject->findOne('stats', [new Query("period", Query::TYPE_EQUAL, ["30m"])], 0, ['time'], [Database::ORDER_DESC]); + $latestData[$id]["30m"] = $doc ? $doc->getAttribute('time') : null; + } + $projects = null; + return $latestData; +} + function syncData($client, $projectId, $period, &$latestData, $dbForProject) { $requests = []; From 56e3a2bd6cb95fc8d984d4a6e803f07d6d7b5eeb Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 13 Aug 2021 14:24:46 +0545 Subject: [PATCH 008/206] depricate worker.php, use init.php instead --- app/cli.php | 2 +- app/init.php | 25 ++++++++++++++++++++++++- app/tasks/usage.php | 14 ++++++++------ app/workers/audits.php | 2 +- app/workers/certificates.php | 2 +- app/workers/database.php | 2 +- app/workers/deletes.php | 2 +- app/workers/functions.php | 2 +- app/workers/mails.php | 2 +- app/workers/webhooks.php | 2 +- 10 files changed, 40 insertions(+), 15 deletions(-) diff --git a/app/cli.php b/app/cli.php index b48c853a24..99a77d6382 100644 --- a/app/cli.php +++ b/app/cli.php @@ -1,6 +1,6 @@ set('statsd', function () { // Register DB connection return $statsd; }); - $register->set('smtp', function () { $mail = new PHPMailer(true); @@ -324,6 +324,29 @@ $register->set('smtp', function () { $register->set('geodb', function () { return new Reader(__DIR__.'/db/DBIP/dbip-country-lite-2021-06.mmdb'); }); +$register->set('db', function () { // This is usually for our workers or CLI commands scope + $dbHost = App::getEnv('_APP_DB_HOST', ''); + $dbUser = App::getEnv('_APP_DB_USER', ''); + $dbPass = App::getEnv('_APP_DB_PASS', ''); + $dbScheme = App::getEnv('_APP_DB_SCHEMA', ''); + + $pdo = new PDO("mysql:host={$dbHost};dbname={$dbScheme};charset=utf8mb4", $dbUser, $dbPass, array( + PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8mb4', + PDO::ATTR_TIMEOUT => 3, // Seconds + PDO::ATTR_PERSISTENT => true, + PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, + PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, + )); + + return $pdo; +}); +$register->set('cache', function () { // This is usually for our workers or CLI commands scope + $redis = new Redis(); + $redis->pconnect(App::getEnv('_APP_REDIS_HOST', ''), App::getEnv('_APP_REDIS_PORT', '')); + $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); + + return $redis; +}); /* * Localization diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 4d85535097..ae7de85e82 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -2,10 +2,10 @@ global $cli, $register; -require_once __DIR__ . '/../workers.php'; +require_once __DIR__ . '/../init.php'; use Utopia\App; -use Utopia\Cache\Adapter\None; +use Utopia\Cache\Adapter\Redis; use Utopia\Cache\Cache; use Utopia\CLI\Console; use Utopia\Database\Adapter\MariaDB; @@ -23,7 +23,7 @@ $cli $interval = (int) App::getEnv('_APP_USAGE_SYNC_INTERVAL', '30'); //30 seconds - $cacheAdapter = new Cache(new None()); + $cacheAdapter = new Cache(new Redis($register->get('cache'))); $dbForConsole = new Database(new MariaDB($register->get('db')), $cacheAdapter); $dbForConsole->setNamespace('project_console_internal'); $dbForProject = new Database(new MariaDB($register->get('db')), $cacheAdapter); @@ -47,13 +47,15 @@ $cli $time = date('d-m-Y H:i:s', time()); Console::info("[{$time}] Syncing usage data from influxdb to Appwrite Console DB every {$interval} seconds"); - $client = $register->get('influxdb'); if (!$firstRun) { $projects = $dbForConsole->find('projects', limit:100, orderAfter:$latestProject); - $latestProject = $projects[array_key_last($projects)]; - $latestData = getLatestData($projects, $latestData, $dbForProject); + if (!empty($projects)) { + $latestProject = $projects[array_key_last($projects)]; + $latestData = getLatestData($projects, $latestData, $dbForProject); + } } + $client = $register->get('influxdb'); if ($client) { foreach ($projectIds as $id => $value) { syncData($client, $id, '30m', $latestData, $dbForProject); diff --git a/app/workers/audits.php b/app/workers/audits.php index 663de3ce4f..b3e6759627 100644 --- a/app/workers/audits.php +++ b/app/workers/audits.php @@ -4,7 +4,7 @@ use Appwrite\Resque\Worker; use Utopia\Audit\Audit; use Utopia\CLI\Console; -require_once __DIR__.'/../workers.php'; +require_once __DIR__.'/../init.php'; Console::title('Audits V1 Worker'); Console::success(APP_NAME.' audits worker v1 has started'); diff --git a/app/workers/certificates.php b/app/workers/certificates.php index 96148b9e71..0b64c087f4 100644 --- a/app/workers/certificates.php +++ b/app/workers/certificates.php @@ -9,7 +9,7 @@ use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; use Utopia\Domains\Domain; -require_once __DIR__.'/../workers.php'; +require_once __DIR__.'/../init.php'; Console::title('Certificates V1 Worker'); Console::success(APP_NAME.' certificates worker v1 has started'); diff --git a/app/workers/database.php b/app/workers/database.php index 60c060ee7a..3f551db27c 100644 --- a/app/workers/database.php +++ b/app/workers/database.php @@ -5,7 +5,7 @@ use Utopia\CLI\Console; use Utopia\Database\Document; use Utopia\Database\Validator\Authorization; -require_once __DIR__.'/../workers.php'; +require_once __DIR__.'/../init.php'; Console::title('Database V1 Worker'); Console::success(APP_NAME.' database worker v1 has started'."\n"); diff --git a/app/workers/deletes.php b/app/workers/deletes.php index e98723a5ea..f0e66d2b46 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -11,7 +11,7 @@ use Utopia\Abuse\Adapters\TimeLimit; use Utopia\CLI\Console; use Utopia\Audit\Audit; -require_once __DIR__.'/../workers.php'; +require_once __DIR__.'/../init.php'; Console::title('Deletes V1 Worker'); Console::success(APP_NAME.' deletes worker v1 has started'."\n"); diff --git a/app/workers/functions.php b/app/workers/functions.php index bed0bda138..49b42b9492 100644 --- a/app/workers/functions.php +++ b/app/workers/functions.php @@ -13,7 +13,7 @@ use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Validator\Authorization; -require_once __DIR__.'/../workers.php'; +require_once __DIR__.'/../init.php'; Runtime::enableCoroutine(0); diff --git a/app/workers/mails.php b/app/workers/mails.php index 25abe54aa5..9071c00901 100644 --- a/app/workers/mails.php +++ b/app/workers/mails.php @@ -6,7 +6,7 @@ use Utopia\App; use Utopia\CLI\Console; use Utopia\Locale\Locale; -require_once __DIR__ . '/../workers.php'; +require_once __DIR__ . '/../init.php'; Console::title('Mails V1 Worker'); Console::success(APP_NAME . ' mails worker v1 has started' . "\n"); diff --git a/app/workers/webhooks.php b/app/workers/webhooks.php index 00307bdda2..a73dbb6c79 100644 --- a/app/workers/webhooks.php +++ b/app/workers/webhooks.php @@ -4,7 +4,7 @@ use Appwrite\Resque\Worker; use Utopia\App; use Utopia\CLI\Console; -require_once __DIR__.'/../workers.php'; +require_once __DIR__.'/../init.php'; Console::title('Webhooks V1 Worker'); Console::success(APP_NAME.' webhooks worker v1 has started'); From 19c97d750921a5ebe4857b0310bbaf12d22e60b2 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 13 Aug 2021 14:25:01 +0545 Subject: [PATCH 009/206] fix model task related issue --- src/Appwrite/Utopia/Response.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index 3cd25b6032..de328bce75 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -125,6 +125,7 @@ class Response extends SwooleResponse // Deprecated const MODEL_PERMISSIONS = 'permissions'; const MODEL_RULE = 'rule'; + const MODEL_TASK = 'task'; // Tests (keep last) const MODEL_MOCK = 'mock'; From 9499211fdb835d24bf08258b775c8588ef59deb1 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 13 Aug 2021 14:25:10 +0545 Subject: [PATCH 010/206] delete worker.php --- app/workers.php | 47 ----------------------------------------------- 1 file changed, 47 deletions(-) delete mode 100644 app/workers.php diff --git a/app/workers.php b/app/workers.php deleted file mode 100644 index 56dc862fc6..0000000000 --- a/app/workers.php +++ /dev/null @@ -1,47 +0,0 @@ -set('db', function () { - $dbHost = App::getEnv('_APP_DB_HOST', ''); - $dbUser = App::getEnv('_APP_DB_USER', ''); - $dbPass = App::getEnv('_APP_DB_PASS', ''); - $dbScheme = App::getEnv('_APP_DB_SCHEMA', ''); - - $pdo = new PDO("mysql:host={$dbHost};dbname={$dbScheme};charset=utf8mb4", $dbUser, $dbPass, array( - PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8mb4', - PDO::ATTR_TIMEOUT => 3, // Seconds - PDO::ATTR_PERSISTENT => true, - PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, - PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, - )); - - return $pdo; -}); - -$register->set('cache', function () { // Register cache connection - $redis = new Redis(); - $redis->pconnect(App::getEnv('_APP_REDIS_HOST', ''), App::getEnv('_APP_REDIS_PORT', '')); - $redis->setOption(Redis::OPT_READ_TIMEOUT, -1); - - return $redis; -}); - -$register->set('influxdb', function () { // Register DB connection - $host = App::getEnv('_APP_INFLUXDB_HOST', ''); - $port = App::getEnv('_APP_INFLUXDB_PORT', ''); - - if (empty($host) || empty($port)) { - return; - } - $driver = new InfluxDB\Driver\Curl("http://{$host}:{$port}"); - $client = new InfluxDB\Client($host, $port, '', '', false, false, 5); - $client->setDriver($driver); - - return $client; -}); \ No newline at end of file From 00e0e0ae0e69631371f5599b656e294460757182 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 13 Aug 2021 14:27:56 +0545 Subject: [PATCH 011/206] usage daemon to production compose stack --- app/views/install/compose.phtml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index f2a214a8f9..e0a4515de2 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -288,6 +288,26 @@ services: - _APP_MAINTENANCE_RETENTION_ABUSE - _APP_MAINTENANCE_RETENTION_AUDIT + appwrite-usage: + image: /: + entrypoint: usage + container_name: appwrite-usage + restart: unless-stopped + networks: + - appwrite + depends_on: + - influxdb + - mariadb + environment: + - _APP_ENV + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_INFLUXDB_HOST + - _APP_INFLUXDB_PORT + - _APP_USAGE_SYNC_INTERVAL appwrite-schedule: image: /: From 87de85fd1e65ce4b4b7ba2afdceb184abd142333 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 13 Aug 2021 15:30:46 +0545 Subject: [PATCH 012/206] refactor duplicat codes --- app/tasks/usage.php | 117 +++++++++++++------------------------------- 1 file changed, 35 insertions(+), 82 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index ae7de85e82..c6f49ea5cc 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -5,6 +5,7 @@ global $cli, $register; require_once __DIR__ . '/../init.php'; use Utopia\App; +use Utopia\Cache\Adapter\None; use Utopia\Cache\Adapter\Redis; use Utopia\Cache\Cache; use Utopia\CLI\Console; @@ -22,11 +23,28 @@ $cli Console::success(APP_NAME . ' usage sync process v1 has started'); $interval = (int) App::getEnv('_APP_USAGE_SYNC_INTERVAL', '30'); //30 seconds + $attempts = 0; + $max = 10; + $sleep = 1; + do { + try { + $attempts++; + $db = $register->get('db'); + $redis = $register->get('cache'); + break; // leave the do-while if successful + } catch (\Exception$e) { + Console::warning("Database not ready. Retrying connection ({$attempts})..."); + if ($attempts >= $max) { + throw new \Exception('Failed to connect to database: ' . $e->getMessage()); + } + sleep($sleep); + } + } while ($attempts < $max); - $cacheAdapter = new Cache(new Redis($register->get('cache'))); - $dbForConsole = new Database(new MariaDB($register->get('db')), $cacheAdapter); + $cacheAdapter = new Cache(new None()); + $dbForConsole = new Database(new MariaDB($db), $cacheAdapter); $dbForConsole->setNamespace('project_console_internal'); - $dbForProject = new Database(new MariaDB($register->get('db')), $cacheAdapter); + $dbForProject = new Database(new MariaDB($db), $cacheAdapter); Authorization::disable(); $projectIds = []; @@ -36,7 +54,7 @@ $cli $projects = $dbForConsole->find('projects', [], 100, orderAfter:$latestProject); if (!empty($projects)) { $latestProject = $projects[array_key_last($projects)]; - $latestData = getLatestData($projects, $latestData, $dbForProject); + $latestData = getLatestData($projects, $latestData, $dbForProject, $projectIds); } } while (!empty($projects)); @@ -51,7 +69,7 @@ $cli $projects = $dbForConsole->find('projects', limit:100, orderAfter:$latestProject); if (!empty($projects)) { $latestProject = $projects[array_key_last($projects)]; - $latestData = getLatestData($projects, $latestData, $dbForProject); + $latestData = getLatestData($projects, $latestData, $dbForProject, $projectIds); } } @@ -66,7 +84,7 @@ $cli }, $interval); }); -function getLatestData(&$projects, &$latestData, $dbForProject) +function getLatestData(&$projects, &$latestData, $dbForProject, &$projectIds) { foreach ($projects as $project) { $id = $project->getId(); @@ -83,61 +101,27 @@ function getLatestData(&$projects, &$latestData, $dbForProject) function syncData($client, $projectId, $period, &$latestData, $dbForProject) { - $requests = []; - $network = []; - $functions = []; $start = DateTime::createFromFormat('U', \strtotime($period == '1d' ? '-90 days' : '-1 days'))->format(DateTime::RFC3339); if (!empty($latestData[$projectId][$period])) { $start = DateTime::createFromFormat('U', $latestData[$projectId][$period])->format(DateTime::RFC3339); } $end = DateTime::createFromFormat('U', \strtotime('now'))->format(DateTime::RFC3339); $database = $client->selectDB('telegraf'); - - // Requests - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_requests_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $projectId . '\' GROUP BY time(' . $period . ') FILL(null)'); - $points = $result->getPoints(); - $dbForProject->setNamespace("project_{$projectId}_internal"); - foreach ($points as $point) { - $requests[] = [ - 'value' => (!empty($point['value'])) ? $point['value'] : 0, - 'date' => \strtotime($point['time']), - 'dateStr' => $point['time'], - 'point' => $point, - ]; - $time = \strtotime($point['time']); - $id = \md5($time . '_' . $period . '_requests'); - $value = (!empty($point['value'])) ? $point['value'] : 0; - $document = $dbForProject->getDocument('stats', $id); - if ($document->isEmpty()) { - $dbForProject->createDocument('stats', new Document([ - '$id' => $id, - 'period' => $period, - 'time' => $time, - 'metric' => 'requests', - 'value' => $value, - 'type' => 0, - ])); - } else { - $dbForProject->updateDocument('stats', $document->getId(), - $document->setAttribute('value', $value)); - } - $latestData[$id]["30m"] = $time; - } + + syncMetric($database, $projectId, $period, 'requests', $start, $end, $dbForProject); + syncMetric($database, $projectId, $period, 'network', $start, $end, $dbForProject); + syncMetric($database, $projectId, $period, 'executions', $start, $end, $dbForProject); +} - // Network - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_network_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $projectId . '\'GROUP BY time(' . $period . ') FILL(null)'); +function syncMetric($database, $projectId, $period, $metric, $start, $end, $dbForProject) +{ + $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_' . $metric . '_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $projectId . '\'GROUP BY time(' . $period . ') FILL(null)'); $points = $result->getPoints(); foreach ($points as $point) { - $network[] = [ - 'value' => (!empty($point['value'])) ? $point['value'] : 0, - 'date' => \strtotime($point['time']), - 'dateStr' => $point['time'], - 'point' => $point, - ]; $time = \strtotime($point['time']); - $id = \md5($time . '_' . $period . '_network'); + $id = \md5($time . '_' . $period . '_' . $metric); $value = (!empty($point['value'])) ? $point['value'] : 0; $document = $dbForProject->getDocument('stats', $id); if ($document->isEmpty()) { @@ -145,7 +129,7 @@ function syncData($client, $projectId, $period, &$latestData, $dbForProject) '$id' => $id, 'period' => $period, 'time' => $time, - 'metric' => 'network', + 'metric' => $metric, 'value' => $value, 'type' => 0, ])); @@ -153,37 +137,6 @@ function syncData($client, $projectId, $period, &$latestData, $dbForProject) $dbForProject->updateDocument('stats', $document->getId(), $document->setAttribute('value', $value)); } - $latestData[$id]["30m"] = $time; - } - - // Functions - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_executions_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $id . '\'GROUP BY time(' . $period . ') FILL(null)'); - $points = $result->getPoints(); - - foreach ($points as $point) { - $functions[] = [ - 'value' => (!empty($point['value'])) ? $point['value'] : 0, - 'date' => \strtotime($point['time']), - 'dateStr' => $point['time'], - 'point' => $point, - ]; - $time = \strtotime($point['time']); - $id = \md5($time . '_' . $period . '_functions'); - $value = (!empty($point['value'])) ? $point['value'] : 0; - $document = $dbForProject->getDocument('stats', $id); - if ($document->isEmpty()) { - $dbForProject->createDocument('stats', new Document([ - '$id' => $id, - 'period' => $period, - 'time' => $time, - 'metric' => 'functions', - 'value' => $value, - 'type' => 0, - ])); - } else { - $dbForProject->updateDocument('stats', $document->getId(), - $document->setAttribute('value', $value)); - } - $latestData[$id]["30m"] = $time; + $latestData[$id][$period] = $time; } } From 3e3dbb69578a40876050ff4c4d78580aca557d1e Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 13 Aug 2021 15:35:18 +0545 Subject: [PATCH 013/206] refactor --- app/tasks/usage.php | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index c6f49ea5cc..688f58a901 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -76,8 +76,7 @@ $cli $client = $register->get('influxdb'); if ($client) { foreach ($projectIds as $id => $value) { - syncData($client, $id, '30m', $latestData, $dbForProject); - syncData($client, $id, '1d', $latestData, $dbForProject); + syncData($client, $id, $latestData, $dbForProject); } } $firstRun = false; @@ -99,19 +98,22 @@ function getLatestData(&$projects, &$latestData, $dbForProject, &$projectIds) return $latestData; } -function syncData($client, $projectId, $period, &$latestData, $dbForProject) +function syncData($client, $projectId, &$latestData, $dbForProject) { - $start = DateTime::createFromFormat('U', \strtotime($period == '1d' ? '-90 days' : '-1 days'))->format(DateTime::RFC3339); - if (!empty($latestData[$projectId][$period])) { - $start = DateTime::createFromFormat('U', $latestData[$projectId][$period])->format(DateTime::RFC3339); + foreach (['30m', '1d'] as $period) { + $start = DateTime::createFromFormat('U', \strtotime($period == '1d' ? '-90 days' : '-1 days'))->format(DateTime::RFC3339); + if (!empty($latestData[$projectId][$period])) { + $start = DateTime::createFromFormat('U', $latestData[$projectId][$period])->format(DateTime::RFC3339); + } + $end = DateTime::createFromFormat('U', \strtotime('now'))->format(DateTime::RFC3339); + $database = $client->selectDB('telegraf'); + $dbForProject->setNamespace("project_{$projectId}_internal"); + + syncMetric($database, $projectId, $period, 'requests', $start, $end, $dbForProject); + syncMetric($database, $projectId, $period, 'network', $start, $end, $dbForProject); + syncMetric($database, $projectId, $period, 'executions', $start, $end, $dbForProject); } - $end = DateTime::createFromFormat('U', \strtotime('now'))->format(DateTime::RFC3339); - $database = $client->selectDB('telegraf'); - $dbForProject->setNamespace("project_{$projectId}_internal"); - - syncMetric($database, $projectId, $period, 'requests', $start, $end, $dbForProject); - syncMetric($database, $projectId, $period, 'network', $start, $end, $dbForProject); - syncMetric($database, $projectId, $period, 'executions', $start, $end, $dbForProject); + } function syncMetric($database, $projectId, $period, $metric, $start, $end, $dbForProject) From d94b20387095eec601f819ed9de44c0e19da30a4 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 13 Aug 2021 15:45:25 +0545 Subject: [PATCH 014/206] add http path to usage request stats --- app/controllers/shared/api.php | 1 + src/Appwrite/Stats/Stats.php | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index 674a5004b4..1c5e0f1064 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -101,6 +101,7 @@ App::init(function ($utopia, $request, $response, $project, $user, $register, $e ->setParam('httpRequest', 1) ->setParam('httpUrl', $request->getHostname().$request->getURI()) ->setParam('httpMethod', $request->getMethod()) + ->setParam('httpPath', $route->getURL()) ->setParam('networkRequestSize', 0) ->setParam('networkResponseSize', 0) ->setParam('storage', 0) diff --git a/src/Appwrite/Stats/Stats.php b/src/Appwrite/Stats/Stats.php index b07c4c85c5..46c263ac38 100644 --- a/src/Appwrite/Stats/Stats.php +++ b/src/Appwrite/Stats/Stats.php @@ -87,6 +87,7 @@ class Stats $networkResponseSize = $this->params['networkResponseSize'] ?? 0; $httpMethod = $this->params['httpMethod'] ?? ''; + $httpPath = $this->params['httpPath'] ?? ''; $httpRequest = $this->params['httpRequest'] ?? 0; $functionId = $this->params['functionId'] ?? ''; @@ -100,7 +101,7 @@ class Stats $this->statsd->setNamespace($this->namespace); if ($httpRequest >= 1) { - $this->statsd->increment('requests.all' . $tags . ',method=' . \strtolower($httpMethod)); + $this->statsd->increment('requests.all' . $tags . ',method=' . \strtolower($httpMethod).',path='.$httpPath); } if ($functionExecution >= 1) { From a7c4edba9533d2ece4611d694ac1c247846281ad Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Sat, 14 Aug 2021 21:56:28 +0300 Subject: [PATCH 015/206] Updated collection structures to support fulltext search --- app/config/collections2.php | 106 ++++++++++++++++++++++++++---- app/controllers/api/account.php | 36 +++++----- app/controllers/api/database.php | 6 +- app/controllers/api/functions.php | 21 +++++- app/controllers/api/projects.php | 23 ++++--- app/controllers/api/storage.php | 12 +++- app/controllers/api/teams.php | 13 +++- app/controllers/api/users.php | 17 +++-- 8 files changed, 181 insertions(+), 53 deletions(-) diff --git a/app/config/collections2.php b/app/config/collections2.php index ae9a4feeb1..d757752323 100644 --- a/app/config/collections2.php +++ b/app/config/collections2.php @@ -221,13 +221,24 @@ $collections = [ 'array' => true, 'filters' => ['json'], ], + [ + '$id' => 'search', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 16384, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], ], 'indexes' => [ [ - '$id' => '_fulltext_name', + '$id' => '_key_search', 'type' => Database::INDEX_FULLTEXT, - 'attributes' => ['name'], - 'lengths' => [1024], + 'attributes' => ['search'], + 'lengths' => [2048], 'orders' => [Database::ORDER_ASC], ], ], @@ -370,6 +381,17 @@ $collections = [ 'array' => true, 'filters' => ['json'], ], + [ + '$id' => 'search', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 16384, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], ], 'indexes' => [ [ @@ -379,6 +401,13 @@ $collections = [ 'lengths' => [1024], 'orders' => [Database::ORDER_ASC], ], + [ + '$id' => '_key_search', + 'type' => Database::INDEX_FULLTEXT, + 'attributes' => ['search'], + 'lengths' => [2048], + 'orders' => [Database::ORDER_ASC], + ], ], ], @@ -668,13 +697,24 @@ $collections = [ 'array' => false, 'filters' => [], ], + [ + '$id' => 'search', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 16384, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], ], 'indexes' => [ [ - '$id' => '_fulltext_name', + '$id' => '_key_search', 'type' => Database::INDEX_FULLTEXT, - 'attributes' => ['name'], - 'lengths' => [1024], + 'attributes' => ['search'], + 'lengths' => [2048], 'orders' => [Database::ORDER_ASC], ], ], @@ -948,6 +988,17 @@ $collections = [ 'array' => false, 'filters' => [], ], + [ + '$id' => 'search', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 16384, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], ], 'indexes' => [ [ @@ -958,10 +1009,10 @@ $collections = [ 'orders' => [Database::ORDER_ASC], ], [ - '$id' => '_fulltext_name', + '$id' => '_key_search', 'type' => Database::INDEX_FULLTEXT, - 'attributes' => ['name'], - 'lengths' => [1024], + 'attributes' => ['search'], + 'lengths' => [2048], 'orders' => [Database::ORDER_ASC], ], ], @@ -1116,13 +1167,24 @@ $collections = [ 'array' => false, 'filters' => [], ], + [ + '$id' => 'search', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 16384, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], ], 'indexes' => [ [ - '$id' => '_fulltext_name', + '$id' => '_key_search', 'type' => Database::INDEX_FULLTEXT, - 'attributes' => ['name'], - 'lengths' => [1024], + 'attributes' => ['search'], + 'lengths' => [2048], 'orders' => [Database::ORDER_ASC], ], ], @@ -1189,6 +1251,17 @@ $collections = [ 'array' => false, 'filters' => [], ], + [ + '$id' => 'search', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 16384, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], ], 'indexes' => [ [ @@ -1198,6 +1271,13 @@ $collections = [ 'lengths' => [Database::LENGTH_KEY], 'orders' => [Database::ORDER_ASC], ], + [ + '$id' => '_key_search', + 'type' => Database::INDEX_FULLTEXT, + 'attributes' => ['search'], + 'lengths' => [2048], + 'orders' => [Database::ORDER_ASC], + ], ], ], @@ -1405,4 +1485,4 @@ $collections = [ ], ]; -return $collections; +return $collections; \ No newline at end of file diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index ef08805117..11649966d6 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -103,6 +103,7 @@ App::post('/v1/account') 'sessions' => [], 'tokens' => [], 'memberships' => [], + 'search' => implode(' ', [$userId, $email, $name]), ])); } catch (Duplicate $th) { throw new Exception('Account already exists', 409); @@ -195,8 +196,8 @@ App::post('/v1/account/sessions') Authorization::setRole('user:' . $profile->getId()); $session = $dbForInternal->createDocument('sessions', $session - ->setAttribute('$read', ['user:' . $profile->getId()]) - ->setAttribute('$write', ['user:' . $profile->getId()]) + ->setAttribute('$read', ['user:' . $profile->getId()]) + ->setAttribute('$write', ['user:' . $profile->getId()]) ); $profile->setAttribute('sessions', $session, Document::SET_TYPE_APPEND); @@ -481,6 +482,7 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') 'sessions' => [], 'tokens' => [], 'memberships' => [], + 'search' => implode(' ', [$userId, $email, $name]), ])); } catch (Duplicate $th) { throw new Exception('Account already exists', 409); @@ -530,8 +532,8 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') Authorization::setRole('user:' . $user->getId()); $session = $dbForInternal->createDocument('sessions', $session - ->setAttribute('$read', ['user:' . $user->getId()]) - ->setAttribute('$write', ['user:' . $user->getId()]) + ->setAttribute('$read', ['user:' . $user->getId()]) + ->setAttribute('$write', ['user:' . $user->getId()]) ); $user = $dbForInternal->updateDocument('users', $user->getId(), $user); @@ -644,6 +646,7 @@ App::post('/v1/account/sessions/anonymous') 'sessions' => [], 'tokens' => [], 'memberships' => [], + 'search' => $userId, ])); Authorization::reset(); @@ -978,7 +981,10 @@ App::patch('/v1/account/name') /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $audits */ - $user = $dbForInternal->updateDocument('users', $user->getId(), $user->setAttribute('name', $name)); + $user = $dbForInternal->updateDocument('users', $user->getId(), $user + ->setAttribute('name', $name) + ->setAttribute('search', implode(' ', [$user->getId(), $name, $user->getAttribute('email')])) + ); $audits ->setParam('userId', $user->getId()) @@ -1073,9 +1079,10 @@ App::patch('/v1/account/email') } $user = $dbForInternal->updateDocument('users', $user->getId(), $user - ->setAttribute('password', $isAnonymousUser ? Auth::passwordHash($password) : $user->getAttribute('password', '')) - ->setAttribute('email', $email) - ->setAttribute('emailVerification', false) // After this user needs to confirm mail again + ->setAttribute('password', $isAnonymousUser ? Auth::passwordHash($password) : $user->getAttribute('password', '')) + ->setAttribute('email', $email) + ->setAttribute('emailVerification', false) // After this user needs to confirm mail again + ->setAttribute('search', implode(' ', [$user->getId(), $user->getAttribute('name'), $user->getAttribute('email')])) ); $audits @@ -1489,10 +1496,9 @@ App::put('/v1/account/recovery') ); /** - * We act like we're updating and validating - * the recovery token but actually we don't need it anymore. - */ - + * We act like we're updating and validating + * the recovery token but actually we don't need it anymore. + */ foreach ($tokens as $key => $token) { if ($recovery === $token->getId()) { $recovery = $token; @@ -1650,9 +1656,9 @@ App::put('/v1/account/verification') $profile = $dbForInternal->updateDocument('users', $profile->getId(), $profile->setAttribute('emailVerification', true)); /** - * We act like we're updating and validating - * the verification token but actually we don't need it anymore. - */ + * We act like we're updating and validating + * the verification token but actually we don't need it anymore. + */ foreach ($tokens as $key => $token) { if ($token->getId() === $verification) { $verification = $token; diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 1ebb857da3..05f74c94f3 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -189,7 +189,11 @@ App::get('/v1/database/collections') /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal */ - $queries = ($search) ? [new Query('name', Query::TYPE_SEARCH, [$search])] : []; + $queries = []; + + if (!empty($search)) { + $queries[] = new Query('name', Query::TYPE_SEARCH, [$search]); + } $response->dynamic(new Document([ 'collections' => $dbForExternal->find(Database::COLLECTIONS, $queries, $limit, $offset, ['_id'], [$orderType]), diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index a0bcb535be..9994c9eaa9 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -53,8 +53,9 @@ App::post('/v1/functions') /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + $functionId = ($functionId == 'unique()') ? $dbForInternal->getId() : $functionId; $function = $dbForInternal->createDocument('functions', new Document([ - '$id' => $functionId == 'unique()' ? $dbForInternal->getId() : $functionId, + '$id' => $functionId, 'execute' => $execute, 'dateCreated' => time(), 'dateUpdated' => time(), @@ -68,6 +69,7 @@ App::post('/v1/functions') 'schedulePrevious' => 0, 'scheduleNext' => 0, 'timeout' => $timeout, + 'search' => implode(' ', [$functionId, $name, $runtime]), ])); $response->setStatusCode(Response::STATUS_CODE_CREATED); @@ -95,7 +97,11 @@ App::get('/v1/functions') /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ - $queries = ($search) ? [new Query('name', Query::TYPE_SEARCH, [$search])] : []; + $queries = []; + + if (!empty($search)) { + $queries[] = new Query('search', Query::TYPE_SEARCH, [$search]); + } $response->dynamic(new Document([ 'functions' => $dbForInternal->find('functions', $queries, $limit, $offset, ['_id'], [$orderType]), @@ -296,6 +302,7 @@ App::put('/v1/functions/:functionId') 'schedule' => $schedule, 'scheduleNext' => (int)$next, 'timeout' => $timeout, + 'search' => implode(' ', [$functionId, $name, $function->getAttribute('runtime')]), ]))); if ($next && $schedule !== $original) { @@ -472,8 +479,9 @@ App::post('/v1/functions/:functionId/tags') throw new Exception('Failed moving file', 500); } + $tagId = $dbForInternal->getId(); $tag = $dbForInternal->createDocument('tags', new Document([ - '$id' => $dbForInternal->getId(), + '$id' => $tagId, '$read' => [], '$write' => [], 'functionId' => $function->getId(), @@ -481,6 +489,7 @@ App::post('/v1/functions/:functionId/tags') 'command' => $command, 'path' => $path, 'size' => $size, + 'search' => implode(' ', [$tagId, $command]), ])); $usage @@ -519,6 +528,12 @@ App::get('/v1/functions/:functionId/tags') throw new Exception('Function not found', 404); } + $queries = []; + + if (!empty($search)) { + $queries[] = new Query('search', Query::TYPE_SEARCH, [$search]); + } + $queries[] = new Query('functionId', Query::TYPE_EQUAL, [$function->getId()]); $results = $dbForInternal->find('tags', $queries, $limit, $offset, ['_id'], [$orderType]); diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 493eb65dcb..dfce1525de 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -5,9 +5,7 @@ use Appwrite\Database\Validator\CustomId; use Appwrite\Network\Validator\CNAME; use Appwrite\Network\Validator\Domain as DomainValidator; use Appwrite\Network\Validator\URL; -use Appwrite\Task\Validator\Cron; use Appwrite\Utopia\Response; -use Cron\CronExpression; use Utopia\App; use Utopia\Config\Config; use Utopia\Database\Document; @@ -78,8 +76,9 @@ App::post('/v1/projects') $auths[$method['key'] ?? ''] = true; } + $projectId = ($projectId == 'unique()') ? $dbForConsole->getId() : $projectId; $project = $dbForConsole->createDocument('projects', new Document([ - '$id' => $projectId == 'unique()' ? $dbForConsole->getId() : $projectId, + '$id' => $projectId, '$collection' => 'projects', '$read' => ['team:' . $teamId], '$write' => ['team:' . $teamId . '/owner', 'team:' . $teamId . '/developer'], @@ -95,12 +94,13 @@ App::post('/v1/projects') 'legalCity' => $legalCity, 'legalAddress' => $legalAddress, 'legalTaxId' => $legalTaxId, + 'auths' => $auths, 'services' => new stdClass(), 'platforms' => [], 'webhooks' => [], 'keys' => [], 'domains' => [], - 'auths' => $auths, + 'search' => implode(' ', [$projectId, $name]), ])); $collections = Config::getParam('collections2', []); /** @var array $collections */ @@ -171,7 +171,11 @@ App::get('/v1/projects') /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForConsole */ - $queries = ($search) ? [new Query('name', Query::TYPE_SEARCH, [$search])] : []; + $queries = []; + + if (!empty($search)) { + $queries[] = new Query('search', Query::TYPE_SEARCH, [$search]); + } $results = $dbForConsole->find('projects', $queries, $limit, $offset, ['_id'], [$orderType]); $sum = $dbForConsole->count('projects', $queries, APP_LIMIT_COUNT); @@ -445,6 +449,7 @@ App::patch('/v1/projects/:projectId') ->setAttribute('legalCity', $legalCity) ->setAttribute('legalAddress', $legalAddress) ->setAttribute('legalTaxId', $legalTaxId) + ->setAttribute('search', implode(' ', [$projectId, $name])) ); $response->dynamic($project, Response::MODEL_PROJECT); @@ -546,7 +551,7 @@ App::patch('/v1/projects/:projectId/auth/limit') $auths['limit'] = $limit; $dbForConsole->updateDocument('projects', $project->getId(), $project - ->setAttribute('auths', $auths) + ->setAttribute('auths', $auths) ); $response->dynamic($project, Response::MODEL_PROJECT); @@ -869,7 +874,7 @@ App::post('/v1/projects/:projectId/keys') ]); $project = $dbForConsole->updateDocument('projects', $project->getId(), $project - ->setAttribute('keys', $key, Document::SET_TYPE_APPEND) + ->setAttribute('keys', $key, Document::SET_TYPE_APPEND) ); $response->setStatusCode(Response::STATUS_CODE_CREATED); @@ -1053,7 +1058,7 @@ App::post('/v1/projects/:projectId/platforms') ]); $project = $dbForConsole->updateDocument('projects', $project->getId(), $project - ->setAttribute('platforms', $platform, Document::SET_TYPE_APPEND) + ->setAttribute('platforms', $platform, Document::SET_TYPE_APPEND) ); $response->setStatusCode(Response::STATUS_CODE_CREATED); @@ -1262,7 +1267,7 @@ App::post('/v1/projects/:projectId/domains') ]); $project = $dbForConsole->updateDocument('projects', $project->getId(), $project - ->setAttribute('domains', $domain, Document::SET_TYPE_APPEND) + ->setAttribute('domains', $domain, Document::SET_TYPE_APPEND) ); $response->setStatusCode(Response::STATUS_CODE_CREATED); diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 4f579a1b31..50e1f4d432 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -123,13 +123,14 @@ App::post('/v1/storage/files') $sizeActual = $device->getFileSize($path); + $fileId = ($fileId == 'unique()') ? $dbForInternal->getId() : $fileId; $file = $dbForInternal->createDocument('files', new Document([ - '$id' => $fileId == 'unique()' ? $dbForInternal->getId() : $fileId, + '$id' => $fileId, '$read' => (is_null($read) && !$user->isEmpty()) ? ['user:'.$user->getId()] : $read ?? [], // By default set read permissions for user '$write' => (is_null($write) && !$user->isEmpty()) ? ['user:'.$user->getId()] : $write ?? [], // By default set write permissions for user 'dateCreated' => \time(), 'bucketId' => '', - 'name' => $file['name'], + 'name' => $file['name'] ?? '', 'path' => $path, 'signature' => $device->getFileHash($path), 'mimeType' => $mimeType, @@ -141,6 +142,7 @@ App::post('/v1/storage/files') 'openSSLCipher' => OpenSSL::CIPHER_AES_128_GCM, 'openSSLTag' => \bin2hex($tag), 'openSSLIV' => \bin2hex($iv), + 'search' => implode(' ', [$fileId, $file['name'] ?? '',]), ])); $audits @@ -178,7 +180,11 @@ App::get('/v1/storage/files') /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ - $queries = ($search) ? [new Query('name', Query::TYPE_SEARCH, $search)] : []; + $queries = []; + + if (!empty($search)) { + $queries[] = new Query('search', Query::TYPE_SEARCH, [$search]); + } $response->dynamic(new Document([ 'files' => $dbForInternal->find('files', $queries, $limit, $offset, ['_id'], [$orderType]), diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index 01ce3c82fd..5e9deedc6c 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -57,6 +57,7 @@ App::post('/v1/teams') 'name' => $name, 'sum' => ($isPrivilegedUser || $isAppUser) ? 0 : 1, 'dateCreated' => \time(), + 'search' => implode(' ', [$teamId, $name]), ])); Authorization::reset(); @@ -106,7 +107,11 @@ App::get('/v1/teams') /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ - $queries = ($search) ? [new Query('name', Query::TYPE_SEARCH, [$search])] : []; + $queries = []; + + if (!empty($search)) { + $queries[] = new Query('search', Query::TYPE_SEARCH, [$search]); + } $results = $dbForInternal->find('teams', $queries, $limit, $offset, ['_id'], [$orderType]); $sum = $dbForInternal->count('teams', $queries, APP_LIMIT_COUNT); @@ -170,7 +175,10 @@ App::put('/v1/teams/:teamId') throw new Exception('Team not found', 404); } - $team = $dbForInternal->updateDocument('teams', $team->getId(), $team->setAttribute('name', $name)); + $team = $dbForInternal->updateDocument('teams', $team->getId(),$team + ->setAttribute('name', $name) + ->setAttribute('search', implode(' ', [$teamId, $name])) + ); $response->dynamic($team, Response::MODEL_TEAM); }); @@ -314,6 +322,7 @@ App::post('/v1/teams/:teamId/memberships') 'sessions' => [], 'tokens' => [], 'memberships' => [], + 'search' => implode(' ', [$userId, $email, $name]), ])); } catch (Duplicate $th) { throw new Exception('Account already exists', 409); diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index d57be233ef..c2c7a9bb3f 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -17,6 +17,7 @@ use Utopia\Database\Exception\Duplicate; use Utopia\Database\Validator\UID; use DeviceDetector\DeviceDetector; use Appwrite\Database\Validator\CustomId; +use Utopia\Database\Query; App::post('/v1/users') ->desc('Create User') @@ -60,6 +61,7 @@ App::post('/v1/users') 'sessions' => [], 'tokens' => [], 'memberships' => [], + 'search' => implode(' ', [$userId, $email, $name]), ])); } catch (Duplicate $th) { throw new Exception('Account already exists', 409); @@ -90,8 +92,14 @@ App::get('/v1/users') /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ - $results = $dbForInternal->find('users', [], $limit, $offset, ['_id'], [$orderType]); - $sum = $dbForInternal->count('users', [], APP_LIMIT_COUNT); + $queries = []; + + if (!empty($search)) { + $queries[] = new Query('search', Query::TYPE_SEARCH, [$search]); + } + + $results = $dbForInternal->find('users', $queries, $limit, $offset, ['_id'], [$orderType]); + $sum = $dbForInternal->count('users', $queries, APP_LIMIT_COUNT); $response->dynamic(new Document([ 'users' => $results, @@ -519,11 +527,6 @@ App::delete('/v1/users/:userId') if (!$dbForInternal->deleteDocument('users', $userId)) { throw new Exception('Failed to remove user from DB', 500); } - - // $dbForInternal->createDocument('users', new Document([ - // '$id' => $userId, - // '$read' => ['role:all'], - // ])); $deletes ->setParam('type', DELETE_TYPE_DOCUMENT) From ca64195eb1dc9e9e4864e6e0f78e996882a66559 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 15 Aug 2021 14:15:51 +0545 Subject: [PATCH 016/206] fix special chars in path in statsd --- src/Appwrite/Stats/Stats.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Stats/Stats.php b/src/Appwrite/Stats/Stats.php index 46c263ac38..9625d1c201 100644 --- a/src/Appwrite/Stats/Stats.php +++ b/src/Appwrite/Stats/Stats.php @@ -101,7 +101,7 @@ class Stats $this->statsd->setNamespace($this->namespace); if ($httpRequest >= 1) { - $this->statsd->increment('requests.all' . $tags . ',method=' . \strtolower($httpMethod).',path='.$httpPath); + $this->statsd->increment('requests.all' . $tags . ',method=' . \strtolower($httpMethod) . ',path=' . str_replace(':', '*', $httpPath)); } if ($functionExecution >= 1) { From 4686e15a74dbe00655b614910c2529a18e86dc8b Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 15 Aug 2021 14:17:01 +0545 Subject: [PATCH 017/206] updated to get database stats --- app/tasks/usage.php | 129 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 116 insertions(+), 13 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 688f58a901..bc838b8b3b 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -89,10 +89,12 @@ function getLatestData(&$projects, &$latestData, $dbForProject, &$projectIds) $id = $project->getId(); $projectIds[$id] = true; $dbForProject->setNamespace("project_{$id}_internal"); - $doc = $dbForProject->findOne('stats', [new Query("period", Query::TYPE_EQUAL, ["1d"])], 0, ['time'], [Database::ORDER_DESC]); - $latestData[$id]["1d"] = $doc ? $doc->getAttribute('time') : null; - $doc = $dbForProject->findOne('stats', [new Query("period", Query::TYPE_EQUAL, ["30m"])], 0, ['time'], [Database::ORDER_DESC]); - $latestData[$id]["30m"] = $doc ? $doc->getAttribute('time') : null; + foreach (['requests', 'network', 'executions', 'database.reads', 'database.creates', 'database.updates', 'database.deletes'] as $metric) { + $doc = $dbForProject->findOne('stats', [new Query("period", Query::TYPE_EQUAL, ["1d"]), new Query("metric", Query::TYPE_EQUAL, [$metric])], 0, ['time'], [Database::ORDER_DESC]); + $latestData[$id][$metric]["1d"] = $doc ? $doc->getAttribute('time') : null; + $doc = $dbForProject->findOne('stats', [new Query("period", Query::TYPE_EQUAL, ["30m"])], 0, ['time'], [Database::ORDER_DESC]); + $latestData[$id][$metric]["30m"] = $doc ? $doc->getAttribute('time') : null; + } } $projects = null; return $latestData; @@ -101,21 +103,122 @@ function getLatestData(&$projects, &$latestData, $dbForProject, &$projectIds) function syncData($client, $projectId, &$latestData, $dbForProject) { foreach (['30m', '1d'] as $period) { - $start = DateTime::createFromFormat('U', \strtotime($period == '1d' ? '-90 days' : '-1 days'))->format(DateTime::RFC3339); - if (!empty($latestData[$projectId][$period])) { - $start = DateTime::createFromFormat('U', $latestData[$projectId][$period])->format(DateTime::RFC3339); - } + $start = DateTime::createFromFormat('U', \strtotime($period == '1d' ? '-90 days' : '-24 hours'))->format(DateTime::RFC3339); $end = DateTime::createFromFormat('U', \strtotime('now'))->format(DateTime::RFC3339); $database = $client->selectDB('telegraf'); $dbForProject->setNamespace("project_{$projectId}_internal"); - - syncMetric($database, $projectId, $period, 'requests', $start, $end, $dbForProject); - syncMetric($database, $projectId, $period, 'network', $start, $end, $dbForProject); - syncMetric($database, $projectId, $period, 'executions', $start, $end, $dbForProject); + foreach (['requests', 'network', 'executions'] as $metric) { + if (!empty($latestData[$projectId][$metric][$period])) { + $start = DateTime::createFromFormat('U', $latestData[$projectId][$metric][$period])->format(DateTime::RFC3339); + } + syncMetric($database, $projectId, $period, 'requests', $start, $end, $dbForProject); + } + // syncMetric($database, $projectId, $period, 'network', $start, $end, $dbForProject); + // syncMetric($database, $projectId, $period, 'executions', $start, $end, $dbForProject); + syncMetricPaths($database, $projectId, $period, $start, $end, $dbForProject); } } +function syncMetricPaths($database, $projectId, $period, $start, $end, $dbForProject) +{ + $start = DateTime::createFromFormat('U', \strtotime($period == '1d' ? '-7 days' : '-24 hours'))->format(DateTime::RFC3339); + if (!empty($latestData[$projectId]['database'][$period])) { + $start = DateTime::createFromFormat('U', $latestData[$projectId]['database'][$period])->format(DateTime::RFC3339); + } + $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_requests_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $projectId . '\'GROUP BY time(' . $period . '), "path", "method" FILL(null)'); + $points = $result->getPoints(); + + $databaseMetrics = [ + 'creates' => [ + 'method' => 'post', + 'paths' => [ + '/v1/database/collections' => true, + '/v1/database/collections/*collectionId/attributes/string' => true, + '/v1/database/collections/*collectionId/attributes/email' => true, + '/v1/database/collections/*collectionId/attributes/ip' => true, + '/v1/database/collections/*collectionId/attributes/url' => true, + '/v1/database/collections/*collectionId/attributes/integer' => true, + '/v1/database/collections/*collectionId/attributes/float' => true, + '/v1/database/collections/*collectionId/attributes/boolean' => true, + '/v1/database/collections/*collectionId/indexes' => true, + '/v1/database/collections/*collectionId/documents' => true, + ], + ], + 'reads' => [ + 'method' => 'get', + 'paths' => [ + '/v1/database/collections' => true, + '/v1/database/collections/*collectionId' => true, + '/v1/database/collections/*collectionId/attributes' => true, + '/v1/database/collections/*collectionId/attributes/*attributeId' => true, + '/v1/database/collections/*collectionId/indexes' => true, + '/v1/database/collections/*collectionId/indexes/*indexId' => true, + '/v1/database/collections/*collectionId/documents' => true, + '/v1/database/collections/*collectionId/documents/*documentId' => true, + ], + ], + 'updates' => [ + 'method' => 'put', + 'paths' => [ + '/v1/database/collections/*collectionId' => true, + '/v1/database/collections/*collectionId/documents/*documentId' => true, + ], + ], + 'deletes' => [ + 'method' => 'delete', + 'paths' => [ + '/v1/database/collections/*collectionId' => true, + '/v1/database/collections/*collectionId/attributes/*attributeId' => true, + '/v1/database/collections/*collectionId/indexes/*indexId' => true, + '/v1/database/collections/*collectionId/documents/*documentId' => true, + ], + ], + ]; + + $dbStats = []; + foreach ($points as $point) { + $time = \strtotime($point['time']); + $value = (!empty($point['value'])) ? $point['value'] : 0; + $path = $point['path'] ?? ''; + $method = $point['method'] ?? ''; + + foreach (['creates', 'reads', 'updates', 'deletes'] as $operation) { + if ($method == $databaseMetrics[$operation]['method'] + && array_key_exists($path, $databaseMetrics[$operation]['paths'])) { + if (empty($dbStats["database.{$operation}"][$period][$time])) { + $dbStats["database.{$operation}"][$period][$time] = 0; + } + $dbStats["database.{$operation}"][$period][$time] += $value; + } + } + } + + $time = \strtotime($start); + foreach ($dbStats as $metric => $stats) { + foreach ($stats as $period => $times) { + foreach ($times as $time => $value) { + $id = \md5($time . '_' . $period . '_' . $metric); + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'period' => $period, + 'time' => $time, + 'metric' => $metric, + 'value' => $value, + 'type' => 0, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $value)); + } + } + } + } + $latestData[$projectId]['database'][$period] = $time; +} + function syncMetric($database, $projectId, $period, $metric, $start, $end, $dbForProject) { $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_' . $metric . '_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $projectId . '\'GROUP BY time(' . $period . ') FILL(null)'); @@ -139,6 +242,6 @@ function syncMetric($database, $projectId, $period, $metric, $start, $end, $dbFo $dbForProject->updateDocument('stats', $document->getId(), $document->setAttribute('value', $value)); } - $latestData[$id][$period] = $time; + $latestData[$projectId][$metric][$period] = $time; } } From 21774decdd0f1f4b698ea5464d0a0e420cee85a5 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 15 Aug 2021 14:23:31 +0545 Subject: [PATCH 018/206] some refactor --- app/tasks/usage.php | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index bc838b8b3b..f227519080 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -5,7 +5,6 @@ global $cli, $register; require_once __DIR__ . '/../init.php'; use Utopia\App; -use Utopia\Cache\Adapter\None; use Utopia\Cache\Adapter\Redis; use Utopia\Cache\Cache; use Utopia\CLI\Console; @@ -41,7 +40,7 @@ $cli } } while ($attempts < $max); - $cacheAdapter = new Cache(new None()); + $cacheAdapter = new Cache(new Redis($redis)); $dbForConsole = new Database(new MariaDB($db), $cacheAdapter); $dbForConsole->setNamespace('project_console_internal'); $dbForProject = new Database(new MariaDB($db), $cacheAdapter); @@ -89,7 +88,7 @@ function getLatestData(&$projects, &$latestData, $dbForProject, &$projectIds) $id = $project->getId(); $projectIds[$id] = true; $dbForProject->setNamespace("project_{$id}_internal"); - foreach (['requests', 'network', 'executions', 'database.reads', 'database.creates', 'database.updates', 'database.deletes'] as $metric) { + foreach (['requests', 'network', 'executions', 'database.reads'] as $metric) { $doc = $dbForProject->findOne('stats', [new Query("period", Query::TYPE_EQUAL, ["1d"]), new Query("metric", Query::TYPE_EQUAL, [$metric])], 0, ['time'], [Database::ORDER_DESC]); $latestData[$id][$metric]["1d"] = $doc ? $doc->getAttribute('time') : null; $doc = $dbForProject->findOne('stats', [new Query("period", Query::TYPE_EQUAL, ["30m"])], 0, ['time'], [Database::ORDER_DESC]); @@ -113,8 +112,6 @@ function syncData($client, $projectId, &$latestData, $dbForProject) } syncMetric($database, $projectId, $period, 'requests', $start, $end, $dbForProject); } - // syncMetric($database, $projectId, $period, 'network', $start, $end, $dbForProject); - // syncMetric($database, $projectId, $period, 'executions', $start, $end, $dbForProject); syncMetricPaths($database, $projectId, $period, $start, $end, $dbForProject); } @@ -124,7 +121,7 @@ function syncMetricPaths($database, $projectId, $period, $start, $end, $dbForPro { $start = DateTime::createFromFormat('U', \strtotime($period == '1d' ? '-7 days' : '-24 hours'))->format(DateTime::RFC3339); if (!empty($latestData[$projectId]['database'][$period])) { - $start = DateTime::createFromFormat('U', $latestData[$projectId]['database'][$period])->format(DateTime::RFC3339); + $start = DateTime::createFromFormat('U', $latestData[$projectId]['database.reads'][$period])->format(DateTime::RFC3339); } $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_requests_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $projectId . '\'GROUP BY time(' . $period . '), "path", "method" FILL(null)'); $points = $result->getPoints(); @@ -216,7 +213,7 @@ function syncMetricPaths($database, $projectId, $period, $start, $end, $dbForPro } } } - $latestData[$projectId]['database'][$period] = $time; + $latestData[$projectId]['database.reads'][$period] = $time; } function syncMetric($database, $projectId, $period, $metric, $start, $end, $dbForProject) From 0c2cfe2efc57d7a4e19ebdb7205926f4e01b6fb3 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 15 Aug 2021 14:25:55 +0545 Subject: [PATCH 019/206] Update app/views/install/compose.phtml Co-authored-by: Eldad A. Fux --- app/views/install/compose.phtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml index e0a4515de2..c016c8ec81 100644 --- a/app/views/install/compose.phtml +++ b/app/views/install/compose.phtml @@ -307,7 +307,7 @@ services: - _APP_DB_PASS - _APP_INFLUXDB_HOST - _APP_INFLUXDB_PORT - - _APP_USAGE_SYNC_INTERVAL + - _APP_USAGE_AGGREGATION_INTERVAL appwrite-schedule: image: /: From d9980cbf89add3dc1c7c864659e539b949fc52b9 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 15 Aug 2021 14:26:19 +0545 Subject: [PATCH 020/206] fix env var --- app/tasks/usage.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index f227519080..d1a2905e02 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -21,7 +21,7 @@ $cli Console::title('Usage Sync V1'); Console::success(APP_NAME . ' usage sync process v1 has started'); - $interval = (int) App::getEnv('_APP_USAGE_SYNC_INTERVAL', '30'); //30 seconds + $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); //30 seconds $attempts = 0; $max = 10; $sleep = 1; From 94b77953325910310c265c2e1a9a9c2620730950 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 15 Aug 2021 14:30:14 +0545 Subject: [PATCH 021/206] usage aggregation environment variable description --- app/config/variables.php | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/app/config/variables.php b/app/config/variables.php index df7346cd59..132e279c2c 100644 --- a/app/config/variables.php +++ b/app/config/variables.php @@ -149,6 +149,15 @@ return [ 'required' => false, 'question' => '', 'filter' => '' + ], + [ + 'name' => '_APP_USAGE_AGGREGATION_INTERVAL', + 'description' => 'Interval value containing the number of seconds that the Appwrite usage process should wait before aggregating stats and syncing it to mariadb from InfluxDB. The default value is 30 seconds.', + 'introduction' => '0.10.0', + 'default' => '30', + 'required' => false, + 'question' => '', + 'filter' => '' ] ], ], From 251f14cd697dae47dacf964942a9c537ee4d5360 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 15 Aug 2021 16:53:32 +0545 Subject: [PATCH 022/206] usage params in database endpoints --- app/controllers/api/database.php | 165 ++++++++++++++++++++++++------- 1 file changed, 132 insertions(+), 33 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index c923fd85b0..6c2e2525bd 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -25,7 +25,7 @@ use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Query; -$attributesCallback = function ($attribute, $response, $dbForExternal, $database, $audits) { +$attributesCallback = function ($attribute, $response, $dbForExternal, $database, $audits, $usage) { /** @var Utopia\Database\Document $document*/ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal*/ @@ -110,6 +110,8 @@ $attributesCallback = function ($attribute, $response, $dbForExternal, $database ->setParam('document', $attribute) ; + $usage->setParam('database.collections.update', 1); + $audits ->setParam('event', 'database.attributes.create') ->setParam('resource', 'database/attributes/'.$attribute->getId()) @@ -139,10 +141,12 @@ App::post('/v1/database/collections') ->inject('response') ->inject('dbForExternal') ->inject('audits') - ->action(function ($collectionId, $name, $read, $write, $response, $dbForExternal, $audits) { + ->inject('usage') + ->action(function ($collectionId, $name, $read, $write, $response, $dbForExternal, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal*/ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $collectionId = $collectionId == 'unique()' ? $dbForExternal->getId() : $collectionId; @@ -164,6 +168,8 @@ App::post('/v1/database/collections') ->setParam('data', $collection->getArrayCopy()) ; + $usage->setParam('database.collections.create', 1); + $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($collection, Response::MODEL_COLLECTION); }); @@ -186,9 +192,11 @@ App::get('/v1/database/collections') ->param('orderType', 'ASC', new WhiteList(['ASC', 'DESC'], true), 'Order result by ASC or DESC order.', true) ->inject('response') ->inject('dbForExternal') - ->action(function ($search, $limit, $offset, $after, $orderType, $response, $dbForExternal) { + ->inject('usage') + ->action(function ($search, $limit, $offset, $after, $orderType, $response, $dbForExternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal */ + /** @var Appwrite\Stats\Stats $usage */ $queries = ($search) ? [new Query('name', Query::TYPE_SEARCH, [$search])] : []; @@ -200,6 +208,8 @@ App::get('/v1/database/collections') } } + $usage->setParam('database.collections.read', 1); + $response->dynamic(new Document([ 'collections' => $dbForExternal->find(Database::COLLECTIONS, $queries, $limit, $offset, [], [$orderType], $afterCollection ?? null), 'sum' => $dbForExternal->count(Database::COLLECTIONS, $queries, APP_LIMIT_COUNT), @@ -220,9 +230,11 @@ App::get('/v1/database/collections/:collectionId') ->param('collectionId', '', new UID(), 'Collection unique ID.') ->inject('response') ->inject('dbForExternal') - ->action(function ($collectionId, $response, $dbForExternal) { + ->inject('usage') + ->action(function ($collectionId, $response, $dbForExternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal */ + /** @var Appwrite\Stats\Stats $usage */ $collection = $dbForExternal->getCollection($collectionId); @@ -230,6 +242,8 @@ App::get('/v1/database/collections/:collectionId') throw new Exception('Collection not found', 404); } + $usage->setParam('database.collections.read', 1); + $response->dynamic($collection, Response::MODEL_COLLECTION); }); @@ -252,10 +266,12 @@ App::put('/v1/database/collections/:collectionId') ->inject('response') ->inject('dbForExternal') ->inject('audits') - ->action(function ($collectionId, $name, $read, $write, $response, $dbForExternal, $audits) { + ->inject('usage') + ->action(function ($collectionId, $name, $read, $write, $response, $dbForExternal, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $collection = $dbForExternal->getCollection($collectionId); @@ -278,6 +294,8 @@ App::put('/v1/database/collections/:collectionId') throw new Exception('Bad structure. '.$exception->getMessage(), 400); } + $usage->setParam('database.collections.update', 1); + $audits ->setParam('event', 'database.collections.update') ->setParam('resource', 'database/collections/'.$collection->getId()) @@ -304,11 +322,13 @@ App::delete('/v1/database/collections/:collectionId') ->inject('events') ->inject('audits') ->inject('deletes') - ->action(function ($collectionId, $response, $dbForExternal, $events, $audits, $deletes) { + ->inject('usage') + ->action(function ($collectionId, $response, $dbForExternal, $events, $audits, $deletes, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal */ /** @var Appwrite\Event\Event $events */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $audits */ $collection = $dbForExternal->getCollection($collectionId); @@ -318,6 +338,8 @@ App::delete('/v1/database/collections/:collectionId') $dbForExternal->deleteCollection($collectionId); + $usage->setParam('database.collections.delete', 1); + $events ->setParam('eventData', $response->output($collection, Response::MODEL_COLLECTION)) ; @@ -353,11 +375,13 @@ App::post('/v1/database/collections/:collectionId/attributes/string') ->inject('dbForExternal') ->inject('database') ->inject('audits') - ->action(function ($collectionId, $attributeId, $size, $required, $default, $array, $response, $dbForExternal, $database, $audits) use ($attributesCallback) { + ->inject('usage') + ->action(function ($collectionId, $attributeId, $size, $required, $default, $array, $response, $dbForExternal, $database, $audits, $usage) use ($attributesCallback) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal*/ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ return $attributesCallback(new Document([ '$collection' => $collectionId, @@ -367,7 +391,7 @@ App::post('/v1/database/collections/:collectionId/attributes/string') 'required' => $required, 'default' => $default, 'array' => $array, - ]), $response, $dbForExternal, $database, $audits); + ]), $response, $dbForExternal, $database, $audits, $usage); }); App::post('/v1/database/collections/:collectionId/attributes/email') @@ -391,11 +415,13 @@ App::post('/v1/database/collections/:collectionId/attributes/email') ->inject('dbForExternal') ->inject('database') ->inject('audits') - ->action(function ($collectionId, $attributeId, $required, $default, $array, $response, $dbForExternal, $database, $audits) use ($attributesCallback) { + ->inject('usage') + ->action(function ($collectionId, $attributeId, $required, $default, $array, $response, $dbForExternal, $database, $audits, $usage) use ($attributesCallback) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal*/ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ return $attributesCallback(new Document([ '$collection' => $collectionId, @@ -406,7 +432,7 @@ App::post('/v1/database/collections/:collectionId/attributes/email') 'default' => $default, 'array' => $array, 'format' => \json_encode(['name'=>'email']), - ]), $response, $dbForExternal, $database, $audits); + ]), $response, $dbForExternal, $database, $audits, $usage); }); App::post('/v1/database/collections/:collectionId/attributes/ip') @@ -430,11 +456,13 @@ App::post('/v1/database/collections/:collectionId/attributes/ip') ->inject('dbForExternal') ->inject('database') ->inject('audits') - ->action(function ($collectionId, $attributeId, $required, $default, $array, $response, $dbForExternal, $database, $audits) use ($attributesCallback) { + ->inject('usage') + ->action(function ($collectionId, $attributeId, $required, $default, $array, $response, $dbForExternal, $database, $audits, $usage) use ($attributesCallback) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal*/ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ return $attributesCallback(new Document([ '$collection' => $collectionId, @@ -445,7 +473,7 @@ App::post('/v1/database/collections/:collectionId/attributes/ip') 'default' => $default, 'array' => $array, 'format' => \json_encode(['name'=>'ip']), - ]), $response, $dbForExternal, $database, $audits); + ]), $response, $dbForExternal, $database, $audits, $usage); }); App::post('/v1/database/collections/:collectionId/attributes/url') @@ -470,11 +498,13 @@ App::post('/v1/database/collections/:collectionId/attributes/url') ->inject('dbForExternal') ->inject('database') ->inject('audits') - ->action(function ($collectionId, $attributeId, $size, $required, $default, $array, $response, $dbForExternal, $database, $audits) use ($attributesCallback) { + ->inject('usage') + ->action(function ($collectionId, $attributeId, $size, $required, $default, $array, $response, $dbForExternal, $database, $audits, $usage) use ($attributesCallback) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal*/ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ return $attributesCallback(new Document([ '$collection' => $collectionId, @@ -485,7 +515,7 @@ App::post('/v1/database/collections/:collectionId/attributes/url') 'default' => $default, 'array' => $array, 'format' => \json_encode(['name'=>'url']), - ]), $response, $dbForExternal, $database, $audits); + ]), $response, $dbForExternal, $database, $audits, $usage); }); App::post('/v1/database/collections/:collectionId/attributes/integer') @@ -511,12 +541,15 @@ App::post('/v1/database/collections/:collectionId/attributes/integer') ->inject('dbForExternal') ->inject('database') ->inject('audits') - ->action(function ($collectionId, $attributeId, $required, $min, $max, $default, $array, $response, $dbForExternal, $database, $audits) use ($attributesCallback) { + ->inject('usage') + ->action(function ($collectionId, $attributeId, $required, $min, $max, $default, $array, $response, $dbForExternal, $database, $audits, $usage) use ($attributesCallback) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal*/ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ + return $attributesCallback(new Document([ '$collection' => $collectionId, '$id' => $attributeId, @@ -530,7 +563,7 @@ App::post('/v1/database/collections/:collectionId/attributes/integer') 'min' => $min, 'max' => $max, ]), - ]), $response, $dbForExternal, $database, $audits); + ]), $response, $dbForExternal, $database, $audits, $usage); }); App::post('/v1/database/collections/:collectionId/attributes/float') @@ -556,11 +589,13 @@ App::post('/v1/database/collections/:collectionId/attributes/float') ->inject('dbForExternal') ->inject('database') ->inject('audits') - ->action(function ($collectionId, $attributeId, $required, $min, $max, $default, $array, $response, $dbForExternal, $database, $audits) use ($attributesCallback) { + ->inject('usage') + ->action(function ($collectionId, $attributeId, $required, $min, $max, $default, $array, $response, $dbForExternal, $database, $audits, $usage) use ($attributesCallback) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal*/ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ return $attributesCallback(new Document([ '$collection' => $collectionId, @@ -575,7 +610,7 @@ App::post('/v1/database/collections/:collectionId/attributes/float') 'min' => $min, 'max' => $max, ]), - ]), $response, $dbForExternal, $database, $audits); + ]), $response, $dbForExternal, $database, $audits, $usage); }); App::post('/v1/database/collections/:collectionId/attributes/boolean') @@ -599,11 +634,13 @@ App::post('/v1/database/collections/:collectionId/attributes/boolean') ->inject('dbForExternal') ->inject('database') ->inject('audits') - ->action(function ($collectionId, $attributeId, $required, $default, $array, $response, $dbForExternal, $database, $audits) use ($attributesCallback) { + ->inject('usage') + ->action(function ($collectionId, $attributeId, $required, $default, $array, $response, $dbForExternal, $database, $audits, $usage) use ($attributesCallback) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal*/ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ return $attributesCallback(new Document([ '$collection' => $collectionId, @@ -613,7 +650,7 @@ App::post('/v1/database/collections/:collectionId/attributes/boolean') 'required' => $required, 'default' => $default, 'array' => $array, - ]), $response, $dbForExternal, $database, $audits); + ]), $response, $dbForExternal, $database, $audits, $usage); }); App::get('/v1/database/collections/:collectionId/attributes') @@ -630,9 +667,11 @@ App::get('/v1/database/collections/:collectionId/attributes') ->param('collectionId', '', new UID(), 'Collection unique ID. You can create a new collection using the Database service [server integration](/docs/server/database#createCollection).') ->inject('response') ->inject('dbForExternal') - ->action(function ($collectionId, $response, $dbForExternal) { + ->inject('usage') + ->action(function ($collectionId, $response, $dbForExternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal */ + /** @var Appwrite\Stats\Stats $usage */ $collection = $dbForExternal->getCollection($collectionId); @@ -648,6 +687,8 @@ App::get('/v1/database/collections/:collectionId/attributes') ])]); }, $attributes); + $usage->setParam('database.collections.read', 1); + $response->dynamic(new Document([ 'sum' => \count($attributes), 'attributes' => $attributes @@ -669,9 +710,11 @@ App::get('/v1/database/collections/:collectionId/attributes/:attributeId') ->param('attributeId', '', new Key(), 'Attribute ID.') ->inject('response') ->inject('dbForExternal') - ->action(function ($collectionId, $attributeId, $response, $dbForExternal) { + ->inject('usage') + ->action(function ($collectionId, $attributeId, $response, $dbForExternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal */ + /** @var Appwrite\Stats\Stats $usage */ $collection = $dbForExternal->getCollection($collectionId); @@ -691,6 +734,8 @@ App::get('/v1/database/collections/:collectionId/attributes/:attributeId') $attribute = new Document([\array_merge($attributes[$attributeIndex], [ 'collectionId' => $collectionId, ])]); + + $usage->setParam('database.collections.read', 1); $response->dynamic($attribute, Response::MODEL_ATTRIBUTE); }); @@ -713,12 +758,14 @@ App::delete('/v1/database/collections/:collectionId/attributes/:attributeId') ->inject('database') ->inject('events') ->inject('audits') - ->action(function ($collectionId, $attributeId, $response, $dbForExternal, $database, $events, $audits) { + ->inject('usage') + ->action(function ($collectionId, $attributeId, $response, $dbForExternal, $database, $events, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal */ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $events */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $collection = $dbForExternal->getCollection($collectionId); @@ -747,6 +794,8 @@ App::delete('/v1/database/collections/:collectionId/attributes/:attributeId') ->setParam('document', $attribute) ; + $usage->setParam('database.collections.update', 1); + $events ->setParam('payload', $response->output($attribute, Response::MODEL_ATTRIBUTE)) ; @@ -781,11 +830,13 @@ App::post('/v1/database/collections/:collectionId/indexes') ->inject('dbForExternal') ->inject('database') ->inject('audits') - ->action(function ($collectionId, $indexId, $type, $attributes, $orders, $response, $dbForExternal, $database, $audits) { + ->inject('usage') + ->action(function ($collectionId, $indexId, $type, $attributes, $orders, $response, $dbForExternal, $database, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal */ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $audits */ $collection = $dbForExternal->getCollection($collectionId); @@ -837,6 +888,8 @@ App::post('/v1/database/collections/:collectionId/indexes') ->setParam('document', $index) ; + $usage->setParam('database.collections.update', 1); + $audits ->setParam('event', 'database.indexes.create') ->setParam('resource', 'database/indexes/'.$index->getId()) @@ -862,9 +915,11 @@ App::get('/v1/database/collections/:collectionId/indexes') ->param('collectionId', '', new UID(), 'Collection unique ID. You can create a new collection using the Database service [server integration](/docs/server/database#createCollection).') ->inject('response') ->inject('dbForExternal') - ->action(function ($collectionId, $response, $dbForExternal) { + ->inject('usage') + ->action(function ($collectionId, $response, $dbForExternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal */ + /** @var Appwrite\Stats\Stats $usage */ $collection = $dbForExternal->getCollection($collectionId); @@ -880,6 +935,8 @@ App::get('/v1/database/collections/:collectionId/indexes') ])]); }, $indexes); + $usage->setParam('database.collections.read', 1); + $response->dynamic(new Document([ 'sum' => \count($indexes), 'attributes' => $indexes, @@ -901,9 +958,11 @@ App::get('/v1/database/collections/:collectionId/indexes/:indexId') ->param('indexId', null, new Key(), 'Index ID.') ->inject('response') ->inject('dbForExternal') - ->action(function ($collectionId, $indexId, $response, $dbForExternal) { + ->inject('usage') + ->action(function ($collectionId, $indexId, $response, $dbForExternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal */ + /** @var Appwrite\Stats\Stats $usage */ $collection = $dbForExternal->getCollection($collectionId); @@ -923,6 +982,8 @@ App::get('/v1/database/collections/:collectionId/indexes/:indexId') $index = new Document([\array_merge($indexes[$indexIndex], [ 'collectionId' => $collectionId, ])]); + + $usage->setParam('database.collections.read', 1); $response->dynamic($index, Response::MODEL_INDEX); }); @@ -945,12 +1006,14 @@ App::delete('/v1/database/collections/:collectionId/indexes/:indexId') ->inject('database') ->inject('events') ->inject('audits') - ->action(function ($collectionId, $indexId, $response, $dbForExternal, $database, $events, $audits) { + ->inject('usage') + ->action(function ($collectionId, $indexId, $response, $dbForExternal, $database, $events, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal */ /** @var Appwrite\Event\Event $database */ /** @var Appwrite\Event\Event $events */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $collection = $dbForExternal->getCollection($collectionId); @@ -979,6 +1042,8 @@ App::delete('/v1/database/collections/:collectionId/indexes/:indexId') ->setParam('document', $index) ; + $usage->setParam('database.collections.update', 1); + $events ->setParam('payload', $response->output($index, Response::MODEL_INDEX)) ; @@ -1013,11 +1078,13 @@ App::post('/v1/database/collections/:collectionId/documents') ->inject('dbForExternal') ->inject('user') ->inject('audits') - ->action(function ($documentId, $collectionId, $data, $read, $write, $response, $dbForExternal, $user, $audits) { + ->inject('usage') + ->action(function ($documentId, $collectionId, $data, $read, $write, $response, $dbForExternal, $user, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal */ /** @var Utopia\Database\Document $user */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $data = (\is_string($data)) ? \json_decode($data, true) : $data; // Cast to JSON array @@ -1046,6 +1113,11 @@ App::post('/v1/database/collections/:collectionId/documents') throw new Exception($exception->getMessage(), 400); } + $usage + ->setParam('database.documents.create', 1) + ->setParam('database.collections.' . $collectionId . '.documents.create', 1) + ; + $audits ->setParam('event', 'database.documents.create') ->setParam('resource', 'database/document/'.$document->getId()) @@ -1076,9 +1148,11 @@ App::get('/v1/database/collections/:collectionId/documents') ->param('orderTypes', [], new ArrayList(new WhiteList(['DESC', 'ASC'], true)), 'Array of order directions for sorting attribtues. Possible values are DESC for descending order, or ASC for ascending order.', true) ->inject('response') ->inject('dbForExternal') - ->action(function ($collectionId, $queries, $limit, $offset, $after, $orderAttributes, $orderTypes, $response, $dbForExternal) { + ->inject('usage') + ->action(function ($collectionId, $queries, $limit, $offset, $after, $orderAttributes, $orderTypes, $response, $dbForExternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal */ + /** @var Appwrite\Stats\Stats $usage */ $collection = $dbForExternal->getCollection($collectionId); @@ -1112,6 +1186,11 @@ App::get('/v1/database/collections/:collectionId/documents') $documents = $dbForExternal->find($collectionId, $queries, $limit, $offset, $orderAttributes, $orderTypes, $afterDocument ?? null); + $usage + ->setParam('database.documents.read', 1) + ->setParam('database.collections.' . $collectionId . '.documents.read', 1) + ; + $response->dynamic(new Document([ 'sum' => \count($documents), 'documents' => $documents, @@ -1133,7 +1212,8 @@ App::get('/v1/database/collections/:collectionId/documents/:documentId') ->param('documentId', null, new UID(), 'Document unique ID.') ->inject('response') ->inject('dbForExternal') - ->action(function ($collectionId, $documentId, $response, $dbForExternal) { + ->inject('usage') + ->action(function ($collectionId, $documentId, $response, $dbForExternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal */ @@ -1149,6 +1229,11 @@ App::get('/v1/database/collections/:collectionId/documents/:documentId') throw new Exception('No document found', 404); } + $usage + ->setParam('database.documents.read', 1) + ->setParam('database.collections.' . $collectionId . '.documents.read', 1) + ; + $response->dynamic($document, Response::MODEL_DOCUMENT); }); @@ -1172,10 +1257,12 @@ App::patch('/v1/database/collections/:collectionId/documents/:documentId') ->inject('response') ->inject('dbForExternal') ->inject('audits') - ->action(function ($collectionId, $documentId, $data, $read, $write, $response, $dbForExternal, $audits) { + ->inject('usage') + ->action(function ($collectionId, $documentId, $data, $read, $write, $response, $dbForExternal, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $collection = $dbForExternal->getCollection($collectionId); @@ -1212,7 +1299,12 @@ App::patch('/v1/database/collections/:collectionId/documents/:documentId') throw new Exception('Unauthorized permissions', 401); } catch (StructureException $exception) { throw new Exception('Bad structure. '.$exception->getMessage(), 400); - } + } + + $usage + ->setParam('database.documents.update', 1) + ->setParam('database.collections.' . $collectionId . '.documents.update', 1) + ; $audits ->setParam('event', 'database.documents.update') @@ -1240,11 +1332,13 @@ App::delete('/v1/database/collections/:collectionId/documents/:documentId') ->inject('dbForExternal') ->inject('events') ->inject('audits') - ->action(function ($collectionId, $documentId, $response, $dbForExternal, $events, $audits) { + ->inject('usage') + ->action(function ($collectionId, $documentId, $response, $dbForExternal, $events, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal */ /** @var Appwrite\Event\Event $events */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $collection = $dbForExternal->getCollection($collectionId); @@ -1260,6 +1354,11 @@ App::delete('/v1/database/collections/:collectionId/documents/:documentId') $success = $dbForExternal->deleteDocument($collectionId, $documentId); + $usage + ->setParam('database.documents.delete', 1) + ->setParam('database.collections.' . $collectionId . '.documents.delete', 1) + ; + $events ->setParam('eventData', $response->output($document, Response::MODEL_DOCUMENT)) ; From cc49cb6a0450a4e2a63f3ec19f043d90c1e23e1e Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 15 Aug 2021 17:02:46 +0545 Subject: [PATCH 023/206] set collection Id as param --- app/controllers/api/database.php | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 6c2e2525bd..c3ed0edfa5 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -1115,7 +1115,7 @@ App::post('/v1/database/collections/:collectionId/documents') $usage ->setParam('database.documents.create', 1) - ->setParam('database.collections.' . $collectionId . '.documents.create', 1) + ->setParam('collectionId', $collectionId) ; $audits @@ -1188,7 +1188,7 @@ App::get('/v1/database/collections/:collectionId/documents') $usage ->setParam('database.documents.read', 1) - ->setParam('database.collections.' . $collectionId . '.documents.read', 1) + ->setParam('collectionId', $collectionId) ; $response->dynamic(new Document([ @@ -1231,7 +1231,7 @@ App::get('/v1/database/collections/:collectionId/documents/:documentId') $usage ->setParam('database.documents.read', 1) - ->setParam('database.collections.' . $collectionId . '.documents.read', 1) + ->setParam('collectionId', $collectionId) ; $response->dynamic($document, Response::MODEL_DOCUMENT); @@ -1303,7 +1303,7 @@ App::patch('/v1/database/collections/:collectionId/documents/:documentId') $usage ->setParam('database.documents.update', 1) - ->setParam('database.collections.' . $collectionId . '.documents.update', 1) + ->setParam('collectionId', $collectionId) ; $audits @@ -1356,7 +1356,7 @@ App::delete('/v1/database/collections/:collectionId/documents/:documentId') $usage ->setParam('database.documents.delete', 1) - ->setParam('database.collections.' . $collectionId . '.documents.delete', 1) + ->setParam('collectionId', $collectionId) ; $events From 4bab152b3d5a5b61378388932df2c2918f48c78e Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Sun, 15 Aug 2021 17:24:49 +0545 Subject: [PATCH 024/206] initial db metrics collection --- src/Appwrite/Stats/Stats.php | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Stats/Stats.php b/src/Appwrite/Stats/Stats.php index 9625d1c201..2f85be805c 100644 --- a/src/Appwrite/Stats/Stats.php +++ b/src/Appwrite/Stats/Stats.php @@ -87,7 +87,6 @@ class Stats $networkResponseSize = $this->params['networkResponseSize'] ?? 0; $httpMethod = $this->params['httpMethod'] ?? ''; - $httpPath = $this->params['httpPath'] ?? ''; $httpRequest = $this->params['httpRequest'] ?? 0; $functionId = $this->params['functionId'] ?? ''; @@ -101,7 +100,7 @@ class Stats $this->statsd->setNamespace($this->namespace); if ($httpRequest >= 1) { - $this->statsd->increment('requests.all' . $tags . ',method=' . \strtolower($httpMethod) . ',path=' . str_replace(':', '*', $httpPath)); + $this->statsd->increment('requests.all' . $tags . ',method=' . \strtolower($httpMethod)); } if ($functionExecution >= 1) { @@ -113,6 +112,25 @@ class Stats $this->statsd->count('network.outbound' . $tags, $networkResponseSize); $this->statsd->count('network.all' . $tags, $networkRequestSize + $networkResponseSize); + $dbMetrics = [ + 'database.collections.create', + 'database.collections.read', + 'database.collections.update', + 'database.collections.delete', + 'database.documents.create', + 'database.documents.read', + 'database.documents.update', + 'database.documents.delete', + ]; + + foreach ($dbMetrics as $metric) { + $value = $this->params[$metric] ?? 0; + if ($value >= 1) { + $dbTags = ",project={$projectId},collectionId=" . ($this->params['collectionId'] ?? ''); + $this->statsd->increment($metric . $dbTags); + } + } + if ($storage >= 1) { $this->statsd->count('storage.all' . $tags, $storage); } From 8a207f02a0261391ed91d0034d8af6164e1c6614 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 16 Aug 2021 12:43:34 +0545 Subject: [PATCH 025/206] project stats and database stats --- app/controllers/api/projects.php | 6 +- app/tasks/usage.php | 345 ++++++++++++++----------------- src/Appwrite/Stats/Stats.php | 4 +- 3 files changed, 165 insertions(+), 190 deletions(-) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index e34c948837..c302e882b3 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -279,7 +279,7 @@ App::get('/v1/projects/:projectId/usage') $database = $client->selectDB('telegraf'); // Requests - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_requests_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $project->getId() . '\' GROUP BY time(' . $period[$range]['group'] . ') FILL(null)'); + $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_requests_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "projectId"=\'' . $project->getId() . '\' GROUP BY time(' . $period[$range]['group'] . ') FILL(null)'); $points = $result->getPoints(); foreach ($points as $point) { @@ -290,7 +290,7 @@ App::get('/v1/projects/:projectId/usage') } // Network - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_network_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $project->getId() . '\' GROUP BY time(' . $period[$range]['group'] . ') FILL(null)'); + $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_network_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "projectId"=\'' . $project->getId() . '\' GROUP BY time(' . $period[$range]['group'] . ') FILL(null)'); $points = $result->getPoints(); foreach ($points as $point) { @@ -301,7 +301,7 @@ App::get('/v1/projects/:projectId/usage') } // Functions - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_executions_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $project->getId() . '\' GROUP BY time(' . $period[$range]['group'] . ') FILL(null)'); + $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_executions_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "projectId"=\'' . $project->getId() . '\' GROUP BY time(' . $period[$range]['group'] . ') FILL(null)'); $points = $result->getPoints(); foreach ($points as $point) { diff --git a/app/tasks/usage.php b/app/tasks/usage.php index d1a2905e02..fa8de4c86f 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -14,6 +14,34 @@ use Utopia\Database\Document; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; +/** + * 1. Load all the projects + * 2. Load latest data entered entered for each project, for each period + * 3. Start the loop + * 4. Fore each project, for each metric, for each period - sync data + */ + +/** + * Only succefull operations + * + * database.collections.CRUD (project=x) + * database.documents.CRUD (project=x,collection=y) + * + * users.CRUD + * users.sessions.create (project=x,provider=y) + * users.sessions.delete (project=x,provider=y) + * + * storage.buckets.CRUD (project=x) + * storage.files.CRUD (project=x,files=y) + * + * refactor later + * - functions + * - realtime + * - teams + * - webhooks + * - keys - really later! + */ + $cli ->task('usage') ->desc('Schedules syncing data from influxdb to Appwrite console db') @@ -22,10 +50,90 @@ $cli Console::success(APP_NAME . ' usage sync process v1 has started'); $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); //30 seconds + $periods = [ + [ + 'key' => '30m', + 'startTime' => '-24 hours', + ], + [ + 'key' => '1d', + 'startTime' => '-90 days', + ], + ]; + + //use projectId from influxdb instead of iterating over projects from DB + + $globalMetrics = [ + 'requests' => [ + 'method' => 'getGlobalMetrics', + 'table' => 'appwrite_usage_requests_all', + ], + 'network' => [ + 'method' => 'getGlobalMetrics', + 'table' => 'appwrite_usage_network_all', + ], + 'executions' => [ + 'method' => 'getGlobalMetrics', + 'table' => 'appwrite_usage_executions_all', + ], + 'database.collections.create' => [ + 'method' => 'getDatabaseMetrics', + 'table' => 'appwrite_usage_database_collections_create', + ], + 'database.collections.read' => [ + 'method' => 'getDatabaseMetrics', + 'table' => 'appwrite_usage_database_collections_read', + ], + 'database.collections.update' => [ + 'method' => 'getDatabaseMetrics', + 'table' => 'appwrite_usage_database_collections_update', + ], + 'database.collections.delete' => [ + 'method' => 'getDatabaseMetrics', + 'table' => 'appwrite_usage_database_collections_delete', + ], + 'database.documents.create' => [ + 'method' => 'getDatabaseMetrics', + 'table' => 'appwrite_usage_database_documents_create', + ], + 'database.documents.read' => [ + 'method' => 'getDatabaseMetrics', + 'table' => 'appwrite_usage_database_documents_read', + ], + 'database.documents.update' => [ + 'method' => 'getDatabaseMetrics', + 'table' => 'appwrite_usage_database_documents_update', + ], + 'database.documents.delete' => [ + 'method' => 'getDatabaseMetrics', + 'table' => 'appwrite_usage_database_documents_delete', + ], + 'database.documents.collectionId.create' => [ + 'method' => 'getDatabaseMetrics', + 'table' => 'appwrite_usage_database_documents_create', + 'groupBy' => 'collectionId', + ], + 'database.documents.collectionId.read' => [ + 'method' => 'getDatabaseMetrics', + 'table' => 'appwrite_usage_database_documents_read', + 'groupBy' => 'collectionId', + ], + 'database.documents.collectionId.update' => [ + 'method' => 'getDatabaseMetrics', + 'table' => 'appwrite_usage_database_documents_update', + 'groupBy' => 'collectionId', + ], + 'database.documents.collectionId.delete' => [ + 'method' => 'getDatabaseMetrics', + 'table' => 'appwrite_usage_database_documents_delete', + 'groupBy' => 'collectionId', + ], + ]; + $attempts = 0; $max = 10; $sleep = 1; - do { + do { // connect to db try { $attempts++; $db = $register->get('db'); @@ -46,199 +154,66 @@ $cli $dbForProject = new Database(new MariaDB($db), $cacheAdapter); Authorization::disable(); - $projectIds = []; - $latestProject = null; $latestData = []; - do { - $projects = $dbForConsole->find('projects', [], 100, orderAfter:$latestProject); - if (!empty($projects)) { - $latestProject = $projects[array_key_last($projects)]; - $latestData = getLatestData($projects, $latestData, $dbForProject, $projectIds); - } - } while (!empty($projects)); - - $projects = null; $firstRun = true; - Console::loop(function () use ($interval, $register, &$projectIds, &$latestData, $dbForProject, $dbForConsole, &$firstRun, &$latestProject) { + Console::loop(function () use ($interval, $register, &$latestData, $dbForProject, $dbForConsole, &$firstRun, $globalMetrics, $periods) { $time = date('d-m-Y H:i:s', time()); - Console::info("[{$time}] Syncing usage data from influxdb to Appwrite Console DB every {$interval} seconds"); - - if (!$firstRun) { - $projects = $dbForConsole->find('projects', limit:100, orderAfter:$latestProject); - if (!empty($projects)) { - $latestProject = $projects[array_key_last($projects)]; - $latestData = getLatestData($projects, $latestData, $dbForProject, $projectIds); - } - } + Console::info("[{$time}] Aggregating usage data every {$interval} seconds"); $client = $register->get('influxdb'); if ($client) { - foreach ($projectIds as $id => $value) { - syncData($client, $id, $latestData, $dbForProject); + $database = $client->selectDB('telegraf'); + // sync data + foreach ($globalMetrics as $metric => $options) { + foreach ($periods as $period) { + $start = DateTime::createFromFormat('U', \strtotime($period['startTime']))->format(DateTime::RFC3339); + $end = DateTime::createFromFormat('U', \strtotime('now'))->format(DateTime::RFC3339); + + $table = $options['table']; + $groupBy = $options['groupBy'] ?? ''; + + $query = 'SELECT sum(value) AS "value" FROM "' . $table . '" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' GROUP BY time(' . $period['key'] . '), "projectId"' . (empty($groupBy) ? '' : ', "' . $groupBy . '"') . ' FILL(null)'; + $result = $database->query($query); + $points = $result->getPoints(); + foreach ($points as $point) { + $projectId = $point['projectId']; + if (!empty($projectId) && $projectId != 'console') { + $dbForProject->setNamespace('project_' . $projectId . '_internal'); + if (!empty($groupBy)) { + $groupedBy = $point[$groupBy]; + if (empty($groupedBy)) { + continue; + } + $metric = str_replace($groupBy, $groupedBy, $metric); + } + $time = \strtotime($point['time']); + $id = \md5($time . '_' . $period['key'] . '_' . $metric); + $value = (!empty($point['value'])) ? $point['value'] : 0; + try { + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'period' => $period['key'], + 'time' => $time, + 'metric' => $metric, + 'value' => $value, + 'type' => 0, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $value)); + } + $latestData[$projectId][$metric][$period['key']] = $time; + } catch (\Exception$e) { + Console::warning("Failed to save data for project {$projectId} and metric {$metric}"); + } + } + } + } } } $firstRun = false; }, $interval); }); - -function getLatestData(&$projects, &$latestData, $dbForProject, &$projectIds) -{ - foreach ($projects as $project) { - $id = $project->getId(); - $projectIds[$id] = true; - $dbForProject->setNamespace("project_{$id}_internal"); - foreach (['requests', 'network', 'executions', 'database.reads'] as $metric) { - $doc = $dbForProject->findOne('stats', [new Query("period", Query::TYPE_EQUAL, ["1d"]), new Query("metric", Query::TYPE_EQUAL, [$metric])], 0, ['time'], [Database::ORDER_DESC]); - $latestData[$id][$metric]["1d"] = $doc ? $doc->getAttribute('time') : null; - $doc = $dbForProject->findOne('stats', [new Query("period", Query::TYPE_EQUAL, ["30m"])], 0, ['time'], [Database::ORDER_DESC]); - $latestData[$id][$metric]["30m"] = $doc ? $doc->getAttribute('time') : null; - } - } - $projects = null; - return $latestData; -} - -function syncData($client, $projectId, &$latestData, $dbForProject) -{ - foreach (['30m', '1d'] as $period) { - $start = DateTime::createFromFormat('U', \strtotime($period == '1d' ? '-90 days' : '-24 hours'))->format(DateTime::RFC3339); - $end = DateTime::createFromFormat('U', \strtotime('now'))->format(DateTime::RFC3339); - $database = $client->selectDB('telegraf'); - $dbForProject->setNamespace("project_{$projectId}_internal"); - foreach (['requests', 'network', 'executions'] as $metric) { - if (!empty($latestData[$projectId][$metric][$period])) { - $start = DateTime::createFromFormat('U', $latestData[$projectId][$metric][$period])->format(DateTime::RFC3339); - } - syncMetric($database, $projectId, $period, 'requests', $start, $end, $dbForProject); - } - syncMetricPaths($database, $projectId, $period, $start, $end, $dbForProject); - } - -} - -function syncMetricPaths($database, $projectId, $period, $start, $end, $dbForProject) -{ - $start = DateTime::createFromFormat('U', \strtotime($period == '1d' ? '-7 days' : '-24 hours'))->format(DateTime::RFC3339); - if (!empty($latestData[$projectId]['database'][$period])) { - $start = DateTime::createFromFormat('U', $latestData[$projectId]['database.reads'][$period])->format(DateTime::RFC3339); - } - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_requests_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $projectId . '\'GROUP BY time(' . $period . '), "path", "method" FILL(null)'); - $points = $result->getPoints(); - - $databaseMetrics = [ - 'creates' => [ - 'method' => 'post', - 'paths' => [ - '/v1/database/collections' => true, - '/v1/database/collections/*collectionId/attributes/string' => true, - '/v1/database/collections/*collectionId/attributes/email' => true, - '/v1/database/collections/*collectionId/attributes/ip' => true, - '/v1/database/collections/*collectionId/attributes/url' => true, - '/v1/database/collections/*collectionId/attributes/integer' => true, - '/v1/database/collections/*collectionId/attributes/float' => true, - '/v1/database/collections/*collectionId/attributes/boolean' => true, - '/v1/database/collections/*collectionId/indexes' => true, - '/v1/database/collections/*collectionId/documents' => true, - ], - ], - 'reads' => [ - 'method' => 'get', - 'paths' => [ - '/v1/database/collections' => true, - '/v1/database/collections/*collectionId' => true, - '/v1/database/collections/*collectionId/attributes' => true, - '/v1/database/collections/*collectionId/attributes/*attributeId' => true, - '/v1/database/collections/*collectionId/indexes' => true, - '/v1/database/collections/*collectionId/indexes/*indexId' => true, - '/v1/database/collections/*collectionId/documents' => true, - '/v1/database/collections/*collectionId/documents/*documentId' => true, - ], - ], - 'updates' => [ - 'method' => 'put', - 'paths' => [ - '/v1/database/collections/*collectionId' => true, - '/v1/database/collections/*collectionId/documents/*documentId' => true, - ], - ], - 'deletes' => [ - 'method' => 'delete', - 'paths' => [ - '/v1/database/collections/*collectionId' => true, - '/v1/database/collections/*collectionId/attributes/*attributeId' => true, - '/v1/database/collections/*collectionId/indexes/*indexId' => true, - '/v1/database/collections/*collectionId/documents/*documentId' => true, - ], - ], - ]; - - $dbStats = []; - foreach ($points as $point) { - $time = \strtotime($point['time']); - $value = (!empty($point['value'])) ? $point['value'] : 0; - $path = $point['path'] ?? ''; - $method = $point['method'] ?? ''; - - foreach (['creates', 'reads', 'updates', 'deletes'] as $operation) { - if ($method == $databaseMetrics[$operation]['method'] - && array_key_exists($path, $databaseMetrics[$operation]['paths'])) { - if (empty($dbStats["database.{$operation}"][$period][$time])) { - $dbStats["database.{$operation}"][$period][$time] = 0; - } - $dbStats["database.{$operation}"][$period][$time] += $value; - } - } - } - - $time = \strtotime($start); - foreach ($dbStats as $metric => $stats) { - foreach ($stats as $period => $times) { - foreach ($times as $time => $value) { - $id = \md5($time . '_' . $period . '_' . $metric); - $document = $dbForProject->getDocument('stats', $id); - if ($document->isEmpty()) { - $dbForProject->createDocument('stats', new Document([ - '$id' => $id, - 'period' => $period, - 'time' => $time, - 'metric' => $metric, - 'value' => $value, - 'type' => 0, - ])); - } else { - $dbForProject->updateDocument('stats', $document->getId(), - $document->setAttribute('value', $value)); - } - } - } - } - $latestData[$projectId]['database.reads'][$period] = $time; -} - -function syncMetric($database, $projectId, $period, $metric, $start, $end, $dbForProject) -{ - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_' . $metric . '_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "project"=\'' . $projectId . '\'GROUP BY time(' . $period . ') FILL(null)'); - $points = $result->getPoints(); - - foreach ($points as $point) { - $time = \strtotime($point['time']); - $id = \md5($time . '_' . $period . '_' . $metric); - $value = (!empty($point['value'])) ? $point['value'] : 0; - $document = $dbForProject->getDocument('stats', $id); - if ($document->isEmpty()) { - $dbForProject->createDocument('stats', new Document([ - '$id' => $id, - 'period' => $period, - 'time' => $time, - 'metric' => $metric, - 'value' => $value, - 'type' => 0, - ])); - } else { - $dbForProject->updateDocument('stats', $document->getId(), - $document->setAttribute('value', $value)); - } - $latestData[$projectId][$metric][$period] = $time; - } -} diff --git a/src/Appwrite/Stats/Stats.php b/src/Appwrite/Stats/Stats.php index 2f85be805c..60609f1a4e 100644 --- a/src/Appwrite/Stats/Stats.php +++ b/src/Appwrite/Stats/Stats.php @@ -94,7 +94,7 @@ class Stats $functionExecutionTime = $this->params['functionExecutionTime'] ?? 0; $functionStatus = $this->params['functionStatus'] ?? ''; - $tags = ",project={$projectId},version=" . App::getEnv('_APP_VERSION', 'UNKNOWN'); + $tags = ",projectId={$projectId},version=" . App::getEnv('_APP_VERSION', 'UNKNOWN'); // the global namespace is prepended to every key (optional) $this->statsd->setNamespace($this->namespace); @@ -126,7 +126,7 @@ class Stats foreach ($dbMetrics as $metric) { $value = $this->params[$metric] ?? 0; if ($value >= 1) { - $dbTags = ",project={$projectId},collectionId=" . ($this->params['collectionId'] ?? ''); + $dbTags = ",projectId={$projectId},collectionId=" . ($this->params['collectionId'] ?? ''); $this->statsd->increment($metric . $dbTags); } } From 1e9f3e38b200aec910774686b8e80bb20ed8757d Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 16 Aug 2021 13:10:20 +0545 Subject: [PATCH 026/206] storage stats --- app/controllers/api/storage.php | 64 ++++++++++++++++++++++++++++----- app/tasks/usage.php | 22 +++++++++++- src/Appwrite/Stats/Stats.php | 15 ++++++++ 3 files changed, 91 insertions(+), 10 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 5a5c43de58..651985057a 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -54,7 +54,7 @@ App::post('/v1/storage/files') /** @var Utopia\Database\Database $dbForInternal */ /** @var Utopia\Database\Document $user */ /** @var Appwrite\Event\Event $audits */ - /** @var Appwrite\Event\Event $usage */ + /** @var Appwrite\Stats\Stats $usage */ $file = $request->getFiles('file'); @@ -150,6 +150,8 @@ App::post('/v1/storage/files') $usage ->setParam('storage', $sizeActual) + ->setParam('storage.files.create', 1) + ->setParam('bucketId', 'default') ; $response->setStatusCode(Response::STATUS_CODE_CREATED); @@ -175,9 +177,11 @@ App::get('/v1/storage/files') ->param('orderType', 'ASC', new WhiteList(['ASC', 'DESC'], true), 'Order result by ASC or DESC order.', true) ->inject('response') ->inject('dbForInternal') - ->action(function ($search, $limit, $offset, $after, $orderType, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($search, $limit, $offset, $after, $orderType, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $queries = ($search) ? [new Query('name', Query::TYPE_SEARCH, $search)] : []; @@ -189,6 +193,11 @@ App::get('/v1/storage/files') } } + $usage + ->setParam('storage.files.read', 1) + ->setParam('bucketId', 'default') + ; + $response->dynamic(new Document([ 'files' => $dbForInternal->find('files', $queries, $limit, $offset, [], [$orderType], $afterFile ?? null), 'sum' => $dbForInternal->count('files', $queries, APP_LIMIT_COUNT), @@ -209,16 +218,21 @@ App::get('/v1/storage/files/:fileId') ->param('fileId', '', new UID(), 'File unique ID.') ->inject('response') ->inject('dbForInternal') - ->action(function ($fileId, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($fileId, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $file = $dbForInternal->getDocument('files', $fileId); if (empty($file->getId())) { throw new Exception('File not found', 404); } - + $usage + ->setParam('storage.files.read', 1) + ->setParam('bucketId', 'default') + ; $response->dynamic($file, Response::MODEL_FILE); }); @@ -249,11 +263,13 @@ App::get('/v1/storage/files/:fileId/preview') ->inject('response') ->inject('project') ->inject('dbForInternal') - ->action(function ($fileId, $width, $height, $gravity, $quality, $borderWidth, $borderColor, $borderRadius, $opacity, $rotation, $background, $output, $request, $response, $project, $dbForInternal) { + ->inject('usage') + ->action(function ($fileId, $width, $height, $gravity, $quality, $borderWidth, $borderColor, $borderRadius, $opacity, $rotation, $background, $output, $request, $response, $project, $dbForInternal, $usage) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $project */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $stats */ $storage = 'files'; @@ -366,6 +382,11 @@ App::get('/v1/storage/files/:fileId/preview') $cache->save($key, $data); + $usage + ->setParam('storage.files.read', 1) + ->setParam('bucketId', 'default') + ; + $response ->setContentType($outputs[$output]) ->addHeader('Expires', $date) @@ -390,9 +411,11 @@ App::get('/v1/storage/files/:fileId/download') ->param('fileId', '', new UID(), 'File unique ID.') ->inject('response') ->inject('dbForInternal') - ->action(function ($fileId, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($fileId, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $file = $dbForInternal->getDocument('files', $fileId); @@ -424,6 +447,11 @@ App::get('/v1/storage/files/:fileId/download') $source = $compressor->decompress($source); + $usage + ->setParam('storage.files.read', 1) + ->setParam('bucketId', 'default') + ; + // Response $response ->setContentType($file->getAttribute('mimeType')) @@ -448,9 +476,11 @@ App::get('/v1/storage/files/:fileId/view') ->param('fileId', '', new UID(), 'File unique ID.') ->inject('response') ->inject('dbForInternal') - ->action(function ($fileId, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($fileId, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $file = $dbForInternal->getDocument('files', $fileId); $mimes = Config::getParam('storage-mimes'); @@ -490,6 +520,11 @@ App::get('/v1/storage/files/:fileId/view') $output = $compressor->decompress($source); $fileName = $file->getAttribute('name', ''); + $usage + ->setParam('storage.files.read', 1) + ->setParam('bucketId', 'default') + ; + // Response $response ->setContentType($contentType) @@ -520,7 +555,8 @@ App::put('/v1/storage/files/:fileId') ->inject('response') ->inject('dbForInternal') ->inject('audits') - ->action(function ($fileId, $read, $write, $response, $dbForInternal, $audits) { + ->inject('usage') + ->action(function ($fileId, $read, $write, $response, $dbForInternal, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $audits */ @@ -542,6 +578,11 @@ App::put('/v1/storage/files/:fileId') ->setParam('resource', 'storage/files/'.$file->getId()) ; + $usage + ->setParam('storage.files.update', 1) + ->setParam('bucketId', 'default') + ; + $response->dynamic($file, Response::MODEL_FILE); }); @@ -567,7 +608,7 @@ App::delete('/v1/storage/files/:fileId') /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $events */ /** @var Appwrite\Event\Event $audits */ - /** @var Appwrite\Event\Event $usage */ + /** @var Appwrite\Stats\Stats $usage */ $file = $dbForInternal->getDocument('files', $fileId); @@ -596,5 +637,10 @@ App::delete('/v1/storage/files/:fileId') ->setParam('eventData', $response->output($file, Response::MODEL_FILE)) ; + $usage + ->setParam('storage.files.delete', 1) + ->setParam('bucketId', 'default') + ; + $response->noContent(); }); \ No newline at end of file diff --git a/app/tasks/usage.php b/app/tasks/usage.php index fa8de4c86f..5230963098 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -32,7 +32,7 @@ use Utopia\Database\Validator\Authorization; * users.sessions.delete (project=x,provider=y) * * storage.buckets.CRUD (project=x) - * storage.files.CRUD (project=x,files=y) + * storage.files.CRUD (project=x,bucket=y) * * refactor later * - functions @@ -128,6 +128,26 @@ $cli 'table' => 'appwrite_usage_database_documents_delete', 'groupBy' => 'collectionId', ], + 'storage.buckets.bucketId.files.create' => [ + 'method' => 'getDatabaseMetrics', + 'table' => 'appwrite_usage_storage_files_create', + 'groupBy' => 'bucketId', + ], + 'storage.buckets.bucketId.files.read' => [ + 'method' => 'getDatabaseMetrics', + 'table' => 'appwrite_usage_storage_files_read', + 'groupBy' => 'bucketId', + ], + 'storage.buckets.bucketId.files.update' => [ + 'method' => 'getDatabaseMetrics', + 'table' => 'appwrite_usage_storage_files_update', + 'groupBy' => 'bucketId', + ], + 'storage.buckets.bucketId.files.delete' => [ + 'method' => 'getDatabaseMetrics', + 'table' => 'appwrite_usage_storage_files_delete', + 'groupBy' => 'bucketId', + ], ]; $attempts = 0; diff --git a/src/Appwrite/Stats/Stats.php b/src/Appwrite/Stats/Stats.php index 60609f1a4e..5c24e1ae94 100644 --- a/src/Appwrite/Stats/Stats.php +++ b/src/Appwrite/Stats/Stats.php @@ -131,6 +131,21 @@ class Stats } } + $storageMertics = [ + 'storage.files.create', + 'storage.files.read', + 'storage.files.update', + 'storage.files.delete', + ]; + + foreach ($storageMertics as $metric) { + $value = $this->params[$metric] ?? 0; + if ($value >= 1) { + $storageTags = ",projectId={$projectId},bucketId=" . ($this->params['bucketId'] ?? ''); + $this->statsd->increment($metric . $storageTags); + } + } + if ($storage >= 1) { $this->statsd->count('storage.all' . $tags, $storage); } From b4c794c7ba8bdf507f466286bc763b9acb76f988 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 16 Aug 2021 13:41:18 +0545 Subject: [PATCH 027/206] users usage params --- app/controllers/api/users.php | 94 ++++++++++++++++++++++++++++------- 1 file changed, 77 insertions(+), 17 deletions(-) diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 11beca048b..eb3abd12cf 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -36,9 +36,11 @@ App::post('/v1/users') ->param('name', '', new Text(128), 'User name. Max length: 128 chars.', true) ->inject('response') ->inject('dbForInternal') - ->action(function ($userId, $email, $password, $name, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($userId, $email, $password, $name, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $email = \strtolower($email); @@ -65,6 +67,10 @@ App::post('/v1/users') throw new Exception('Account already exists', 409); } + $usage + ->setParam('users.create', 1) + ; + $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($user, Response::MODEL_USER); }); @@ -87,9 +93,11 @@ App::get('/v1/users') ->param('orderType', 'ASC', new WhiteList(['ASC', 'DESC'], true), 'Order result by ASC or DESC order.', true) ->inject('response') ->inject('dbForInternal') - ->action(function ($search, $limit, $offset, $after, $orderType, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($search, $limit, $offset, $after, $orderType, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ if (!empty($after)) { $afterUser = $dbForInternal->getDocument('users', $after); @@ -101,6 +109,10 @@ App::get('/v1/users') $results = $dbForInternal->find('users', [], $limit, $offset, [], [$orderType], $afterUser ?? null); $sum = $dbForInternal->count('users', [], APP_LIMIT_COUNT); + + $usage + ->setParam('users.read', 1) + ; $response->dynamic(new Document([ 'users' => $results, @@ -122,9 +134,11 @@ App::get('/v1/users/:userId') ->param('userId', '', new UID(), 'User unique ID.') ->inject('response') ->inject('dbForInternal') - ->action(function ($userId, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($userId, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->getDocument('users', $userId); @@ -132,6 +146,9 @@ App::get('/v1/users/:userId') throw new Exception('User not found', 404); } + $usage + ->setParam('users.read', 1) + ; $response->dynamic($user, Response::MODEL_USER); }); @@ -149,9 +166,11 @@ App::get('/v1/users/:userId/prefs') ->param('userId', '', new UID(), 'User unique ID.') ->inject('response') ->inject('dbForInternal') - ->action(function ($userId, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($userId, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->getDocument('users', $userId); @@ -161,6 +180,9 @@ App::get('/v1/users/:userId/prefs') $prefs = $user->getAttribute('prefs', new \stdClass()); + $usage + ->setParam('users.read', 1) + ; $response->dynamic(new Document($prefs), Response::MODEL_PREFERENCES); }); @@ -179,10 +201,12 @@ App::get('/v1/users/:userId/sessions') ->inject('response') ->inject('dbForInternal') ->inject('locale') - ->action(function ($userId, $response, $dbForInternal, $locale) { + ->inject('usage') + ->action(function ($userId, $response, $dbForInternal, $locale, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Utopia\Locale\Locale $locale */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->getDocument('users', $userId); @@ -202,6 +226,9 @@ App::get('/v1/users/:userId/sessions') $sessions[$key] = $session; } + $usage + ->setParam('users.read', 1) + ; $response->dynamic(new Document([ 'sessions' => $sessions, 'sum' => count($sessions), @@ -224,12 +251,14 @@ App::get('/v1/users/:userId/logs') ->inject('dbForInternal') ->inject('locale') ->inject('geodb') - ->action(function ($userId, $response, $dbForInternal, $locale, $geodb) { + ->inject('usage') + ->action(function ($userId, $response, $dbForInternal, $locale, $geodb, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $project */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Utopia\Locale\Locale $locale */ /** @var MaxMind\Db\Reader $geodb */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->getDocument('users', $userId); @@ -311,6 +340,9 @@ App::get('/v1/users/:userId/logs') } } + $usage + ->setParam('users.read', 1) + ; $response->dynamic(new Document(['logs' => $output]), Response::MODEL_LOG_LIST); }); @@ -330,9 +362,11 @@ App::patch('/v1/users/:userId/status') ->param('status', null, new Boolean(true), 'User Status. To activate the user pass `true` and to block the user pass `false`') ->inject('response') ->inject('dbForInternal') - ->action(function ($userId, $status, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($userId, $status, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->getDocument('users', $userId); @@ -342,6 +376,9 @@ App::patch('/v1/users/:userId/status') $user = $dbForInternal->updateDocument('users', $user->getId(), $user->setAttribute('status', (bool) $status)); + $usage + ->setParam('users.update', 1) + ; $response->dynamic($user, Response::MODEL_USER); }); @@ -361,9 +398,11 @@ App::patch('/v1/users/:userId/verification') ->param('emailVerification', false, new Boolean(), 'User Email Verification Status.') ->inject('response') ->inject('dbForInternal') - ->action(function ($userId, $emailVerification, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($userId, $emailVerification, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->getDocument('users', $userId); @@ -373,6 +412,9 @@ App::patch('/v1/users/:userId/verification') $user = $dbForInternal->updateDocument('users', $user->getId(), $user->setAttribute('emailVerification', $emailVerification)); + $usage + ->setParam('users.update', 1) + ; $response->dynamic($user, Response::MODEL_USER); }); @@ -392,9 +434,11 @@ App::patch('/v1/users/:userId/prefs') ->param('prefs', '', new Assoc(), 'Prefs key-value JSON object.') ->inject('response') ->inject('dbForInternal') - ->action(function ($userId, $prefs, $response, $dbForInternal) { + ->inject('usage') + ->action(function ($userId, $prefs, $response, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->getDocument('users', $userId); @@ -404,6 +448,9 @@ App::patch('/v1/users/:userId/prefs') $user = $dbForInternal->updateDocument('users', $user->getId(), $user->setAttribute('prefs', $prefs)); + $usage + ->setParam('users.update', 1) + ; $response->dynamic(new Document($prefs), Response::MODEL_PREFERENCES); }); @@ -423,10 +470,12 @@ App::delete('/v1/users/:userId/sessions/:sessionId') ->inject('response') ->inject('dbForInternal') ->inject('events') - ->action(function ($userId, $sessionId, $response, $dbForInternal, $events) { + ->inject('usage') + ->action(function ($userId, $sessionId, $response, $dbForInternal, $events, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $events */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->getDocument('users', $userId); @@ -453,6 +502,11 @@ App::delete('/v1/users/:userId/sessions/:sessionId') } } + $usage + ->setParam('users.update', 1) + ->setParam('users.sessions.delete', 1) + ; + $response->noContent(); }); @@ -471,10 +525,12 @@ App::delete('/v1/users/:userId/sessions') ->inject('response') ->inject('dbForInternal') ->inject('events') - ->action(function ($userId, $response, $dbForInternal, $events) { + ->inject('usage') + ->action(function ($userId, $response, $dbForInternal, $events, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $events */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->getDocument('users', $userId); @@ -494,6 +550,10 @@ App::delete('/v1/users/:userId/sessions') ->setParam('eventData', $response->output($user, Response::MODEL_USER)) ; + $usage + ->setParam('users.update', 1) + ->setParam('users.sessions.delete', 1) + ; $response->noContent(); }); @@ -513,11 +573,13 @@ App::delete('/v1/users/:userId') ->inject('dbForInternal') ->inject('events') ->inject('deletes') - ->action(function ($userId, $response, $dbForInternal, $events, $deletes) { + ->inject('users') + ->action(function ($userId, $response, $dbForInternal, $events, $deletes, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $events */ /** @var Appwrite\Event\Event $deletes */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->getDocument('users', $userId); @@ -528,11 +590,6 @@ App::delete('/v1/users/:userId') if (!$dbForInternal->deleteDocument('users', $userId)) { throw new Exception('Failed to remove user from DB', 500); } - - // $dbForInternal->createDocument('users', new Document([ - // '$id' => $userId, - // '$read' => ['role:all'], - // ])); $deletes ->setParam('type', DELETE_TYPE_DOCUMENT) @@ -543,5 +600,8 @@ App::delete('/v1/users/:userId') ->setParam('eventData', $response->output($user, Response::MODEL_USER)) ; + $usage + ->setParam('users.delete', 1) + ; $response->noContent(); }); From 00a1612b2431967cae6388a195e5b5b3ff167e7b Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 16 Aug 2021 14:38:34 +0545 Subject: [PATCH 028/206] users and sessions stats --- app/controllers/api/account.php | 150 +++++++++++++++++++++++++++----- app/controllers/api/users.php | 2 +- app/tasks/usage.php | 83 ++++++------------ src/Appwrite/Stats/Stats.php | 36 +++++++- 4 files changed, 189 insertions(+), 82 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index ef08805117..b2baa4fcbe 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -52,12 +52,14 @@ App::post('/v1/account') ->inject('project') ->inject('dbForInternal') ->inject('audits') - ->action(function ($userId, $email, $password, $name, $request, $response, $project, $dbForInternal, $audits) { + ->inject('usage') + ->action(function ($userId, $email, $password, $name, $request, $response, $project, $dbForInternal, $audits, $usage) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $project */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Usage\Usage $usage */ $email = \strtolower($email); if ('console' === $project->getId()) { @@ -120,6 +122,9 @@ App::post('/v1/account') ->setParam('resource', 'users/' . $user->getId()) ; + $usage + ->setParam('users.create', 1) + ; $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($user, Response::MODEL_USER); }); @@ -147,13 +152,15 @@ App::post('/v1/account/sessions') ->inject('locale') ->inject('geodb') ->inject('audits') - ->action(function ($email, $password, $request, $response, $dbForInternal, $locale, $geodb, $audits) { + ->inject('usage') + ->action(function ($email, $password, $request, $response, $dbForInternal, $locale, $geodb, $audits, $usage) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Utopia\Locale\Locale $locale */ /** @var MaxMind\Db\Reader $geodb */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Usage\Usage $usage */ $email = \strtolower($email); $protocol = $request->getProtocol(); @@ -227,6 +234,11 @@ App::post('/v1/account/sessions') ->setAttribute('countryName', $countryName) ; + $usage + ->setParam('users.update', 1) + ->setParam('users.sessions.create', 1) + ->setParam('provider', 'email') + ; $response->dynamic($session, Response::MODEL_SESSION); }); @@ -357,7 +369,8 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') ->inject('geodb') ->inject('audits') ->inject('events') - ->action(function ($provider, $code, $state, $request, $response, $project, $user, $dbForInternal, $geodb, $audits, $events) use ($oauthDefaultSuccess) { + ->inject('usage') + ->action(function ($provider, $code, $state, $request, $response, $project, $user, $dbForInternal, $geodb, $audits, $events, $usage) use ($oauthDefaultSuccess) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $project */ @@ -365,6 +378,7 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') /** @var Utopia\Database\Database $dbForInternal */ /** @var MaxMind\Db\Reader $geodb */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $protocol = $request->getProtocol(); $callback = $protocol . '://' . $request->getHostname() . '/v1/account/sessions/oauth2/callback/' . $provider . '/' . $project->getId(); @@ -545,6 +559,11 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') $events->setParam('eventData', $response->output($session, Response::MODEL_SESSION)); + $usage + ->setParam('users.sessions.create', 1) + ->setParam('projectId', $project->getId()) + ->setParam('provider', $provider) + ; if (!Config::getParam('domainVerification')) { $response ->addHeader('X-Fallback-Cookies', \json_encode([Auth::$cookieName => Auth::encodeSession($user->getId(), $secret)])) @@ -595,7 +614,8 @@ App::post('/v1/account/sessions/anonymous') ->inject('dbForInternal') ->inject('geodb') ->inject('audits') - ->action(function ($request, $response, $locale, $user, $project, $dbForInternal, $geodb, $audits) { + ->inject('usage') + ->action(function ($request, $response, $locale, $user, $project, $dbForInternal, $geodb, $audits, $usage) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Locale\Locale $locale */ @@ -604,6 +624,7 @@ App::post('/v1/account/sessions/anonymous') /** @var Utopia\Database\Database $dbForInternal */ /** @var MaxMind\Db\Reader $geodb */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $protocol = $request->getProtocol(); @@ -686,6 +707,11 @@ App::post('/v1/account/sessions/anonymous') ->setParam('resource', 'users/' . $user->getId()) ; + $usage + ->setParam('users.sessions.create', 1) + ->setParam('provider', 'anonymous') + ; + if (!Config::getParam('domainVerification')) { $response ->addHeader('X-Fallback-Cookies', \json_encode([Auth::$cookieName => Auth::encodeSession($user->getId(), $secret)])) @@ -771,10 +797,15 @@ App::get('/v1/account') ->label('sdk.response.model', Response::MODEL_USER) ->inject('response') ->inject('user') - ->action(function ($response, $user) { + ->inject('usage') + ->action(function ($response, $user, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ + /** @var Appwrite\Stats\Stats $usage */ + $usage + ->setParam('users.read', 1) + ; $response->dynamic($user, Response::MODEL_USER); }); @@ -791,12 +822,17 @@ App::get('/v1/account/prefs') ->label('sdk.response.model', Response::MODEL_PREFERENCES) ->inject('response') ->inject('user') - ->action(function ($response, $user) { + ->inject('usage') + ->action(function ($response, $user, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ + /** @var Appwrite\Stats\Stats $usage */ $prefs = $user->getAttribute('prefs', new \stdClass()); + $usage + ->setParam('users.read', 1) + ; $response->dynamic(new Document($prefs), Response::MODEL_PREFERENCES); }); @@ -814,10 +850,12 @@ App::get('/v1/account/sessions') ->inject('response') ->inject('user') ->inject('locale') - ->action(function ($response, $user, $locale) { + ->inject('usage') + ->action(function ($response, $user, $locale, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ /** @var Utopia\Locale\Locale $locale */ + /** @var Appwrite\Stats\Stats $usage */ $sessions = $user->getAttribute('sessions', []); $current = Auth::sessionVerify($sessions, Auth::$secret); @@ -831,6 +869,9 @@ App::get('/v1/account/sessions') $sessions[$key] = $session; } + $usage + ->setParam('users.read', 1) + ; $response->dynamic(new Document([ 'sessions' => $sessions, 'sum' => count($sessions), @@ -853,13 +894,15 @@ App::get('/v1/account/logs') ->inject('locale') ->inject('geodb') ->inject('dbForInternal') - ->action(function ($response, $user, $locale, $geodb, $dbForInternal) { + ->inject('usage') + ->action(function ($response, $user, $locale, $geodb, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $project */ /** @var Utopia\Database\Document $user */ /** @var Utopia\Locale\Locale $locale */ /** @var MaxMind\Db\Reader $geodb */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $audit = new Audit($dbForInternal); @@ -906,6 +949,9 @@ App::get('/v1/account/logs') } + $usage + ->setParam('users.read', 1) + ; $response->dynamic(new Document(['logs' => $output]), Response::MODEL_LOG_LIST); }); @@ -925,11 +971,13 @@ App::get('/v1/account/sessions/:sessionId') ->inject('user') ->inject('locale') ->inject('dbForInternal') - ->action(function ($sessionId, $response, $user, $locale, $dbForInternal) { + ->inject('usage') + ->action(function ($sessionId, $response, $user, $locale, $dbForInternal, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ /** @var Utopia\Locale\Locale $locale */ /** @var Utopia\Database\Database $dbForInternal */ + /** @var Appwrite\Stats\Stats $usage */ $sessions = $user->getAttribute('sessions', []); $sessionId = ($sessionId === 'current') @@ -948,6 +996,10 @@ App::get('/v1/account/sessions/:sessionId') ->setAttribute('countryName', $countryName) ; + $usage + ->setParam('users.read', 1) + ; + return $response->dynamic($session, Response::MODEL_SESSION); } } @@ -972,11 +1024,13 @@ App::patch('/v1/account/name') ->inject('user') ->inject('dbForInternal') ->inject('audits') - ->action(function ($name, $response, $user, $dbForInternal, $audits) { + ->inject('usage') + ->action(function ($name, $response, $user, $dbForInternal, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->updateDocument('users', $user->getId(), $user->setAttribute('name', $name)); @@ -986,6 +1040,10 @@ App::patch('/v1/account/name') ->setParam('resource', 'users/' . $user->getId()) ; + $usage + ->setParam('users.update', 1) + ; + $response->dynamic($user, Response::MODEL_USER); }); @@ -1007,11 +1065,13 @@ App::patch('/v1/account/password') ->inject('user') ->inject('dbForInternal') ->inject('audits') - ->action(function ($password, $oldPassword, $response, $user, $dbForInternal, $audits) { + ->inject('usage') + ->action(function ($password, $oldPassword, $response, $user, $dbForInternal, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ // Check old password only if its an existing user. if ($user->getAttribute('passwordUpdate') !== 0 && !Auth::passwordVerify($oldPassword, $user->getAttribute('password'))) { // Double check user password @@ -1029,6 +1089,9 @@ App::patch('/v1/account/password') ->setParam('resource', 'users/' . $user->getId()) ; + $usage + ->setParam('users.update', 1) + ; $response->dynamic($user, Response::MODEL_USER); }); @@ -1050,11 +1113,13 @@ App::patch('/v1/account/email') ->inject('user') ->inject('dbForInternal') ->inject('audits') - ->action(function ($email, $password, $response, $user, $dbForInternal, $audits) { + ->inject('usage') + ->action(function ($email, $password, $response, $user, $dbForInternal, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $isAnonymousUser = is_null($user->getAttribute('email')) && is_null($user->getAttribute('password')); // Check if request is from an anonymous account for converting @@ -1084,6 +1149,9 @@ App::patch('/v1/account/email') ->setParam('resource', 'users/' . $user->getId()) ; + $usage + ->setParam('users.update', 1) + ; $response->dynamic($user, Response::MODEL_USER); }); @@ -1104,11 +1172,13 @@ App::patch('/v1/account/prefs') ->inject('user') ->inject('dbForInternal') ->inject('audits') - ->action(function ($prefs, $response, $user, $dbForInternal, $audits) { + ->inject('usage') + ->action(function ($prefs, $response, $user, $dbForInternal, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $user = $dbForInternal->updateDocument('users', $user->getId(), $user->setAttribute('prefs', $prefs)); @@ -1117,6 +1187,9 @@ App::patch('/v1/account/prefs') ->setParam('resource', 'users/' . $user->getId()) ; + $usage + ->setParam('users.update', 1) + ; $response->dynamic($user, Response::MODEL_USER); }); @@ -1137,13 +1210,15 @@ App::delete('/v1/account') ->inject('dbForInternal') ->inject('audits') ->inject('events') - ->action(function ($request, $response, $user, $dbForInternal, $audits, $events) { + ->inject('usage') + ->action(function ($request, $response, $user, $dbForInternal, $audits, $events, $usage) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $audits */ /** @var Appwrite\Event\Event $events */ + /** @var Appwrite\Stats\Stats $usage */ $protocol = $request->getProtocol(); $user = $dbForInternal->updateDocument('users', $user->getId(), $user->setAttribute('status', false)); @@ -1173,6 +1248,9 @@ App::delete('/v1/account') ; } + $usage + ->setParam('users.delete', 1) + ; $response ->addCookie(Auth::$cookieName . '_legacy', '', \time() - 3600, '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, null) ->addCookie(Auth::$cookieName, '', \time() - 3600, '/', Config::getParam('cookieDomain'), ('https' == $protocol), true, Config::getParam('cookieSamesite')) @@ -1200,7 +1278,8 @@ App::delete('/v1/account/sessions/:sessionId') ->inject('locale') ->inject('audits') ->inject('events') - ->action(function ($sessionId, $request, $response, $user, $dbForInternal, $locale, $audits, $events) { + ->inject('usage') + ->action(function ($sessionId, $request, $response, $user, $dbForInternal, $locale, $audits, $events, $usage) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ @@ -1208,6 +1287,7 @@ App::delete('/v1/account/sessions/:sessionId') /** @var Utopia\Locale\Locale $locale */ /** @var Appwrite\Event\Event $audits */ /** @var Appwrite\Event\Event $events */ + /** @var Appwrite\Stats\Stats $usage */ $protocol = $request->getProtocol(); $sessionId = ($sessionId === 'current') @@ -1254,6 +1334,10 @@ App::delete('/v1/account/sessions/:sessionId') ->setParam('eventData', $response->output($session, Response::MODEL_SESSION)) ; + $usage + ->setParam('users.sessions.delete', 1) + ->setParam('users.update', 1) + ; return $response->noContent(); } } @@ -1280,7 +1364,8 @@ App::delete('/v1/account/sessions') ->inject('locale') ->inject('audits') ->inject('events') - ->action(function ($request, $response, $user, $dbForInternal, $locale, $audits, $events) { + ->inject('usage') + ->action(function ($request, $response, $user, $dbForInternal, $locale, $audits, $events, $usage) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ @@ -1288,6 +1373,7 @@ App::delete('/v1/account/sessions') /** @var Utopia\Locale\Locale $locale */ /** @var Appwrite\Event\Event $audits */ /** @var Appwrite\Event\Event $events */ + /** @var Appwrite\Stats\Stats $usage */ $protocol = $request->getProtocol(); $sessions = $user->getAttribute('sessions', []); @@ -1330,6 +1416,10 @@ App::delete('/v1/account/sessions') ]), Response::MODEL_SESSION_LIST)) ; + $usage + ->setParam('users.sessions.delete', 1) + ->setParam('users.update', 1) + ; $response->noContent(); }); @@ -1357,7 +1447,8 @@ App::post('/v1/account/recovery') ->inject('mails') ->inject('audits') ->inject('events') - ->action(function ($email, $url, $request, $response, $dbForInternal, $project, $locale, $mails, $audits, $events) { + ->inject('usage') + ->action(function ($email, $url, $request, $response, $dbForInternal, $project, $locale, $mails, $audits, $events, $usage) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ @@ -1366,6 +1457,7 @@ App::post('/v1/account/recovery') /** @var Appwrite\Event\Event $mails */ /** @var Appwrite\Event\Event $audits */ /** @var Appwrite\Event\Event $events */ + /** @var Appwrite\Stats\Stats $usage */ $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::$roles); $isAppUser = Auth::isAppUser(Authorization::$roles); @@ -1433,6 +1525,9 @@ App::post('/v1/account/recovery') ->setParam('resource', 'users/' . $profile->getId()) ; + $usage + ->setParam('users.update', 1) + ; $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($recovery, Response::MODEL_TOKEN); }); @@ -1458,10 +1553,12 @@ App::put('/v1/account/recovery') ->inject('response') ->inject('dbForInternal') ->inject('audits') - ->action(function ($userId, $secret, $password, $passwordAgain, $response, $dbForInternal, $audits) { + ->inject('usage') + ->action(function ($userId, $secret, $password, $passwordAgain, $response, $dbForInternal, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ if ($password !== $passwordAgain) { throw new Exception('Passwords must match', 400); @@ -1508,6 +1605,9 @@ App::put('/v1/account/recovery') ->setParam('resource', 'users/' . $profile->getId()) ; + $usage + ->setParam('users.update', 1) + ; $response->dynamic($recovery, Response::MODEL_TOKEN); }); @@ -1535,7 +1635,8 @@ App::post('/v1/account/verification') ->inject('audits') ->inject('events') ->inject('mails') - ->action(function ($url, $request, $response, $project, $user, $dbForInternal, $locale, $audits, $events, $mails) { + ->inject('usage') + ->action(function ($url, $request, $response, $project, $user, $dbForInternal, $locale, $audits, $events, $mails, $usage) { /** @var Utopia\Swoole\Request $request */ /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $project */ @@ -1545,6 +1646,7 @@ App::post('/v1/account/verification') /** @var Appwrite\Event\Event $audits */ /** @var Appwrite\Event\Event $events */ /** @var Appwrite\Event\Event $mails */ + /** @var Appwrite\Stats\Stats $usage */ $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::$roles); $isAppUser = Auth::isAppUser(Authorization::$roles); @@ -1602,6 +1704,9 @@ App::post('/v1/account/verification') ->setParam('resource', 'users/' . $user->getId()) ; + $usage + ->setParam('users.update', 1) + ; $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($verification, Response::MODEL_TOKEN); }); @@ -1626,11 +1731,13 @@ App::put('/v1/account/verification') ->inject('user') ->inject('dbForInternal') ->inject('audits') - ->action(function ($userId, $secret, $response, $user, $dbForInternal, $audits) { + ->inject('usage') + ->action(function ($userId, $secret, $response, $user, $dbForInternal, $audits, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Document $user */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $audits */ + /** @var Appwrite\Stats\Stats $usage */ $profile = $dbForInternal->getDocument('users', $userId); @@ -1668,5 +1775,8 @@ App::put('/v1/account/verification') ->setParam('resource', 'users/' . $user->getId()) ; + $usage + ->setParam('users.update', 1) + ; $response->dynamic($verification, Response::MODEL_TOKEN); }); diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index eb3abd12cf..ad600ce480 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -573,7 +573,7 @@ App::delete('/v1/users/:userId') ->inject('dbForInternal') ->inject('events') ->inject('deletes') - ->inject('users') + ->inject('usage') ->action(function ($userId, $response, $dbForInternal, $events, $deletes, $usage) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 5230963098..fc8d0478c1 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -11,37 +11,8 @@ use Utopia\CLI\Console; use Utopia\Database\Adapter\MariaDB; use Utopia\Database\Database; use Utopia\Database\Document; -use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; -/** - * 1. Load all the projects - * 2. Load latest data entered entered for each project, for each period - * 3. Start the loop - * 4. Fore each project, for each metric, for each period - sync data - */ - -/** - * Only succefull operations - * - * database.collections.CRUD (project=x) - * database.documents.CRUD (project=x,collection=y) - * - * users.CRUD - * users.sessions.create (project=x,provider=y) - * users.sessions.delete (project=x,provider=y) - * - * storage.buckets.CRUD (project=x) - * storage.files.CRUD (project=x,bucket=y) - * - * refactor later - * - functions - * - realtime - * - teams - * - webhooks - * - keys - really later! - */ - $cli ->task('usage') ->desc('Schedules syncing data from influxdb to Appwrite console db') @@ -61,93 +32,91 @@ $cli ], ]; - //use projectId from influxdb instead of iterating over projects from DB - $globalMetrics = [ 'requests' => [ - 'method' => 'getGlobalMetrics', 'table' => 'appwrite_usage_requests_all', ], 'network' => [ - 'method' => 'getGlobalMetrics', 'table' => 'appwrite_usage_network_all', ], 'executions' => [ - 'method' => 'getGlobalMetrics', 'table' => 'appwrite_usage_executions_all', ], 'database.collections.create' => [ - 'method' => 'getDatabaseMetrics', 'table' => 'appwrite_usage_database_collections_create', ], 'database.collections.read' => [ - 'method' => 'getDatabaseMetrics', 'table' => 'appwrite_usage_database_collections_read', ], 'database.collections.update' => [ - 'method' => 'getDatabaseMetrics', 'table' => 'appwrite_usage_database_collections_update', ], 'database.collections.delete' => [ - 'method' => 'getDatabaseMetrics', 'table' => 'appwrite_usage_database_collections_delete', ], 'database.documents.create' => [ - 'method' => 'getDatabaseMetrics', 'table' => 'appwrite_usage_database_documents_create', ], 'database.documents.read' => [ - 'method' => 'getDatabaseMetrics', 'table' => 'appwrite_usage_database_documents_read', ], 'database.documents.update' => [ - 'method' => 'getDatabaseMetrics', 'table' => 'appwrite_usage_database_documents_update', ], 'database.documents.delete' => [ - 'method' => 'getDatabaseMetrics', 'table' => 'appwrite_usage_database_documents_delete', ], 'database.documents.collectionId.create' => [ - 'method' => 'getDatabaseMetrics', 'table' => 'appwrite_usage_database_documents_create', 'groupBy' => 'collectionId', ], 'database.documents.collectionId.read' => [ - 'method' => 'getDatabaseMetrics', 'table' => 'appwrite_usage_database_documents_read', 'groupBy' => 'collectionId', ], 'database.documents.collectionId.update' => [ - 'method' => 'getDatabaseMetrics', 'table' => 'appwrite_usage_database_documents_update', 'groupBy' => 'collectionId', ], 'database.documents.collectionId.delete' => [ - 'method' => 'getDatabaseMetrics', 'table' => 'appwrite_usage_database_documents_delete', 'groupBy' => 'collectionId', ], 'storage.buckets.bucketId.files.create' => [ - 'method' => 'getDatabaseMetrics', 'table' => 'appwrite_usage_storage_files_create', 'groupBy' => 'bucketId', ], 'storage.buckets.bucketId.files.read' => [ - 'method' => 'getDatabaseMetrics', 'table' => 'appwrite_usage_storage_files_read', 'groupBy' => 'bucketId', ], 'storage.buckets.bucketId.files.update' => [ - 'method' => 'getDatabaseMetrics', 'table' => 'appwrite_usage_storage_files_update', 'groupBy' => 'bucketId', ], 'storage.buckets.bucketId.files.delete' => [ - 'method' => 'getDatabaseMetrics', 'table' => 'appwrite_usage_storage_files_delete', 'groupBy' => 'bucketId', ], + 'users.create' => [ + 'table' => 'appwrite_usage_users_create', + ], + 'users.read' => [ + 'table' => 'appwrite_usage_users_read', + ], + 'users.update' => [ + 'table' => 'appwrite_usage_users_update', + ], + 'users.delete' => [ + 'table' => 'appwrite_usage_users_delete', + ], + 'users.sessions.create' => [ + 'table' => 'appwrite_usage_users_sessions_create', + 'groupBy' => 'provider', + ], + 'users.sessions.delete' => [ + 'table' => 'appwrite_usage_users_sessions_delete', + ], ]; $attempts = 0; @@ -169,18 +138,16 @@ $cli } while ($attempts < $max); $cacheAdapter = new Cache(new Redis($redis)); - $dbForConsole = new Database(new MariaDB($db), $cacheAdapter); - $dbForConsole->setNamespace('project_console_internal'); $dbForProject = new Database(new MariaDB($db), $cacheAdapter); Authorization::disable(); - $latestData = []; - $firstRun = true; - Console::loop(function () use ($interval, $register, &$latestData, $dbForProject, $dbForConsole, &$firstRun, $globalMetrics, $periods) { + Console::loop(function () use ($interval, $register, $dbForProject, $globalMetrics, $periods) { $time = date('d-m-Y H:i:s', time()); Console::info("[{$time}] Aggregating usage data every {$interval} seconds"); + $loopStart = microtime(true); + $client = $register->get('influxdb'); if ($client) { $database = $client->selectDB('telegraf'); @@ -225,7 +192,6 @@ $cli $dbForProject->updateDocument('stats', $document->getId(), $document->setAttribute('value', $value)); } - $latestData[$projectId][$metric][$period['key']] = $time; } catch (\Exception$e) { Console::warning("Failed to save data for project {$projectId} and metric {$metric}"); } @@ -234,6 +200,9 @@ $cli } } } - $firstRun = false; + + $loopTook = microtime(true) - $loopStart; + $time = date('d-m-Y H:i:s', time()); + Console::info("[{$time}] Aggregation took {$loopTook} seconds"); }, $interval); }); diff --git a/src/Appwrite/Stats/Stats.php b/src/Appwrite/Stats/Stats.php index 5c24e1ae94..f56f6a4e7e 100644 --- a/src/Appwrite/Stats/Stats.php +++ b/src/Appwrite/Stats/Stats.php @@ -126,8 +126,8 @@ class Stats foreach ($dbMetrics as $metric) { $value = $this->params[$metric] ?? 0; if ($value >= 1) { - $dbTags = ",projectId={$projectId},collectionId=" . ($this->params['collectionId'] ?? ''); - $this->statsd->increment($metric . $dbTags); + $tags = ",projectId={$projectId},collectionId=" . ($this->params['collectionId'] ?? ''); + $this->statsd->increment($metric . $tags); } } @@ -141,8 +141,36 @@ class Stats foreach ($storageMertics as $metric) { $value = $this->params[$metric] ?? 0; if ($value >= 1) { - $storageTags = ",projectId={$projectId},bucketId=" . ($this->params['bucketId'] ?? ''); - $this->statsd->increment($metric . $storageTags); + $tags = ",projectId={$projectId},bucketId=" . ($this->params['bucketId'] ?? ''); + $this->statsd->increment($metric . $tags); + } + } + + $usersMetrics = [ + 'users.create', + 'users.read', + 'users.update', + 'users.delete', + ]; + + foreach ($usersMetrics as $metric) { + $value = $this->params[$metric] ?? 0; + if ($value >= 1) { + $tags = ",projectId={$projectId}"; + $this->statsd->increment($metric . $tags); + } + } + + $sessionsMetrics = [ + 'users.sessions.create', + 'users.sessions.delete', + ]; + + foreach ($sessionsMetrics as $metric) { + $value = $this->params[$metric] ?? 0; + if ($value >= 1) { + $tags = ",projectId={$projectId},provider=". ($this->params['provider'] ?? ''); + $this->statsd->increment($metric . $tags); } } From 22f0aaf161f270adff600009a07c66bc347ec674 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 16 Aug 2021 14:47:35 +0545 Subject: [PATCH 029/206] improvement with latest data --- app/tasks/usage.php | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index fc8d0478c1..e5b8ac509c 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -140,11 +140,13 @@ $cli $cacheAdapter = new Cache(new Redis($redis)); $dbForProject = new Database(new MariaDB($db), $cacheAdapter); + $latestTime = []; + Authorization::disable(); - Console::loop(function () use ($interval, $register, $dbForProject, $globalMetrics, $periods) { - $time = date('d-m-Y H:i:s', time()); - Console::info("[{$time}] Aggregating usage data every {$interval} seconds"); + Console::loop(function () use ($interval, $register, $dbForProject, $globalMetrics, $periods, &$latestTime) { + $now = date('d-m-Y H:i:s', time()); + Console::info("[{$now}] Aggregating usage data every {$interval} seconds"); $loopStart = microtime(true); @@ -155,6 +157,9 @@ $cli foreach ($globalMetrics as $metric => $options) { foreach ($periods as $period) { $start = DateTime::createFromFormat('U', \strtotime($period['startTime']))->format(DateTime::RFC3339); + if(!empty($latestTime[$metric][$period['key']])) { + $start = DateTime::createFromFormat('U', $latestTime[$metric][$period['key']])->format(DateTime::RFC3339); + } $end = DateTime::createFromFormat('U', \strtotime('now'))->format(DateTime::RFC3339); $table = $options['table']; @@ -192,6 +197,7 @@ $cli $dbForProject->updateDocument('stats', $document->getId(), $document->setAttribute('value', $value)); } + $latestTime[$metric][$period['key']] = $time; } catch (\Exception$e) { Console::warning("Failed to save data for project {$projectId} and metric {$metric}"); } @@ -202,7 +208,7 @@ $cli } $loopTook = microtime(true) - $loopStart; - $time = date('d-m-Y H:i:s', time()); - Console::info("[{$time}] Aggregation took {$loopTook} seconds"); + $now = date('d-m-Y H:i:s', time()); + Console::info("[{$now}] Aggregation took {$loopTook} seconds"); }, $interval); }); From 13a91907b436f43d3532cbccb041c7f1bce8ceb5 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 16 Aug 2021 18:07:54 +0545 Subject: [PATCH 030/206] fix db metric name --- app/tasks/usage.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index e5b8ac509c..f8cf70070d 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -66,19 +66,19 @@ $cli 'database.documents.delete' => [ 'table' => 'appwrite_usage_database_documents_delete', ], - 'database.documents.collectionId.create' => [ + 'database.collections.collectionId.documents.create' => [ 'table' => 'appwrite_usage_database_documents_create', 'groupBy' => 'collectionId', ], - 'database.documents.collectionId.read' => [ + 'database.collections.collectionId.documents.read' => [ 'table' => 'appwrite_usage_database_documents_read', 'groupBy' => 'collectionId', ], - 'database.documents.collectionId.update' => [ + 'database.collections.collectionId.documents.update' => [ 'table' => 'appwrite_usage_database_documents_update', 'groupBy' => 'collectionId', ], - 'database.documents.collectionId.delete' => [ + 'database.collections.collectionId.documents.delete' => [ 'table' => 'appwrite_usage_database_documents_delete', 'groupBy' => 'collectionId', ], From 3e0cc30bddc622e8eb980f4e327f504c6b9278b0 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 16 Aug 2021 18:16:49 +0545 Subject: [PATCH 031/206] some comments and small fixes --- app/controllers/api/account.php | 2 +- app/tasks/usage.php | 21 +++++++++++---------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index b2baa4fcbe..0f357e045c 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -562,7 +562,7 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') $usage ->setParam('users.sessions.create', 1) ->setParam('projectId', $project->getId()) - ->setParam('provider', $provider) + ->setParam('provider', 'oauth2-'.$provider) ; if (!Config::getParam('domainVerification')) { $response diff --git a/app/tasks/usage.php b/app/tasks/usage.php index f8cf70070d..7c821b2306 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -17,8 +17,8 @@ $cli ->task('usage') ->desc('Schedules syncing data from influxdb to Appwrite console db') ->action(function () use ($register) { - Console::title('Usage Sync V1'); - Console::success(APP_NAME . ' usage sync process v1 has started'); + Console::title('Usage Aggregation V1'); + Console::success(APP_NAME . ' usage aggregation process v1 has started'); $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); //30 seconds $periods = [ @@ -32,6 +32,7 @@ $cli ], ]; + // all the metrics that we are collecting at the moment $globalMetrics = [ 'requests' => [ 'table' => 'appwrite_usage_requests_all', @@ -154,19 +155,18 @@ $cli if ($client) { $database = $client->selectDB('telegraf'); // sync data - foreach ($globalMetrics as $metric => $options) { - foreach ($periods as $period) { + foreach ($globalMetrics as $metric => $options) { //for each metrics + foreach ($periods as $period) { // aggregate data for each period $start = DateTime::createFromFormat('U', \strtotime($period['startTime']))->format(DateTime::RFC3339); if(!empty($latestTime[$metric][$period['key']])) { $start = DateTime::createFromFormat('U', $latestTime[$metric][$period['key']])->format(DateTime::RFC3339); } $end = DateTime::createFromFormat('U', \strtotime('now'))->format(DateTime::RFC3339); - $table = $options['table']; - $groupBy = $options['groupBy'] ?? ''; + $table = $options['table']; //which influxdb table to query for this metric + $groupBy = empty($options['groupBy']) ? '' : ', "' . $options['groupBy'] . '"'; //some sub level metrics may be grouped by other tags like collectionId, bucketId, etc - $query = 'SELECT sum(value) AS "value" FROM "' . $table . '" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' GROUP BY time(' . $period['key'] . '), "projectId"' . (empty($groupBy) ? '' : ', "' . $groupBy . '"') . ' FILL(null)'; - $result = $database->query($query); + $result = $database->query('SELECT sum(value) AS "value" FROM "' . $table . '" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' GROUP BY time(' . $period['key'] . '), "projectId"' . $groupBy . ' FILL(null)'); $points = $result->getPoints(); foreach ($points as $point) { $projectId = $point['projectId']; @@ -180,7 +180,7 @@ $cli $metric = str_replace($groupBy, $groupedBy, $metric); } $time = \strtotime($point['time']); - $id = \md5($time . '_' . $period['key'] . '_' . $metric); + $id = \md5($time . '_' . $period['key'] . '_' . $metric); //construct unique id for each metric using time, period and metric $value = (!empty($point['value'])) ? $point['value'] : 0; try { $document = $dbForProject->getDocument('stats', $id); @@ -198,7 +198,8 @@ $cli $document->setAttribute('value', $value)); } $latestTime[$metric][$period['key']] = $time; - } catch (\Exception$e) { + } catch (\Exception $e) { + // if projects are deleted this might fail Console::warning("Failed to save data for project {$projectId} and metric {$metric}"); } } From 62e422b76ef1a58f59bc26b707f99827208a5736 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 16 Aug 2021 19:12:15 +0545 Subject: [PATCH 032/206] project usage from appwrite db (wip) --- app/controllers/api/projects.php | 301 ++++++++++++++++++------------- 1 file changed, 178 insertions(+), 123 deletions(-) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index c302e882b3..950a4df0c5 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -5,13 +5,15 @@ use Appwrite\Database\Validator\CustomId; use Appwrite\Network\Validator\CNAME; use Appwrite\Network\Validator\Domain as DomainValidator; use Appwrite\Network\Validator\URL; -use Appwrite\Task\Validator\Cron; use Appwrite\Utopia\Response; -use Cron\CronExpression; +use Utopia\Abuse\Adapters\TimeLimit; use Utopia\App; +use Utopia\Audit\Audit; use Utopia\Config\Config; +use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Query; +use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\UID; use Utopia\Domains\Domain; use Utopia\Exception; @@ -21,13 +23,11 @@ use Utopia\Validator\Integer; use Utopia\Validator\Range; use Utopia\Validator\Text; use Utopia\Validator\WhiteList; -use Utopia\Audit\Audit; -use Utopia\Abuse\Adapters\TimeLimit; App::init(function ($project) { /** @var Utopia\Database\Document $project */ - if($project->getId() !== 'console') { + if ($project->getId() !== 'console') { throw new Exception('Access to this API is forbidden.', 401); } }, ['project'], 'projects'); @@ -71,7 +71,7 @@ App::post('/v1/projects') if ($team->isEmpty()) { throw new Exception('Team not found', 404); } - + $auth = Config::getParam('auth', []); $auths = ['limit' => 0]; foreach ($auth as $index => $method) { @@ -228,12 +228,12 @@ App::get('/v1/projects/:projectId/usage') ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) ->inject('response') ->inject('dbForConsole') - ->inject('projectDB') + ->inject('dbForInternal') ->inject('register') - ->action(function ($projectId, $range, $response, $dbForConsole, $projectDB, $register) { + ->action(function ($projectId, $range, $response, $dbForConsole, $dbForInternal, $register) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForConsole */ - /** @var Appwrite\Database\Database $projectDB */ + /** @var Utopia\Database\Database $dbForInternal */ /** @var Utopia\Registry\Registry $register */ $project = $dbForConsole->getDocument('projects', $projectId); @@ -246,71 +246,126 @@ App::get('/v1/projects/:projectId/usage') $period = [ '24h' => [ - 'start' => DateTime::createFromFormat('U', \strtotime('-24 hours')), - 'end' => DateTime::createFromFormat('U', \strtotime('+1 hour')), - 'group' => '30m', + // 'start' => DateTime::createFromFormat('U', \strtotime('-24 hours')), + // 'end' => DateTime::createFromFormat('U', \strtotime('+1 hour')), + 'period' => '30m', + 'limit' => 48, ], '7d' => [ - 'start' => DateTime::createFromFormat('U', \strtotime('-7 days')), - 'end' => DateTime::createFromFormat('U', \strtotime('now')), - 'group' => '1d', + // 'start' => DateTime::createFromFormat('U', \strtotime('-7 days')), + // 'end' => DateTime::createFromFormat('U', \strtotime('now')), + 'period' => '1d', + 'limit' => 7, ], '30d' => [ - 'start' => DateTime::createFromFormat('U', \strtotime('-30 days')), - 'end' => DateTime::createFromFormat('U', \strtotime('now')), - 'group' => '1d', + // 'start' => DateTime::createFromFormat('U', \strtotime('-30 days')), + // 'end' => DateTime::createFromFormat('U', \strtotime('now')), + 'period' => '1d', + 'limit' => 30, ], '90d' => [ - 'start' => DateTime::createFromFormat('U', \strtotime('-90 days')), - 'end' => DateTime::createFromFormat('U', \strtotime('now')), - 'group' => '1d', + // 'start' => DateTime::createFromFormat('U', \strtotime('-90 days')), + // 'end' => DateTime::createFromFormat('U', \strtotime('now')), + 'period' => '1d', + 'limit' => 90, ], ]; - $client = $register->get('influxdb'); + $dbForInternal->setNamespace('project_' . $projectId . '_internal'); + + $requestDocs = Authorization::skip(function () use ($dbForInternal, $period, $range) { + return $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, ['requests']), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + }); $requests = []; - $network = []; - $functions = []; - - if ($client) { - $start = $period[$range]['start']->format(DateTime::RFC3339); - $end = $period[$range]['end']->format(DateTime::RFC3339); - $database = $client->selectDB('telegraf'); - - // Requests - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_requests_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "projectId"=\'' . $project->getId() . '\' GROUP BY time(' . $period[$range]['group'] . ') FILL(null)'); - $points = $result->getPoints(); - - foreach ($points as $point) { - $requests[] = [ - 'value' => (!empty($point['value'])) ? $point['value'] : 0, - 'date' => \strtotime($point['time']), - ]; - } - - // Network - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_network_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "projectId"=\'' . $project->getId() . '\' GROUP BY time(' . $period[$range]['group'] . ') FILL(null)'); - $points = $result->getPoints(); - - foreach ($points as $point) { - $network[] = [ - 'value' => (!empty($point['value'])) ? $point['value'] : 0, - 'date' => \strtotime($point['time']), - ]; - } - - // Functions - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_executions_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "projectId"=\'' . $project->getId() . '\' GROUP BY time(' . $period[$range]['group'] . ') FILL(null)'); - $points = $result->getPoints(); - - foreach ($points as $point) { - $functions[] = [ - 'value' => (!empty($point['value'])) ? $point['value'] : 0, - 'date' => \strtotime($point['time']), - ]; - } + foreach ($requestDocs as $requestDoc) { + $requests[] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; } + + $requests = array_reverse($requests); + + $networkDocs = Authorization::skip(function () use ($dbForInternal, $period, $range) { + return $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, ['network']), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + }); + + $network = []; + foreach ($networkDocs as $networkDoc) { + $network[] = [ + 'value' => $networkDoc->getAttribute('value'), + 'date' => $networkDoc->getAttribute('time'), + ]; + } + + $network = array_reverse($network); + + $functionsDocs = Authorization::skip(function () use ($dbForInternal, $period, $range) { + return $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, ['executions']), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + }); + + $functions = []; + foreach ($functionsDocs as $functionDoc) { + $functions[] = [ + 'value' => $functionDoc->getAttribute('value'), + 'date' => $functionDoc->getAttribute('time'), + ]; + } + $functions = array_reverse($functions); + + // $requests = []; + // $network = []; + // $functions = []; + + // $client = $register->get('influxdb'); + // if ($client) { + // $start = $period[$range]['start']->format(DateTime::RFC3339); + // $end = $period[$range]['end']->format(DateTime::RFC3339); + // $database = $client->selectDB('telegraf'); + + // // Requests + // $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_requests_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "projectId"=\'' . $project->getId() . '\' GROUP BY time(' . $period[$range]['group'] . ') FILL(null)'); + // $points = $result->getPoints(); + + // foreach ($points as $point) { + // $requests[] = [ + // 'value' => (!empty($point['value'])) ? $point['value'] : 0, + // 'date' => \strtotime($point['time']), + // ]; + // } + + // // Network + // $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_network_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "projectId"=\'' . $project->getId() . '\' GROUP BY time(' . $period[$range]['group'] . ') FILL(null)'); + // $points = $result->getPoints(); + + // foreach ($points as $point) { + // $network[] = [ + // 'value' => (!empty($point['value'])) ? $point['value'] : 0, + // 'date' => \strtotime($point['time']), + // ]; + // } + + // // Functions + // $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_executions_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "projectId"=\'' . $project->getId() . '\' GROUP BY time(' . $period[$range]['group'] . ') FILL(null)'); + // $points = $result->getPoints(); + + // foreach ($points as $point) { + // $functions[] = [ + // 'value' => (!empty($point['value'])) ? $point['value'] : 0, + // 'date' => \strtotime($point['time']), + // ]; + // } + // } } else { $requests = []; $network = []; @@ -319,41 +374,41 @@ App::get('/v1/projects/:projectId/usage') // Users - $projectDB->getCollection([ - 'limit' => 0, - 'offset' => 0, - 'filters' => [ - '$collection=users', - ], - ]); + // $projectDB->getCollection([ + // 'limit' => 0, + // 'offset' => 0, + // 'filters' => [ + // '$collection=users', + // ], + // ]); - $usersTotal = $projectDB->getSum(); + // $usersTotal = $projectDB->getSum(); - // Documents + // // Documents - $collections = $projectDB->getCollection([ - 'limit' => 100, - 'offset' => 0, - 'filters' => [ - '$collection=collections', - ], - ]); + // $collections = $projectDB->getCollection([ + // 'limit' => 100, + // 'offset' => 0, + // 'filters' => [ + // '$collection=collections', + // ], + // ]); - $collectionsTotal = $projectDB->getSum(); + // $collectionsTotal = $projectDB->getSum(); - $documents = []; + // $documents = []; - foreach ($collections as $collection) { - $result = $projectDB->getCollection([ - 'limit' => 0, - 'offset' => 0, - 'filters' => [ - '$collection=' . $collection['$id'], - ], - ]); + // foreach ($collections as $collection) { + // $result = $projectDB->getCollection([ + // 'limit' => 0, + // 'offset' => 0, + // 'filters' => [ + // '$collection=' . $collection['$id'], + // ], + // ]); - $documents[] = ['name' => $collection['name'], 'total' => $projectDB->getSum()]; - } + // $documents[] = ['name' => $collection['name'], 'total' => $projectDB->getSum()]; + // } $response->json([ 'range' => $range, @@ -375,38 +430,38 @@ App::get('/v1/projects/:projectId/usage') return $item['value']; }, $functions)), ], - 'collections' => [ - 'data' => $collections, - 'total' => $collectionsTotal, - ], - 'documents' => [ - 'data' => $documents, - 'total' => \array_sum(\array_map(function ($item) { - return $item['total']; - }, $documents)), - ], - 'users' => [ - 'data' => [], - 'total' => $usersTotal, - ], - 'storage' => [ - 'total' => $projectDB->getCount( - [ - 'attribute' => 'sizeOriginal', - 'filters' => [ - '$collection=files', - ], - ] - ) + - $projectDB->getCount( - [ - 'attribute' => 'size', - 'filters' => [ - '$collection=tags', - ], - ] - ), - ], + // 'collections' => [ + // 'data' => $collections, + // 'total' => $collectionsTotal, + // ], + // 'documents' => [ + // 'data' => $documents, + // 'total' => \array_sum(\array_map(function ($item) { + // return $item['total']; + // }, $documents)), + // ], + // 'users' => [ + // 'data' => [], + // 'total' => $usersTotal, + // ], + // 'storage' => [ + // 'total' => $projectDB->getCount( + // [ + // 'attribute' => 'sizeOriginal', + // 'filters' => [ + // '$collection=files', + // ], + // ] + // ) + + // $projectDB->getCount( + // [ + // 'attribute' => 'size', + // 'filters' => [ + // '$collection=tags', + // ], + // ] + // ), + // ], ]); }); @@ -470,7 +525,7 @@ App::patch('/v1/projects/:projectId/service') ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_PROJECT) ->param('projectId', '', new UID(), 'Project unique ID.') - ->param('service', '', new WhiteList(array_keys(array_filter(Config::getParam('services'), function($element) {return $element['optional'];})), true), 'Service name.') + ->param('service', '', new WhiteList(array_keys(array_filter(Config::getParam('services'), function ($element) {return $element['optional'];})), true), 'Service name.') ->param('status', null, new Boolean(), 'Service status.') ->inject('response') ->inject('dbForConsole') From 354633afa8275392ed7e57963b484b1388a21e09 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 17 Aug 2021 10:47:31 +0545 Subject: [PATCH 033/206] fix usage stats --- src/Appwrite/Stats/Stats.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Appwrite/Stats/Stats.php b/src/Appwrite/Stats/Stats.php index f56f6a4e7e..2d5a5cb16d 100644 --- a/src/Appwrite/Stats/Stats.php +++ b/src/Appwrite/Stats/Stats.php @@ -175,6 +175,7 @@ class Stats } if ($storage >= 1) { + $tags = ",projectId={$projectId},bucketId={($this->params['bucketId'] ?? '')}"; $this->statsd->count('storage.all' . $tags, $storage); } From 0d73ac953dc15500a57968e14f52a2a0a4851d9d Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 17 Aug 2021 11:08:43 +0545 Subject: [PATCH 034/206] count total number of sessions deleted --- app/controllers/api/account.php | 6 ++++-- src/Appwrite/Stats/Stats.php | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 0f357e045c..9674d668e4 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -1409,15 +1409,17 @@ App::delete('/v1/account/sessions') $dbForInternal->updateDocument('users', $user->getId(), $user->setAttribute('sessions', [])); + $numOfSessions = count($sessions); + $events ->setParam('eventData', $response->output(new Document([ 'sessions' => $sessions, - 'sum' => count($sessions), + 'sum' => $numOfSessions, ]), Response::MODEL_SESSION_LIST)) ; $usage - ->setParam('users.sessions.delete', 1) + ->setParam('users.sessions.delete', $numOfSessions) ->setParam('users.update', 1) ; $response->noContent(); diff --git a/src/Appwrite/Stats/Stats.php b/src/Appwrite/Stats/Stats.php index 2d5a5cb16d..e7720088f5 100644 --- a/src/Appwrite/Stats/Stats.php +++ b/src/Appwrite/Stats/Stats.php @@ -170,7 +170,7 @@ class Stats $value = $this->params[$metric] ?? 0; if ($value >= 1) { $tags = ",projectId={$projectId},provider=". ($this->params['provider'] ?? ''); - $this->statsd->increment($metric . $tags); + $this->statsd->count($metric . $tags, $value); } } From 583c447b4e345c3b76b04ee5a0eb5006648bd85c Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 17 Aug 2021 11:30:07 +0545 Subject: [PATCH 035/206] database object count aggregation --- app/tasks/usage.php | 47 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 7c821b2306..34e7dcb2d6 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -11,6 +11,7 @@ use Utopia\CLI\Console; use Utopia\Database\Adapter\MariaDB; use Utopia\Database\Database; use Utopia\Database\Document; +use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; $cli @@ -158,7 +159,7 @@ $cli foreach ($globalMetrics as $metric => $options) { //for each metrics foreach ($periods as $period) { // aggregate data for each period $start = DateTime::createFromFormat('U', \strtotime($period['startTime']))->format(DateTime::RFC3339); - if(!empty($latestTime[$metric][$period['key']])) { + if (!empty($latestTime[$metric][$period['key']])) { $start = DateTime::createFromFormat('U', $latestTime[$metric][$period['key']])->format(DateTime::RFC3339); } $end = DateTime::createFromFormat('U', \strtotime('now'))->format(DateTime::RFC3339); @@ -198,7 +199,7 @@ $cli $document->setAttribute('value', $value)); } $latestTime[$metric][$period['key']] = $time; - } catch (\Exception $e) { + } catch (\Exception$e) { // if projects are deleted this might fail Console::warning("Failed to save data for project {$projectId} and metric {$metric}"); } @@ -212,4 +213,46 @@ $cli $now = date('d-m-Y H:i:s', time()); Console::info("[{$now}] Aggregation took {$loopTook} seconds"); }, $interval); + + //aggregate number of objects in database + $dbForConsole = new Database(new MariaDB($db), $cacheAdapter); + $dbForConsole->setNamespace('project_console_internal'); + + $dbCountInterval = 15*60; //aggregate data every 15 minutes + Console::loop(function () use ($dbForConsole, $dbForProject, $dbCountInterval) { + $now = date('d-m-Y H:i:s', time()); + Console::info("[{$now}] Aggregating Database object count every {$dbCountInterval} seconds"); + $loopStart = microtime(true); + + $latestProject = null; + do { + $projects = $dbForConsole->find('projects', [], 100, orderAfter:$latestProject); + if (!empty($projects)) { + $latestProject = $projects[array_key_last($projects)]; + + foreach ($projects as $project) { + $id = $project->getId(); + $dbForProject->setNamespace("project_{$id}_internal"); + $collections = ['users', 'collections', 'files']; + foreach ($collections as $collection) { + $count = $dbForProject->count($collection); + $dbForProject->createDocument('stats', new Document([ + '$id' => $dbForProject->getId(), + 'time' => time(), + 'period' => '15m', + 'metric' => "{$collection}.count", + 'value' => $count, + 'type' => 1, + ])); + } + } + } + + } while (!empty($projects)); + + $loopTook = microtime(true) - $loopStart; + $now = date('d-m-Y H:i:s', time()); + Console::info("[{$now}] DB objects count aggregation took {$loopTook} seconds"); + + }, $dbCountInterval); }); From eeee0f712bcd4080de016cacead2318c0029ea82 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 17 Aug 2021 11:48:27 +0545 Subject: [PATCH 036/206] refactor db object counts --- app/tasks/usage.php | 85 ++++++++++++++++++++++----------------------- 1 file changed, 41 insertions(+), 44 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 34e7dcb2d6..afe5958406 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -141,12 +141,15 @@ $cli $cacheAdapter = new Cache(new Redis($redis)); $dbForProject = new Database(new MariaDB($db), $cacheAdapter); + $dbForConsole = new Database(new MariaDB($db), $cacheAdapter); + $dbForConsole->setNamespace('project_console_internal'); $latestTime = []; Authorization::disable(); - Console::loop(function () use ($interval, $register, $dbForProject, $globalMetrics, $periods, &$latestTime) { + $iterations = 0; + Console::loop(function () use ($interval, $register, $dbForProject, $dbForConsole, $globalMetrics, $periods, &$latestTime, &$iterations) { $now = date('d-m-Y H:i:s', time()); Console::info("[{$now}] Aggregating usage data every {$interval} seconds"); @@ -174,7 +177,7 @@ $cli if (!empty($projectId) && $projectId != 'console') { $dbForProject->setNamespace('project_' . $projectId . '_internal'); if (!empty($groupBy)) { - $groupedBy = $point[$groupBy]; + $groupedBy = $point[$groupBy] ?? ''; if (empty($groupedBy)) { continue; } @@ -209,50 +212,44 @@ $cli } } + if ($iterations % 30 == 0) { + //aggregate number of objects in database + $latestProject = null; + do { + $projects = $dbForConsole->find('projects', [], 100, orderAfter:$latestProject); + if (!empty($projects)) { + $latestProject = $projects[array_key_last($projects)]; + + foreach ($projects as $project) { + $id = $project->getId(); + $collections = ['users' => [ + 'namespace' => 'internal', + ], 'collections' => [ + 'namespace' => 'external', + ], 'files' => [ + 'namespace' => 'internal', + ]]; + foreach ($collections as $collection => $options) { + $dbForProject->setNamespace("project_{$id}_{$options['namespace']}"); + $count = $dbForProject->count($collection); + $dbForProject->setNamespace("project_{$id}_internal"); + $dbForProject->createDocument('stats', new Document([ + '$id' => $dbForProject->getId(), + 'time' => time(), + 'period' => '15m', + 'metric' => "{$collection}.count", + 'value' => $count, + 'type' => 1, + ])); + } + } + } + + } while (!empty($projects)); + } + $iterations++; $loopTook = microtime(true) - $loopStart; $now = date('d-m-Y H:i:s', time()); Console::info("[{$now}] Aggregation took {$loopTook} seconds"); }, $interval); - - //aggregate number of objects in database - $dbForConsole = new Database(new MariaDB($db), $cacheAdapter); - $dbForConsole->setNamespace('project_console_internal'); - - $dbCountInterval = 15*60; //aggregate data every 15 minutes - Console::loop(function () use ($dbForConsole, $dbForProject, $dbCountInterval) { - $now = date('d-m-Y H:i:s', time()); - Console::info("[{$now}] Aggregating Database object count every {$dbCountInterval} seconds"); - $loopStart = microtime(true); - - $latestProject = null; - do { - $projects = $dbForConsole->find('projects', [], 100, orderAfter:$latestProject); - if (!empty($projects)) { - $latestProject = $projects[array_key_last($projects)]; - - foreach ($projects as $project) { - $id = $project->getId(); - $dbForProject->setNamespace("project_{$id}_internal"); - $collections = ['users', 'collections', 'files']; - foreach ($collections as $collection) { - $count = $dbForProject->count($collection); - $dbForProject->createDocument('stats', new Document([ - '$id' => $dbForProject->getId(), - 'time' => time(), - 'period' => '15m', - 'metric' => "{$collection}.count", - 'value' => $count, - 'type' => 1, - ])); - } - } - } - - } while (!empty($projects)); - - $loopTook = microtime(true) - $loopStart; - $now = date('d-m-Y H:i:s', time()); - Console::info("[{$now}] DB objects count aggregation took {$loopTook} seconds"); - - }, $dbCountInterval); }); From ee048caf93a17cba2f88bce560e0e9e0867de653 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 17 Aug 2021 12:18:32 +0545 Subject: [PATCH 037/206] fix usage small issues --- app/controllers/api/storage.php | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 651985057a..7daeee1546 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -631,16 +631,13 @@ App::delete('/v1/storage/files/:fileId') $usage ->setParam('storage', $file->getAttribute('size', 0) * -1) + ->setParam('storage.files.delete', 1) + ->setParam('bucketId', 'default') ; $events ->setParam('eventData', $response->output($file, Response::MODEL_FILE)) ; - $usage - ->setParam('storage.files.delete', 1) - ->setParam('bucketId', 'default') - ; - $response->noContent(); }); \ No newline at end of file From 4bb911867c1fe28542b4194ac500e926828d096b Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 17 Aug 2021 12:18:51 +0545 Subject: [PATCH 038/206] more stats from db --- app/controllers/api/projects.php | 100 ++++++++----------------------- 1 file changed, 26 insertions(+), 74 deletions(-) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 950a4df0c5..cf337a83e2 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -322,79 +322,22 @@ App::get('/v1/projects/:projectId/usage') ]; } $functions = array_reverse($functions); - - // $requests = []; - // $network = []; - // $functions = []; - - // $client = $register->get('influxdb'); - // if ($client) { - // $start = $period[$range]['start']->format(DateTime::RFC3339); - // $end = $period[$range]['end']->format(DateTime::RFC3339); - // $database = $client->selectDB('telegraf'); - - // // Requests - // $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_requests_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "projectId"=\'' . $project->getId() . '\' GROUP BY time(' . $period[$range]['group'] . ') FILL(null)'); - // $points = $result->getPoints(); - - // foreach ($points as $point) { - // $requests[] = [ - // 'value' => (!empty($point['value'])) ? $point['value'] : 0, - // 'date' => \strtotime($point['time']), - // ]; - // } - - // // Network - // $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_network_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "projectId"=\'' . $project->getId() . '\' GROUP BY time(' . $period[$range]['group'] . ') FILL(null)'); - // $points = $result->getPoints(); - - // foreach ($points as $point) { - // $network[] = [ - // 'value' => (!empty($point['value'])) ? $point['value'] : 0, - // 'date' => \strtotime($point['time']), - // ]; - // } - - // // Functions - // $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_executions_all" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' AND "projectId"=\'' . $project->getId() . '\' GROUP BY time(' . $period[$range]['group'] . ') FILL(null)'); - // $points = $result->getPoints(); - - // foreach ($points as $point) { - // $functions[] = [ - // 'value' => (!empty($point['value'])) ? $point['value'] : 0, - // 'date' => \strtotime($point['time']), - // ]; - // } - // } } else { $requests = []; $network = []; $functions = []; } - // Users + $usersCount = Authorization::skip(function () use ($dbForInternal) { + return $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['users.count'])], 0, ['time'], [Database::ORDER_DESC]); + }); + $usersTotal = $usersCount ? $usersCount->getAttribute('value', 0) : 0; - // $projectDB->getCollection([ - // 'limit' => 0, - // 'offset' => 0, - // 'filters' => [ - // '$collection=users', - // ], - // ]); - - // $usersTotal = $projectDB->getSum(); - - // // Documents - - // $collections = $projectDB->getCollection([ - // 'limit' => 100, - // 'offset' => 0, - // 'filters' => [ - // '$collection=collections', - // ], - // ]); - - // $collectionsTotal = $projectDB->getSum(); + $collectionsCount = Authorization::skip(function () use ($dbForInternal, $period, $range) { + return $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['collections.count'])], 0, ['time'], [Database::ORDER_DESC]); + }); + $collectionsTotal = $collectionsCount ? $collectionsCount->getAttribute('value', 0) : 0; + // $documents = []; @@ -410,6 +353,11 @@ App::get('/v1/projects/:projectId/usage') // $documents[] = ['name' => $collection['name'], 'total' => $projectDB->getSum()]; // } + $filesCount = Authorization::skip(function () use ($dbForInternal, $period, $range) { + return $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['files.count'])], 0, ['time'], [Database::ORDER_DESC]); + }); + $filesTotal = $filesCount ? $filesCount->getAttribute('value', 0) : 0; + $response->json([ 'range' => $range, 'requests' => [ @@ -430,20 +378,24 @@ App::get('/v1/projects/:projectId/usage') return $item['value']; }, $functions)), ], - // 'collections' => [ - // 'data' => $collections, - // 'total' => $collectionsTotal, - // ], + 'collections' => [ + 'data' => [], + 'total' => $collectionsTotal, + ], + 'files' => [ + 'data' => [], + 'total' => $filesTotal, + ], // 'documents' => [ // 'data' => $documents, // 'total' => \array_sum(\array_map(function ($item) { // return $item['total']; // }, $documents)), // ], - // 'users' => [ - // 'data' => [], - // 'total' => $usersTotal, - // ], + 'users' => [ + 'data' => [], + 'total' => $usersTotal, + ], // 'storage' => [ // 'total' => $projectDB->getCount( // [ From 89f55a17277361f14b84fac7f07659920c5e74eb Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Wed, 18 Aug 2021 16:42:03 +0300 Subject: [PATCH 039/206] Re-orgenized controller flow --- app/controllers/api/database.php | 12 ++++++------ app/controllers/api/functions.php | 28 ++++++++++++++-------------- app/controllers/api/projects.php | 12 ++++++------ app/controllers/api/storage.php | 12 ++++++------ app/controllers/api/teams.php | 12 ++++++------ 5 files changed, 38 insertions(+), 38 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 6cc933e6cd..01f3e7df7a 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -193,12 +193,6 @@ App::get('/v1/database/collections') /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForExternal */ - $queries = []; - - if (!empty($search)) { - $queries[] = new Query('name', Query::TYPE_SEARCH, [$search]); - } - if (!empty($after)) { $afterCollection = $dbForExternal->getDocument('collections', $after); @@ -207,6 +201,12 @@ App::get('/v1/database/collections') } } + $queries = []; + + if (!empty($search)) { + $queries[] = new Query('name', Query::TYPE_SEARCH, [$search]); + } + $response->dynamic(new Document([ 'collections' => $dbForExternal->find(Database::COLLECTIONS, $queries, $limit, $offset, [], [$orderType], $afterCollection ?? null), 'sum' => $dbForExternal->count(Database::COLLECTIONS, $queries, APP_LIMIT_COUNT), diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index f862bee856..f0f7f61330 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -98,12 +98,6 @@ App::get('/v1/functions') /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ - $queries = []; - - if (!empty($search)) { - $queries[] = new Query('search', Query::TYPE_SEARCH, [$search]); - } - if (!empty($after)) { $afterFunction = $dbForInternal->getDocument('functions', $after); @@ -111,6 +105,12 @@ App::get('/v1/functions') throw new Exception("Function '{$after}' for the 'after' value not found.", 400); } } + + $queries = []; + + if (!empty($search)) { + $queries[] = new Query('search', Query::TYPE_SEARCH, [$search]); + } $response->dynamic(new Document([ 'functions' => $dbForInternal->find('functions', $queries, $limit, $offset, [], [$orderType], $afterFunction ?? null), @@ -538,14 +538,6 @@ App::get('/v1/functions/:functionId/tags') throw new Exception('Function not found', 404); } - $queries = []; - - if (!empty($search)) { - $queries[] = new Query('search', Query::TYPE_SEARCH, [$search]); - } - - $queries[] = new Query('functionId', Query::TYPE_EQUAL, [$function->getId()]); - if (!empty($after)) { $afterTag = $dbForInternal->getDocument('tags', $after); @@ -554,6 +546,14 @@ App::get('/v1/functions/:functionId/tags') } } + $queries = []; + + if (!empty($search)) { + $queries[] = new Query('search', Query::TYPE_SEARCH, [$search]); + } + + $queries[] = new Query('functionId', Query::TYPE_EQUAL, [$function->getId()]); + $results = $dbForInternal->find('tags', $queries, $limit, $offset, [], [$orderType], $afterTag ?? null); $sum = $dbForInternal->count('tags', $queries, APP_LIMIT_COUNT); diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 0a2336920e..df740eb421 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -171,12 +171,6 @@ App::get('/v1/projects') /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForConsole */ - $queries = []; - - if (!empty($search)) { - $queries[] = new Query('search', Query::TYPE_SEARCH, [$search]); - } - if (!empty($after)) { $afterProject = $dbForConsole->getDocument('projects', $after); @@ -185,6 +179,12 @@ App::get('/v1/projects') } } + $queries = []; + + if (!empty($search)) { + $queries[] = new Query('search', Query::TYPE_SEARCH, [$search]); + } + $results = $dbForConsole->find('projects', $queries, $limit, $offset, [], [$orderType], $afterProject ?? null); $sum = $dbForConsole->count('projects', $queries, APP_LIMIT_COUNT); diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index d6735c1380..b73e3facf6 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -181,12 +181,6 @@ App::get('/v1/storage/files') /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ - $queries = []; - - if (!empty($search)) { - $queries[] = new Query('search', Query::TYPE_SEARCH, [$search]); - } - if (!empty($after)) { $afterFile = $dbForInternal->getDocument('files', $after); @@ -195,6 +189,12 @@ App::get('/v1/storage/files') } } + $queries = []; + + if (!empty($search)) { + $queries[] = new Query('search', Query::TYPE_SEARCH, [$search]); + } + $response->dynamic(new Document([ 'files' => $dbForInternal->find('files', $queries, $limit, $offset, [], [$orderType], $afterFile ?? null), 'sum' => $dbForInternal->count('files', $queries, APP_LIMIT_COUNT), diff --git a/app/controllers/api/teams.php b/app/controllers/api/teams.php index 903668d525..75d6f91cba 100644 --- a/app/controllers/api/teams.php +++ b/app/controllers/api/teams.php @@ -108,12 +108,6 @@ App::get('/v1/teams') /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ - $queries = []; - - if (!empty($search)) { - $queries[] = new Query('search', Query::TYPE_SEARCH, [$search]); - } - if (!empty($after)) { $afterTeam = $dbForInternal->getDocument('teams', $after); @@ -122,6 +116,12 @@ App::get('/v1/teams') } } + $queries = []; + + if (!empty($search)) { + $queries[] = new Query('search', Query::TYPE_SEARCH, [$search]); + } + $results = $dbForInternal->find('teams', $queries, $limit, $offset, [], [$orderType], $afterTeam ?? null); $sum = $dbForInternal->count('teams', $queries, APP_LIMIT_COUNT); From e82cf42fe30727f80dafa1345f4e7f9fbc49a5a8 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 19 Aug 2021 12:23:56 +0545 Subject: [PATCH 040/206] refactor usage endpoint --- app/controllers/api/projects.php | 89 +++++++++----------------------- 1 file changed, 24 insertions(+), 65 deletions(-) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index f8fe0e8a96..f660bf2f60 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -241,30 +241,22 @@ App::get('/v1/projects/:projectId/usage') throw new Exception('Project not found', 404); } + $stats = []; if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { - $period = [ '24h' => [ - // 'start' => DateTime::createFromFormat('U', \strtotime('-24 hours')), - // 'end' => DateTime::createFromFormat('U', \strtotime('+1 hour')), 'period' => '30m', 'limit' => 48, ], '7d' => [ - // 'start' => DateTime::createFromFormat('U', \strtotime('-7 days')), - // 'end' => DateTime::createFromFormat('U', \strtotime('now')), 'period' => '1d', 'limit' => 7, ], '30d' => [ - // 'start' => DateTime::createFromFormat('U', \strtotime('-30 days')), - // 'end' => DateTime::createFromFormat('U', \strtotime('now')), 'period' => '1d', 'limit' => 30, ], '90d' => [ - // 'start' => DateTime::createFromFormat('U', \strtotime('-90 days')), - // 'end' => DateTime::createFromFormat('U', \strtotime('now')), 'period' => '1d', 'limit' => 90, ], @@ -272,59 +264,25 @@ App::get('/v1/projects/:projectId/usage') $dbForInternal->setNamespace('project_' . $projectId . '_internal'); - $requestDocs = Authorization::skip(function () use ($dbForInternal, $period, $range) { - return $dbForInternal->find('stats', [ + Authorization::disable(); + + $metrics = ['requests', 'network', 'executions']; + foreach ($metrics as $metric) { + $requestDocs = $dbForInternal->find('stats', [ new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), - new Query('metric', Query::TYPE_EQUAL, ['requests']), + new Query('metric', Query::TYPE_EQUAL, [$metric]), ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); - }); - $requests = []; - foreach ($requestDocs as $requestDoc) { - $requests[] = [ - 'value' => $requestDoc->getAttribute('value'), - 'date' => $requestDoc->getAttribute('time'), - ]; + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + + $stats[$metric] = array_reverse($stats[$metric]); } - - $requests = array_reverse($requests); - - $networkDocs = Authorization::skip(function () use ($dbForInternal, $period, $range) { - return $dbForInternal->find('stats', [ - new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), - new Query('metric', Query::TYPE_EQUAL, ['network']), - ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); - }); - - $network = []; - foreach ($networkDocs as $networkDoc) { - $network[] = [ - 'value' => $networkDoc->getAttribute('value'), - 'date' => $networkDoc->getAttribute('time'), - ]; - } - - $network = array_reverse($network); - - $functionsDocs = Authorization::skip(function () use ($dbForInternal, $period, $range) { - return $dbForInternal->find('stats', [ - new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), - new Query('metric', Query::TYPE_EQUAL, ['executions']), - ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); - }); - - $functions = []; - foreach ($functionsDocs as $functionDoc) { - $functions[] = [ - 'value' => $functionDoc->getAttribute('value'), - 'date' => $functionDoc->getAttribute('time'), - ]; - } - $functions = array_reverse($functions); - } else { - $requests = []; - $network = []; - $functions = []; } $usersCount = Authorization::skip(function () use ($dbForInternal) { @@ -336,7 +294,6 @@ App::get('/v1/projects/:projectId/usage') return $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['collections.count'])], 0, ['time'], [Database::ORDER_DESC]); }); $collectionsTotal = $collectionsCount ? $collectionsCount->getAttribute('value', 0) : 0; - // $documents = []; @@ -357,25 +314,27 @@ App::get('/v1/projects/:projectId/usage') }); $filesTotal = $filesCount ? $filesCount->getAttribute('value', 0) : 0; + Authorization::reset(); + $response->json([ 'range' => $range, 'requests' => [ - 'data' => $requests, + 'data' => $stats['requests'] ?? [], 'total' => \array_sum(\array_map(function ($item) { return $item['value']; - }, $requests)), + }, $stats['requests'] ?? [])), ], 'network' => [ - 'data' => \array_map(function ($value) {return ['value' => \round($value['value'] / 1000000, 2), 'date' => $value['date']];}, $network), // convert bytes to mb + 'data' => \array_map(function ($value) {return ['value' => \round($value['value'] / 1000000, 2), 'date' => $value['date']];}, $stats['network'] ?? []), // convert bytes to mb 'total' => \array_sum(\array_map(function ($item) { return $item['value']; - }, $network)), + }, $stats['network'] ?? [])), ], 'functions' => [ - 'data' => $functions, + 'data' => $stats['executions'] ?? [], 'total' => \array_sum(\array_map(function ($item) { return $item['value']; - }, $functions)), + }, $stats['executions'] ?? [])), ], 'collections' => [ 'data' => [], From e574bdc3b39a6a74d810814a2349c46c54064069 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 19 Aug 2021 12:39:03 +0545 Subject: [PATCH 041/206] remove influxdb dependency on appwrite container --- docker-compose.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 143f124f20..a9911b1d6d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -73,7 +73,6 @@ services: - mariadb - redis # - clamav - - influxdb entrypoint: - php - -e @@ -112,8 +111,6 @@ services: - _APP_SMTP_USERNAME - _APP_SMTP_PASSWORD - _APP_USAGE_STATS - - _APP_INFLUXDB_HOST - - _APP_INFLUXDB_PORT - _APP_STORAGE_LIMIT - _APP_FUNCTIONS_TIMEOUT - _APP_FUNCTIONS_CONTAINERS From 91dc62d733bf081b86881e7d01cc0beb5f51bb28 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 19 Aug 2021 12:39:32 +0545 Subject: [PATCH 042/206] metrics list comment --- app/tasks/usage.php | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index afe5958406..216949f6d2 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -14,6 +14,43 @@ use Utopia\Database\Document; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; +/** + * Metrics We collect + * + * requests + * network + * executions + * database.collections.create + * database.collections.read + * database.collections.update + * database.collections.delete + * database.documents.create + * database.documents.read + * database.documents.update + * database.documents.delete + * database.collections.{collectionId}.documents.create + * database.collections.{collectionId}.documents.read + * database.collections.{collectionId}.documents.update + * database.collections.{collectionId}.documents.delete + * storage.buckets.{bucketId}.files.create + * storage.buckets.{bucketId}.files.read + * storage.buckets.{bucketId}.files.update + * storage.buckets.{bucketId}.files.delete + * users.create + * users.read + * users.update + * users.delete + * users.sessions.create + * users.sessions.delete + * + * Counters + * + * users.count + * files.count + * collections.count + * + */ + $cli ->task('usage') ->desc('Schedules syncing data from influxdb to Appwrite console db') From 800a76db729a8b85587537860d7366da69d59791 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 19 Aug 2021 13:46:22 +0545 Subject: [PATCH 043/206] counter stats update --- app/controllers/api/projects.php | 37 ++++--------- app/tasks/usage.php | 91 ++++++++++++++++++++++++++------ 2 files changed, 84 insertions(+), 44 deletions(-) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index f660bf2f60..bea5332f11 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -285,33 +285,16 @@ App::get('/v1/projects/:projectId/usage') } } - $usersCount = Authorization::skip(function () use ($dbForInternal) { - return $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['users.count'])], 0, ['time'], [Database::ORDER_DESC]); - }); + $usersCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['users.count'])], 0, ['time'], [Database::ORDER_DESC]); $usersTotal = $usersCount ? $usersCount->getAttribute('value', 0) : 0; - $collectionsCount = Authorization::skip(function () use ($dbForInternal, $period, $range) { - return $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['collections.count'])], 0, ['time'], [Database::ORDER_DESC]); - }); + $collectionsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['database.collections.count'])], 0, ['time'], [Database::ORDER_DESC]); $collectionsTotal = $collectionsCount ? $collectionsCount->getAttribute('value', 0) : 0; - // $documents = []; + $documentsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['database.documents.count'])], 0, ['time'], [Database::ORDER_DESC]); + $documentsTotal = $documentsCount ? $documentsCount->getAttribute('value', 0) : 0; - // foreach ($collections as $collection) { - // $result = $projectDB->getCollection([ - // 'limit' => 0, - // 'offset' => 0, - // 'filters' => [ - // '$collection=' . $collection['$id'], - // ], - // ]); - - // $documents[] = ['name' => $collection['name'], 'total' => $projectDB->getSum()]; - // } - - $filesCount = Authorization::skip(function () use ($dbForInternal, $period, $range) { - return $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['files.count'])], 0, ['time'], [Database::ORDER_DESC]); - }); + $filesCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['storage.files.count'])], 0, ['time'], [Database::ORDER_DESC]); $filesTotal = $filesCount ? $filesCount->getAttribute('value', 0) : 0; Authorization::reset(); @@ -344,12 +327,10 @@ App::get('/v1/projects/:projectId/usage') 'data' => [], 'total' => $filesTotal, ], - // 'documents' => [ - // 'data' => $documents, - // 'total' => \array_sum(\array_map(function ($item) { - // return $item['total']; - // }, $documents)), - // ], + 'documents' => [ + 'data' => [], + 'total' => $documentsTotal, + ], 'users' => [ 'data' => [], 'total' => $usersTotal, diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 216949f6d2..ffa2df490b 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -16,7 +16,7 @@ use Utopia\Database\Validator\Authorization; /** * Metrics We collect - * + * * requests * network * executions @@ -42,13 +42,15 @@ use Utopia\Database\Validator\Authorization; * users.delete * users.sessions.create * users.sessions.delete - * + * * Counters - * + * * users.count - * files.count - * collections.count - * + * storage.files.count + * database.collections.count + * database.documents.count + * database.collections.{collectionId}.documents.count + * */ $cli @@ -249,8 +251,10 @@ $cli } } - if ($iterations % 30 == 0) { - //aggregate number of objects in database + if ($iterations % 30 == 0) { //every 15 minutes + // aggregate number of objects in database + // get count of all the documents per collection - + // buckets will have the same $latestProject = null; do { $projects = $dbForConsole->find('projects', [], 100, orderAfter:$latestProject); @@ -259,25 +263,80 @@ $cli foreach ($projects as $project) { $id = $project->getId(); - $collections = ['users' => [ - 'namespace' => 'internal', - ], 'collections' => [ - 'namespace' => 'external', - ], 'files' => [ - 'namespace' => 'internal', - ]]; + $collections = [ + 'users' => [ + 'namespace' => 'internal', + ], + 'collections' => [ + 'metricPrefix' => 'database', + 'namespace' => 'external', //new change will make this internal + 'subCollections' => [ + 'documents' => [ + 'namespace' => 'external', + ], + ], + ], + 'files' => [ + 'metricPrefix' => 'storage', + 'namespace' => 'internal', + ], + ]; foreach ($collections as $collection => $options) { $dbForProject->setNamespace("project_{$id}_{$options['namespace']}"); $count = $dbForProject->count($collection); $dbForProject->setNamespace("project_{$id}_internal"); + $metricPrefix = $options['metricPrefix'] ?? ''; + $metric = empty($metricPrefix) ? "{$collection}.count" : "{$metricPrefix}.{$collection}.count"; $dbForProject->createDocument('stats', new Document([ '$id' => $dbForProject->getId(), 'time' => time(), 'period' => '15m', - 'metric' => "{$collection}.count", + 'metric' => $metric, 'value' => $count, 'type' => 1, ])); + + $subCollections = $options['subCollections'] ?? []; + if (!empty($subCollections)) { + $latestParent = null; + $subCollectionCounts = []; //total project level count of sub collections + do { + $dbForProject->setNamespace("project_{$id}_{$options['namespace']}"); + $parents = $dbForProject->find($collection, [], 100, orderAfter:$latestParent); + if (!empty($parents)) { + $latestParent = $parents[array_key_last($parents)]; + foreach ($parents as $parent) { + foreach ($subCollections as $subCollection => $subOptions) { + $dbForProject->setNamespace("project_{$id}_{$subOptions['namespace']}"); + $count = $dbForProject->count($parent->getId()); + $subCollectionsCounts[$subCollection] = ($subCollectionCounts[$subCollection] ?? 0) + $count; + + $dbForProject->setNamespace("project_{$id}_internal"); + $dbForProject->createDocument('stats', new Document([ + '$id' => $dbForProject->getId(), + 'time' => time(), + 'period' => '15m', + 'metric' => empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.count" : "{$metricPrefix}.{$collection}.{$parent->getId()}.{$subCollection}.count", + 'value' => $count, + 'type' => 1, + ])); + } + } + } + } while (!empty($parents)); + + foreach ($subCollectionsCounts as $subCollection => $count) { + $dbForProject->setNamespace("project_{$id}_internal"); + $dbForProject->createDocument('stats', new Document([ + '$id' => $dbForProject->getId(), + 'time' => time(), + 'period' => '15m', + 'metric' => empty($metricPrefix) ? "{$subCollection}.count" : "{$metricPrefix}.{$subCollection}.count", + 'value' => $count, + 'type' => 1, + ])); + } + } } } } From 423ecb10712425d131c04b5d2c60ed2835540ba7 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 19 Aug 2021 13:47:28 +0545 Subject: [PATCH 044/206] update stats table indexes --- app/config/collections2.php | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/app/config/collections2.php b/app/config/collections2.php index 2b2419204e..0ed14dc369 100644 --- a/app/config/collections2.php +++ b/app/config/collections2.php @@ -1473,9 +1473,16 @@ $collections = [ 'orders' => [Database::ORDER_DESC], ], [ - '$id' => '_key_time_period', + '$id' => '_key_metric', 'type' => Database::INDEX_KEY, - 'attributes' => ['time', 'period'], + 'attributes' => ['metric'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + ], + [ + '$id' => '_key_metric_period', + 'type' => Database::INDEX_KEY, + 'attributes' => ['metric', 'period'], 'lengths' => [], 'orders' => [Database::ORDER_DESC], ], From 39d4924fe82e2d0a71eba466df77a223a2982dd0 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 19 Aug 2021 13:59:23 +0545 Subject: [PATCH 045/206] storage total stats --- app/controllers/api/projects.php | 24 ++++++------------------ app/tasks/usage.php | 20 +++++++++++++++++++- 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index bea5332f11..b655f7a6f0 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -296,6 +296,9 @@ App::get('/v1/projects/:projectId/usage') $filesCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['storage.files.count'])], 0, ['time'], [Database::ORDER_DESC]); $filesTotal = $filesCount ? $filesCount->getAttribute('value', 0) : 0; + + $storageTotal = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['storage.total'])], 0, ['time'], [Database::ORDER_DESC]); + $storage = $storageTotal ? $storageTotal->getAttribute('value', 0) : 0; Authorization::reset(); @@ -335,24 +338,9 @@ App::get('/v1/projects/:projectId/usage') 'data' => [], 'total' => $usersTotal, ], - // 'storage' => [ - // 'total' => $projectDB->getCount( - // [ - // 'attribute' => 'sizeOriginal', - // 'filters' => [ - // '$collection=files', - // ], - // ] - // ) + - // $projectDB->getCount( - // [ - // 'attribute' => 'size', - // 'filters' => [ - // '$collection=tags', - // ], - // ] - // ), - // ], + 'storage' => [ + 'total' => $storage, + ], ]); }); diff --git a/app/tasks/usage.php b/app/tasks/usage.php index ffa2df490b..3f938e46bb 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -51,6 +51,10 @@ use Utopia\Database\Validator\Authorization; * database.documents.count * database.collections.{collectionId}.documents.count * + * Totals + * + * storage.total + * */ $cli @@ -263,6 +267,20 @@ $cli foreach ($projects as $project) { $id = $project->getId(); + + // get total storage + $dbForProject->setNamespace('project_' . $id . '_internal'); + $storageTotal = $dbForProject->sum('files', 'sizeOriginal') + $dbForProject->sum('tags', 'size'); + + $dbForProject->createDocument('stats', new Document([ + '$id' => $dbForProject->getId(), + 'period' => '15m', + 'time' => time(), + 'metric' => 'storage.total', + 'value' => $storageTotal, + 'type' => 1, + ])); + $collections = [ 'users' => [ 'namespace' => 'internal', @@ -310,7 +328,7 @@ $cli $dbForProject->setNamespace("project_{$id}_{$subOptions['namespace']}"); $count = $dbForProject->count($parent->getId()); $subCollectionsCounts[$subCollection] = ($subCollectionCounts[$subCollection] ?? 0) + $count; - + $dbForProject->setNamespace("project_{$id}_internal"); $dbForProject->createDocument('stats', new Document([ '$id' => $dbForProject->getId(), From 87415c3d953e60fb23bda48f4ac51eecfafd81f8 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 19 Aug 2021 14:43:10 +0545 Subject: [PATCH 046/206] use internals table for collections --- app/tasks/usage.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 3f938e46bb..21b67a6217 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -287,7 +287,7 @@ $cli ], 'collections' => [ 'metricPrefix' => 'database', - 'namespace' => 'external', //new change will make this internal + 'namespace' => 'internal', 'subCollections' => [ 'documents' => [ 'namespace' => 'external', From 71eabfff0d2df4931a90b7f41ac12142d3826fda Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 19 Aug 2021 14:47:38 +0545 Subject: [PATCH 047/206] handling errors --- app/tasks/usage.php | 104 +++++++++++++++++++++++--------------------- 1 file changed, 54 insertions(+), 50 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 21b67a6217..7e7f7d0a12 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -300,60 +300,64 @@ $cli ], ]; foreach ($collections as $collection => $options) { - $dbForProject->setNamespace("project_{$id}_{$options['namespace']}"); - $count = $dbForProject->count($collection); - $dbForProject->setNamespace("project_{$id}_internal"); - $metricPrefix = $options['metricPrefix'] ?? ''; - $metric = empty($metricPrefix) ? "{$collection}.count" : "{$metricPrefix}.{$collection}.count"; - $dbForProject->createDocument('stats', new Document([ - '$id' => $dbForProject->getId(), - 'time' => time(), - 'period' => '15m', - 'metric' => $metric, - 'value' => $count, - 'type' => 1, - ])); - - $subCollections = $options['subCollections'] ?? []; - if (!empty($subCollections)) { - $latestParent = null; - $subCollectionCounts = []; //total project level count of sub collections - do { - $dbForProject->setNamespace("project_{$id}_{$options['namespace']}"); - $parents = $dbForProject->find($collection, [], 100, orderAfter:$latestParent); - if (!empty($parents)) { - $latestParent = $parents[array_key_last($parents)]; - foreach ($parents as $parent) { - foreach ($subCollections as $subCollection => $subOptions) { - $dbForProject->setNamespace("project_{$id}_{$subOptions['namespace']}"); - $count = $dbForProject->count($parent->getId()); - $subCollectionsCounts[$subCollection] = ($subCollectionCounts[$subCollection] ?? 0) + $count; - - $dbForProject->setNamespace("project_{$id}_internal"); - $dbForProject->createDocument('stats', new Document([ - '$id' => $dbForProject->getId(), - 'time' => time(), - 'period' => '15m', - 'metric' => empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.count" : "{$metricPrefix}.{$collection}.{$parent->getId()}.{$subCollection}.count", - 'value' => $count, - 'type' => 1, - ])); + try { + $dbForProject->setNamespace("project_{$id}_{$options['namespace']}"); + $count = $dbForProject->count($collection); + $dbForProject->setNamespace("project_{$id}_internal"); + $metricPrefix = $options['metricPrefix'] ?? ''; + $metric = empty($metricPrefix) ? "{$collection}.count" : "{$metricPrefix}.{$collection}.count"; + $dbForProject->createDocument('stats', new Document([ + '$id' => $dbForProject->getId(), + 'time' => time(), + 'period' => '15m', + 'metric' => $metric, + 'value' => $count, + 'type' => 1, + ])); + + $subCollections = $options['subCollections'] ?? []; + if (!empty($subCollections)) { + $latestParent = null; + $subCollectionCounts = []; //total project level count of sub collections + do { + $dbForProject->setNamespace("project_{$id}_{$options['namespace']}"); + $parents = $dbForProject->find($collection, [], 100, orderAfter:$latestParent); + if (!empty($parents)) { + $latestParent = $parents[array_key_last($parents)]; + foreach ($parents as $parent) { + foreach ($subCollections as $subCollection => $subOptions) { + $dbForProject->setNamespace("project_{$id}_{$subOptions['namespace']}"); + $count = $dbForProject->count($parent->getId()); + $subCollectionsCounts[$subCollection] = ($subCollectionCounts[$subCollection] ?? 0) + $count; + + $dbForProject->setNamespace("project_{$id}_internal"); + $dbForProject->createDocument('stats', new Document([ + '$id' => $dbForProject->getId(), + 'time' => time(), + 'period' => '15m', + 'metric' => empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.count" : "{$metricPrefix}.{$collection}.{$parent->getId()}.{$subCollection}.count", + 'value' => $count, + 'type' => 1, + ])); + } } } + } while (!empty($parents)); + + foreach ($subCollectionsCounts as $subCollection => $count) { + $dbForProject->setNamespace("project_{$id}_internal"); + $dbForProject->createDocument('stats', new Document([ + '$id' => $dbForProject->getId(), + 'time' => time(), + 'period' => '15m', + 'metric' => empty($metricPrefix) ? "{$subCollection}.count" : "{$metricPrefix}.{$subCollection}.count", + 'value' => $count, + 'type' => 1, + ])); } - } while (!empty($parents)); - - foreach ($subCollectionsCounts as $subCollection => $count) { - $dbForProject->setNamespace("project_{$id}_internal"); - $dbForProject->createDocument('stats', new Document([ - '$id' => $dbForProject->getId(), - 'time' => time(), - 'period' => '15m', - 'metric' => empty($metricPrefix) ? "{$subCollection}.count" : "{$metricPrefix}.{$subCollection}.count", - 'value' => $count, - 'type' => 1, - ])); } + } catch(\Exception $e) { + Console::warning("Failed to save database counters data for project {$collection}"); } } } From 08bdf089f62f84464fc188701683a04c872a7fda Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 19 Aug 2021 14:48:53 +0545 Subject: [PATCH 048/206] remove unused stats --- app/controllers/api/projects.php | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index b655f7a6f0..7d8f4ed54c 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -288,14 +288,8 @@ App::get('/v1/projects/:projectId/usage') $usersCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['users.count'])], 0, ['time'], [Database::ORDER_DESC]); $usersTotal = $usersCount ? $usersCount->getAttribute('value', 0) : 0; - $collectionsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['database.collections.count'])], 0, ['time'], [Database::ORDER_DESC]); - $collectionsTotal = $collectionsCount ? $collectionsCount->getAttribute('value', 0) : 0; - $documentsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['database.documents.count'])], 0, ['time'], [Database::ORDER_DESC]); $documentsTotal = $documentsCount ? $documentsCount->getAttribute('value', 0) : 0; - - $filesCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['storage.files.count'])], 0, ['time'], [Database::ORDER_DESC]); - $filesTotal = $filesCount ? $filesCount->getAttribute('value', 0) : 0; $storageTotal = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['storage.total'])], 0, ['time'], [Database::ORDER_DESC]); $storage = $storageTotal ? $storageTotal->getAttribute('value', 0) : 0; @@ -322,14 +316,6 @@ App::get('/v1/projects/:projectId/usage') return $item['value']; }, $stats['executions'] ?? [])), ], - 'collections' => [ - 'data' => [], - 'total' => $collectionsTotal, - ], - 'files' => [ - 'data' => [], - 'total' => $filesTotal, - ], 'documents' => [ 'data' => [], 'total' => $documentsTotal, From 2a1ccafb62a597ff3ab1d0e05387467a23d78f05 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 19 Aug 2021 16:58:26 +0545 Subject: [PATCH 049/206] Update app/controllers/api/account.php Co-authored-by: Christy Jacob --- app/controllers/api/account.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 3d574054bb..273e7226fa 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -160,7 +160,7 @@ App::post('/v1/account/sessions') /** @var Utopia\Locale\Locale $locale */ /** @var MaxMind\Db\Reader $geodb */ /** @var Appwrite\Event\Event $audits */ - /** @var Appwrite\Usage\Usage $usage */ + /** @var Appwrite\Stats\Stats $usage */ $email = \strtolower($email); $protocol = $request->getProtocol(); From 4b5db30a43c1be52c3f562c9ea68a23c53585960 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 19 Aug 2021 16:58:32 +0545 Subject: [PATCH 050/206] Update app/controllers/api/account.php Co-authored-by: Christy Jacob --- app/controllers/api/account.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index 273e7226fa..dafddb4fee 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -59,7 +59,7 @@ App::post('/v1/account') /** @var Utopia\Database\Document $project */ /** @var Utopia\Database\Database $dbForInternal */ /** @var Appwrite\Event\Event $audits */ - /** @var Appwrite\Usage\Usage $usage */ + /** @var Appwrite\Stats\Stats $usage */ $email = \strtolower($email); if ('console' === $project->getId()) { From 1d7daf951530839a705626b10303f1e8b71ec103 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 20 Aug 2021 11:40:23 +0545 Subject: [PATCH 051/206] new functions stats --- app/tasks/usage.php | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 7e7f7d0a12..6b9c56b718 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -43,6 +43,12 @@ use Utopia\Database\Validator\Authorization; * users.sessions.create * users.sessions.delete * + * Functions + * + * functions.{functionId}.executions + * functions.{functionId}.failures + * functions.{functionId}.compute + * * Counters * * users.count @@ -162,6 +168,21 @@ $cli 'users.sessions.delete' => [ 'table' => 'appwrite_usage_users_sessions_delete', ], + 'functions.functionId.executions' => [ + 'table' => 'appwrite_usage_executions_all', + 'groupBy' => 'functionId', + ], + 'functions.functionId.compute' => [ + 'table' => 'appwrite_usage_executions_time', + 'groupBy' => 'functionId', + ], + 'functions.functionId.failures' => [ + 'table' => 'appwrite_usage_executions_all', + 'groupBy' => 'functionId', + 'filters' => [ + 'functionStatus' => 'failed', + ], + ], ]; $attempts = 0; @@ -213,7 +234,15 @@ $cli $table = $options['table']; //which influxdb table to query for this metric $groupBy = empty($options['groupBy']) ? '' : ', "' . $options['groupBy'] . '"'; //some sub level metrics may be grouped by other tags like collectionId, bucketId, etc - $result = $database->query('SELECT sum(value) AS "value" FROM "' . $table . '" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\' GROUP BY time(' . $period['key'] . '), "projectId"' . $groupBy . ' FILL(null)'); + $filters = $options['filters'] ?? []; + if (!empty($filters)) { + $filters = ' AND ' . implode(' AND ', array_map(function ($filter, $value) { + return '"' . $filter . '"=\'' . $value . '\''; + }, array_keys($filters), array_values($filters))); + } + + $result = $database->query('SELECT sum(value) AS "value" FROM "' . $table . '" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\'' . (empty($filters) ? '' : $filters) . ' GROUP BY time(' . $period['key'] . '), "projectId"' . $groupBy . ' FILL(null)'); + $points = $result->getPoints(); foreach ($points as $point) { $projectId = $point['projectId']; @@ -314,7 +343,7 @@ $cli 'value' => $count, 'type' => 1, ])); - + $subCollections = $options['subCollections'] ?? []; if (!empty($subCollections)) { $latestParent = null; @@ -329,7 +358,7 @@ $cli $dbForProject->setNamespace("project_{$id}_{$subOptions['namespace']}"); $count = $dbForProject->count($parent->getId()); $subCollectionsCounts[$subCollection] = ($subCollectionCounts[$subCollection] ?? 0) + $count; - + $dbForProject->setNamespace("project_{$id}_internal"); $dbForProject->createDocument('stats', new Document([ '$id' => $dbForProject->getId(), @@ -343,7 +372,7 @@ $cli } } } while (!empty($parents)); - + foreach ($subCollectionsCounts as $subCollection => $count) { $dbForProject->setNamespace("project_{$id}_internal"); $dbForProject->createDocument('stats', new Document([ @@ -356,7 +385,7 @@ $cli ])); } } - } catch(\Exception $e) { + } catch (\Exception$e) { Console::warning("Failed to save database counters data for project {$collection}"); } } From 7949d5b1385d9fc574cdd669459ea072111eaeee Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 20 Aug 2021 12:23:35 +0545 Subject: [PATCH 052/206] fixed issues with metrics --- app/tasks/usage.php | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 6b9c56b718..a6c9f3edac 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -248,15 +248,19 @@ $cli $projectId = $point['projectId']; if (!empty($projectId) && $projectId != 'console') { $dbForProject->setNamespace('project_' . $projectId . '_internal'); + if($metric == 'functions.functionId.executions') { + var_dump($points); + } + $metricUpdated = $metric; if (!empty($groupBy)) { - $groupedBy = $point[$groupBy] ?? ''; + $groupedBy = $point[$options['groupBy']] ?? ''; if (empty($groupedBy)) { continue; } - $metric = str_replace($groupBy, $groupedBy, $metric); + $metricUpdated = str_replace($options['groupBy'], $groupedBy, $metric); } $time = \strtotime($point['time']); - $id = \md5($time . '_' . $period['key'] . '_' . $metric); //construct unique id for each metric using time, period and metric + $id = \md5($time . '_' . $period['key'] . '_' . $metricUpdated); //construct unique id for each metric using time, period and metric $value = (!empty($point['value'])) ? $point['value'] : 0; try { $document = $dbForProject->getDocument('stats', $id); @@ -265,7 +269,7 @@ $cli '$id' => $id, 'period' => $period['key'], 'time' => $time, - 'metric' => $metric, + 'metric' => $metricUpdated, 'value' => $value, 'type' => 0, ])); @@ -276,7 +280,7 @@ $cli $latestTime[$metric][$period['key']] = $time; } catch (\Exception$e) { // if projects are deleted this might fail - Console::warning("Failed to save data for project {$projectId} and metric {$metric}"); + Console::warning("Failed to save data for project {$projectId} and metric {$metricUpdated}"); } } } From 2c69fab6cd05994b74987dc6a844493a064a51f5 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 20 Aug 2021 12:29:30 +0545 Subject: [PATCH 053/206] use metadata instead --- app/config/collections2.php | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/app/config/collections2.php b/app/config/collections2.php index 0ed14dc369..4e71c06256 100644 --- a/app/config/collections2.php +++ b/app/config/collections2.php @@ -8,7 +8,7 @@ $auth = Config::getParam('auth', []); $collections = [ 'projects' => [ - '$collection' => Database::COLLECTIONS, + '$collection' => Database::METADATA, '$id' => 'projects', 'name' => 'Projects', 'attributes' => [ @@ -234,7 +234,7 @@ $collections = [ ], 'users' => [ - '$collection' => Database::COLLECTIONS, + '$collection' => Database::METADATA, '$id' => 'users', 'name' => 'Users', 'attributes' => [ @@ -383,7 +383,7 @@ $collections = [ ], 'sessions' => [ - '$collection' => Database::COLLECTIONS, + '$collection' => Database::METADATA, '$id' => 'sessions', 'name' => 'Sessions', 'attributes' => [ @@ -631,7 +631,7 @@ $collections = [ ], 'teams' => [ - '$collection' => Database::COLLECTIONS, + '$collection' => Database::METADATA, '$id' => 'teams', 'name' => 'Teams', 'attributes' => [ @@ -681,7 +681,7 @@ $collections = [ ], 'memberships' => [ - '$collection' => Database::COLLECTIONS, + '$collection' => Database::METADATA, '$id' => 'memberships', 'name' => 'Memberships', 'attributes' => [ @@ -789,7 +789,7 @@ $collections = [ ], 'files' => [ - '$collection' => Database::COLLECTIONS, + '$collection' => Database::METADATA, '$id' => 'files', 'name' => 'Files', 'attributes' => [ @@ -968,7 +968,7 @@ $collections = [ ], 'functions' => [ - '$collection' => Database::COLLECTIONS, + '$collection' => Database::METADATA, '$id' => 'functions', 'name' => 'Functions', 'attributes' => [ @@ -1129,7 +1129,7 @@ $collections = [ ], 'tags' => [ - '$collection' => Database::COLLECTIONS, + '$collection' => Database::METADATA, '$id' => 'tags', 'name' => 'Tags', 'attributes' => [ @@ -1202,7 +1202,7 @@ $collections = [ ], 'executions' => [ - '$collection' => Database::COLLECTIONS, + '$collection' => Database::METADATA, '$id' => 'executions', 'name' => 'Executions', 'attributes' => [ @@ -1319,7 +1319,7 @@ $collections = [ ], 'certificates' => [ - '$collection' => Database::COLLECTIONS, + '$collection' => Database::METADATA, '$id' => 'certificates', 'name' => 'Certificates', 'attributes' => [ @@ -1404,7 +1404,7 @@ $collections = [ ], ], 'stats' => [ - '$collection' => Database::COLLECTIONS, + '$collection' => Database::METADATA, '$id' => 'stats', 'name' => 'Stats', 'attributes' => [ From cfe49b68927fb6b2be41e90dff2362d1071f4a05 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 20 Aug 2021 13:54:45 +0530 Subject: [PATCH 054/206] feat(usage): refactored functions usage endpoint --- app/controllers/api/database.php | 261 ++++++++++++++++++++---------- app/controllers/api/functions.php | 95 +++++++++++ app/controllers/api/projects.php | 2 +- 3 files changed, 273 insertions(+), 85 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index db8a97a9ec..86f4029cdb 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -1,5 +1,6 @@ isValid($default)) { throw new Exception('Length of default attribute exceeds attribute size', 400); } - } + } if (!\is_null($format)) { $name = \json_decode($format, true)['name']; @@ -110,16 +111,14 @@ $attributesCallback = function ($attribute, $response, $dbForExternal, $database $database ->setParam('type', DATABASE_TYPE_CREATE_ATTRIBUTE) - ->setParam('document', $attribute) - ; + ->setParam('document', $attribute); $usage->setParam('database.collections.update', 1); $audits ->setParam('event', 'database.attributes.create') - ->setParam('resource', 'database/collection/'.$collection->getId()) - ->setParam('data', $attribute) - ; + ->setParam('resource', 'database/collection/' . $collection->getId()) + ->setParam('data', $attribute); $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($attribute, Response::MODEL_ATTRIBUTE); @@ -167,9 +166,8 @@ App::post('/v1/database/collections') $audits ->setParam('event', 'database.collections.create') - ->setParam('resource', 'database/collection/'.$collection->getId()) - ->setParam('data', $collection->getArrayCopy()) - ; + ->setParam('resource', 'database/collection/' . $collection->getId()) + ->setParam('data', $collection->getArrayCopy()); $usage->setParam('database.collections.create', 1); @@ -250,6 +248,112 @@ App::get('/v1/database/collections/:collectionId') $response->dynamic($collection, Response::MODEL_COLLECTION); }); +App::get('/v1/database/usage') + ->desc('Get Database Usage') + ->groups(['api', 'database']) + ->label('scope', 'collections.read') + ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) + ->label('sdk.namespace', 'database') + ->label('sdk.method', 'getUsage') + ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) + ->inject('response') + ->inject('dbForConsole') + ->inject('dbForInternal') + ->inject('register') + ->action(function ($range, $response, $dbForConsole, $dbForInternal, $register) { + /** @var Appwrite\Utopia\Response $response */ + /** @var Utopia\Database\Database $dbForConsole */ + /** @var Utopia\Database\Database $dbForInternal */ + /** @var Utopia\Registry\Registry $register */ + + $stats = []; + if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { + $period = [ + '24h' => [ + 'period' => '30m', + 'limit' => 48, + ], + '7d' => [ + 'period' => '1d', + 'limit' => 7, + ], + '30d' => [ + 'period' => '1d', + 'limit' => 30, + ], + '90d' => [ + 'period' => '1d', + 'limit' => 90, + ], + ]; + + Authorization::disable(); + + $metrics = [ + 'database.collections.create', + 'database.collections.read', + 'database.collections.update', + 'database.collections.delete', + 'database.documents.create', + 'database.documents.read', + 'database.documents.update', + 'database.documents.delete' + ]; + + foreach ($metrics as $metric) { + $requestDocs = $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, [$metric]), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + + $stats[$metric] = array_reverse($stats[$metric]); + } + } + + $documentsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['database.documents.count'])], 0, ['time'], [Database::ORDER_DESC]); + $documentsTotal = $documentsCount ? $documentsCount->getAttribute('value', 0) : 0; + + $collectionsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['n + '])], 0, ['time'], [Database::ORDER_DESC]); + $collectionsTotal = $collectionsCount ? $collectionsCount->getAttribute('value', 0) : 0; + + Authorization::reset(); + + $response->json([ + 'range' => $range, + 'stats' => $stats, + 'requests' => [ + 'data' => $stats['requests'] ?? [], + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $stats['requests'] ?? [])), + ], + 'documents' => [ + 'data' => [], + 'total' => $documentsTotal, + ], + 'collections' => [ + 'data' => [], + 'total' => $collectionsTotal, + ] + ]); + }); + + +// :collectionId/usage +// 'reads', +// 'writes', +// 'updates', +// 'delete' + App::get('/v1/database/collections/:collectionId/logs') ->desc('List Collection Logs') ->groups(['api', 'database']) @@ -283,7 +387,7 @@ App::get('/v1/database/collections/:collectionId/logs') $audit = new Audit($dbForInternal); - $logs = $audit->getLogsByResource('database/collection/'.$collection->getId()); + $logs = $audit->getLogsByResource('database/collection/' . $collection->getId()); $output = []; @@ -335,8 +439,8 @@ App::get('/v1/database/collections/:collectionId/logs') $record = $geodb->get($log['ip']); if ($record) { - $output[$i]['countryCode'] = $locale->getText('countries.'.strtolower($record['country']['iso_code']), false) ? \strtolower($record['country']['iso_code']) : '--'; - $output[$i]['countryName'] = $locale->getText('countries.'.strtolower($record['country']['iso_code']), $locale->getText('locale.country.unknown')); + $output[$i]['countryCode'] = $locale->getText('countries.' . strtolower($record['country']['iso_code']), false) ? \strtolower($record['country']['iso_code']) : '--'; + $output[$i]['countryName'] = $locale->getText('countries.' . strtolower($record['country']['iso_code']), $locale->getText('locale.country.unknown')); } else { $output[$i]['countryCode'] = '--'; $output[$i]['countryName'] = $locale->getText('locale.country.unknown'); @@ -390,16 +494,15 @@ App::put('/v1/database/collections/:collectionId') } catch (AuthorizationException $exception) { throw new Exception('Unauthorized permissions', 401); } catch (StructureException $exception) { - throw new Exception('Bad structure. '.$exception->getMessage(), 400); - } + throw new Exception('Bad structure. ' . $exception->getMessage(), 400); + } $usage->setParam('database.collections.update', 1); $audits ->setParam('event', 'database.collections.update') - ->setParam('resource', 'database/collection/'.$collection->getId()) - ->setParam('data', $collection->getArrayCopy()) - ; + ->setParam('resource', 'database/collection/' . $collection->getId()) + ->setParam('data', $collection->getArrayCopy()); $response->dynamic($collection, Response::MODEL_COLLECTION); }); @@ -440,14 +543,12 @@ App::delete('/v1/database/collections/:collectionId') $usage->setParam('database.collections.delete', 1); $events - ->setParam('eventData', $response->output($collection, Response::MODEL_COLLECTION)) - ; + ->setParam('eventData', $response->output($collection, Response::MODEL_COLLECTION)); $audits ->setParam('event', 'database.collections.delete') - ->setParam('resource', 'database/collection/'.$collection->getId()) - ->setParam('data', $collection->getArrayCopy()) - ; + ->setParam('resource', 'database/collection/' . $collection->getId()) + ->setParam('data', $collection->getArrayCopy()); $response->noContent(); }); @@ -530,7 +631,7 @@ App::post('/v1/database/collections/:collectionId/attributes/email') 'required' => $required, 'default' => $default, 'array' => $array, - 'format' => \json_encode(['name'=>'email']), + 'format' => \json_encode(['name' => 'email']), ]), $response, $dbForExternal, $database, $audits, $usage); }); @@ -571,7 +672,7 @@ App::post('/v1/database/collections/:collectionId/attributes/ip') 'required' => $required, 'default' => $default, 'array' => $array, - 'format' => \json_encode(['name'=>'ip']), + 'format' => \json_encode(['name' => 'ip']), ]), $response, $dbForExternal, $database, $audits, $usage); }); @@ -613,7 +714,7 @@ App::post('/v1/database/collections/:collectionId/attributes/url') 'required' => $required, 'default' => $default, 'array' => $array, - 'format' => \json_encode(['name'=>'url']), + 'format' => \json_encode(['name' => 'url']), ]), $response, $dbForExternal, $database, $audits, $usage); }); @@ -648,7 +749,7 @@ App::post('/v1/database/collections/:collectionId/attributes/integer') /** @var Appwrite\Event\Event $audits */ /** @var Appwrite\Stats\Stats $usage */ - + return $attributesCallback(new Document([ '$collection' => $collectionId, '$id' => $attributeId, @@ -658,7 +759,7 @@ App::post('/v1/database/collections/:collectionId/attributes/integer') 'default' => $default, 'array' => $array, 'format' => \json_encode([ - 'name'=>'int-range', + 'name' => 'int-range', 'min' => $min, 'max' => $max, ]), @@ -705,7 +806,7 @@ App::post('/v1/database/collections/:collectionId/attributes/float') 'default' => $default, 'array' => $array, 'format' => \json_encode([ - 'name'=>'float-range', + 'name' => 'float-range', 'min' => $min, 'max' => $max, ]), @@ -835,7 +936,7 @@ App::get('/v1/database/collections/:collectionId/attributes/:attributeId') ])]); $usage->setParam('database.collections.read', 1); - + $response->dynamic($attribute, Response::MODEL_ATTRIBUTE); }); @@ -890,20 +991,17 @@ App::delete('/v1/database/collections/:collectionId/attributes/:attributeId') $database ->setParam('type', DATABASE_TYPE_DELETE_ATTRIBUTE) - ->setParam('document', $attribute) - ; + ->setParam('document', $attribute); $usage->setParam('database.collections.update', 1); $events - ->setParam('payload', $response->output($attribute, Response::MODEL_ATTRIBUTE)) - ; + ->setParam('payload', $response->output($attribute, Response::MODEL_ATTRIBUTE)); $audits ->setParam('event', 'database.attributes.delete') - ->setParam('resource', 'database/collection/'.$collection->getId()) - ->setParam('data', $attribute->getArrayCopy()) - ; + ->setParam('resource', 'database/collection/' . $collection->getId()) + ->setParam('data', $attribute->getArrayCopy()); $response->noContent(); }); @@ -984,20 +1082,17 @@ App::post('/v1/database/collections/:collectionId/indexes') $database ->setParam('type', DATABASE_TYPE_CREATE_INDEX) - ->setParam('document', $index) - ; + ->setParam('document', $index); $usage->setParam('database.collections.update', 1); $audits ->setParam('event', 'database.indexes.create') - ->setParam('resource', 'database/collection/'.$collection->getId()) - ->setParam('data', $index->getArrayCopy()) - ; + ->setParam('resource', 'database/collection/' . $collection->getId()) + ->setParam('data', $index->getArrayCopy()); $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($index, Response::MODEL_INDEX); - }); App::get('/v1/database/collections/:collectionId/indexes') @@ -1083,7 +1178,7 @@ App::get('/v1/database/collections/:collectionId/indexes/:indexId') ])]); $usage->setParam('database.collections.read', 1); - + $response->dynamic($index, Response::MODEL_INDEX); }); @@ -1124,7 +1219,7 @@ App::delete('/v1/database/collections/:collectionId/indexes/:indexId') $indexes = $collection->getAttribute('indexes'); // find attribute in collection - $index= null; + $index = null; foreach ($indexes as $i) { if ($i->getId() === $indexId) { $index = $i->setAttribute('$collection', $collectionId); // set the collectionId @@ -1138,20 +1233,17 @@ App::delete('/v1/database/collections/:collectionId/indexes/:indexId') $database ->setParam('type', DATABASE_TYPE_DELETE_INDEX) - ->setParam('document', $index) - ; + ->setParam('document', $index); $usage->setParam('database.collections.update', 1); $events - ->setParam('payload', $response->output($index, Response::MODEL_INDEX)) - ; + ->setParam('payload', $response->output($index, Response::MODEL_INDEX)); $audits ->setParam('event', 'database.indexes.delete') - ->setParam('resource', 'database/collection/'.$collection->getId()) - ->setParam('data', $index->getArrayCopy()) - ; + ->setParam('resource', 'database/collection/' . $collection->getId()) + ->setParam('data', $index->getArrayCopy()); $response->noContent(); }); @@ -1184,7 +1276,7 @@ App::post('/v1/database/collections/:collectionId/documents') /** @var Utopia\Database\Document $user */ /** @var Appwrite\Event\Event $audits */ /** @var Appwrite\Stats\Stats $usage */ - + $data = (\is_string($data)) ? \json_decode($data, true) : $data; // Cast to JSON array if (empty($data)) { @@ -1194,7 +1286,7 @@ App::post('/v1/database/collections/:collectionId/documents') if (isset($data['$id'])) { throw new Exception('$id is not allowed for creating new documents, try update instead', 400); } - + $collection = $dbForExternal->getCollection($collectionId); if ($collection->isEmpty()) { @@ -1203,26 +1295,23 @@ App::post('/v1/database/collections/:collectionId/documents') $data['$collection'] = $collection->getId(); // Adding this param to make API easier for developers $data['$id'] = $documentId == 'unique()' ? $dbForExternal->getId() : $documentId; - $data['$read'] = (is_null($read) && !$user->isEmpty()) ? ['user:'.$user->getId()] : $read ?? []; // By default set read permissions for user - $data['$write'] = (is_null($write) && !$user->isEmpty()) ? ['user:'.$user->getId()] : $write ?? []; // By default set write permissions for user + $data['$read'] = (is_null($read) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $read ?? []; // By default set read permissions for user + $data['$write'] = (is_null($write) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $write ?? []; // By default set write permissions for user try { $document = $dbForExternal->createDocument($collectionId, new Document($data)); - } - catch (StructureException $exception) { + } catch (StructureException $exception) { throw new Exception($exception->getMessage(), 400); } $usage ->setParam('database.documents.create', 1) - ->setParam('collectionId', $collectionId) - ; + ->setParam('collectionId', $collectionId); $audits ->setParam('event', 'database.documents.create') - ->setParam('resource', 'database/document/'.$document->getId()) - ->setParam('data', $document->getArrayCopy()) - ; + ->setParam('resource', 'database/document/' . $document->getId()) + ->setParam('data', $document->getArrayCopy()); $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($document, Response::MODEL_DOCUMENT); @@ -1286,9 +1375,8 @@ App::get('/v1/database/collections/:collectionId/documents') $usage ->setParam('database.documents.read', 1) - ->setParam('collectionId', $collectionId) - ; - + ->setParam('collectionId', $collectionId); + $response->dynamic(new Document([ 'sum' => $dbForExternal->count($collectionId, $queries, APP_LIMIT_COUNT), 'documents' => $dbForExternal->find($collectionId, $queries, $limit, $offset, $orderAttributes, $orderTypes, $afterDocument ?? null), @@ -1329,8 +1417,7 @@ App::get('/v1/database/collections/:collectionId/documents/:documentId') $usage ->setParam('database.documents.read', 1) - ->setParam('collectionId', $collectionId) - ; + ->setParam('collectionId', $collectionId); $response->dynamic($document, Response::MODEL_DOCUMENT); }); @@ -1379,7 +1466,7 @@ App::patch('/v1/database/collections/:collectionId/documents/:documentId') if (empty($data)) { throw new Exception('Missing payload', 400); } - + if (!\is_array($data)) { throw new Exception('Data param should be a valid JSON object', 400); } @@ -1393,24 +1480,20 @@ App::patch('/v1/database/collections/:collectionId/documents/:documentId') try { $document = $dbForExternal->updateDocument($collection->getId(), $document->getId(), new Document($data)); - } - catch (AuthorizationException $exception) { + } catch (AuthorizationException $exception) { throw new Exception('Unauthorized permissions', 401); + } catch (StructureException $exception) { + throw new Exception('Bad structure. ' . $exception->getMessage(), 400); } - catch (StructureException $exception) { - throw new Exception('Bad structure. '.$exception->getMessage(), 400); - } - + $usage ->setParam('database.documents.update', 1) - ->setParam('collectionId', $collectionId) - ; + ->setParam('collectionId', $collectionId); $audits ->setParam('event', 'database.documents.update') - ->setParam('resource', 'database/document/'.$document->getId()) - ->setParam('data', $document->getArrayCopy()) - ; + ->setParam('resource', 'database/document/' . $document->getId()) + ->setParam('data', $document->getArrayCopy()); $response->dynamic($document, Response::MODEL_DOCUMENT); }); @@ -1456,18 +1539,28 @@ App::delete('/v1/database/collections/:collectionId/documents/:documentId') $usage ->setParam('database.documents.delete', 1) - ->setParam('collectionId', $collectionId) - ; + ->setParam('collectionId', $collectionId); $events - ->setParam('eventData', $response->output($document, Response::MODEL_DOCUMENT)) - ; + ->setParam('eventData', $response->output($document, Response::MODEL_DOCUMENT)); $audits ->setParam('event', 'database.documents.delete') - ->setParam('resource', 'database/document/'.$document->getId()) + ->setParam('resource', 'database/document/' . $document->getId()) ->setParam('data', $document->getArrayCopy()) // Audit document in case of malicious or disastrous action ; $response->noContent(); }); + + +// Refactor usage endpoint in Functions API + + +// Create usage endpoint in the Database API + + +// Create usage endpoint in the users API + + +// Create usage endpoint in the Storage API \ No newline at end of file diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index fbd27c47f3..0f7a4f9a99 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -259,6 +259,101 @@ App::get('/v1/functions/:functionId/usage') } }); +App::get('/v1/functions/:functionId/usage2') + ->desc('Get Function Usage') + ->groups(['api', 'functions']) + ->label('scope', 'functions.read') + ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) + ->label('sdk.namespace', 'functions') + ->label('sdk.method', 'getUsage') + ->param('functionId', '', new UID(), 'Function unique ID.') + ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d']), 'Date range.', true) + ->inject('response') + ->inject('project') + ->inject('dbForInternal') + ->inject('register') + ->action(function ($functionId, $range, $response, $project, $dbForInternal, $register) { + /** @var Appwrite\Utopia\Response $response */ + /** @var Utopia\Database\Document $project */ + /** @var Utopia\Database\Database $dbForInternal */ + /** @var Utopia\Registry\Registry $register */ + + $function = $dbForInternal->getDocument('functions', $functionId); + + if ($function->isEmpty()) { + throw new Exception('Function not found', 404); + } + + if(App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { + $period = [ + '24h' => [ + 'period' => '30m', + 'limit' => 48, + ], + '7d' => [ + 'period' => '1d', + 'limit' => 7, + ], + '30d' => [ + 'period' => '1d', + 'limit' => 30, + ], + '90d' => [ + 'period' => '1d', + 'limit' => 90, + ], + ]; + + Authorization::disable(); + + $metrics = ["functions.$functionId.executions", "functions.$functionId.failures", "functions.$functionId.compute"]; + foreach ($metrics as $metric) { + $requestDocs = $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, [$metric]), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + + $stats[$metric] = array_reverse($stats[$metric]); + } + + $executions = $stats["functions.$functionId.executions"] ?? []; + $failures = $stats["functions.$functionId.failures"] ?? []; + $compute = $stats["functions.$functionId.compute"] ?? []; + + $response->json([ + 'range' => $range, + 'executions' => [ + 'data' => $executions, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $executions)), + ], + 'failures' => [ + 'data' => $failures, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $failures)), + ], + 'compute' => [ + 'data' => $compute, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $compute)), + ], + ]); + } else { + $response->json([]); + } + }); + App::put('/v1/functions/:functionId') ->groups(['api', 'functions']) ->desc('Update Function') diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 7d8f4ed54c..ba9d5627b5 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -217,7 +217,7 @@ App::get('/v1/projects/:projectId') }); App::get('/v1/projects/:projectId/usage') - ->desc('Get Project') + ->desc('Get Project Usage') ->groups(['api', 'projects']) ->label('scope', 'projects.read') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) From d654e4cd090f68b4f0320998f945f383f7454fa7 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 20 Aug 2021 14:28:42 +0530 Subject: [PATCH 055/206] feat(usage): added usage endpoint to storage api --- app/controllers/api/storage.php | 102 ++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index ed3c61dcdb..daecfb64e6 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -10,6 +10,7 @@ use Utopia\Validator\HexColor; use Utopia\Cache\Cache; use Utopia\Cache\Adapter\Filesystem; use Appwrite\ClamAV\Network; +use Appwrite\Database\Validator\Authorization; use Appwrite\Database\Validator\CustomId; use Utopia\Database\Document; use Utopia\Database\Validator\UID; @@ -22,6 +23,7 @@ use Utopia\Image\Image; use Appwrite\OpenSSL\OpenSSL; use Appwrite\Utopia\Response; use Utopia\Config\Config; +use Utopia\Database\Database; use Utopia\Database\Query; App::post('/v1/storage/files') @@ -640,4 +642,104 @@ App::delete('/v1/storage/files/:fileId') ; $response->noContent(); + }); + +App::get('/v1/storage/:bucketId/usage') + ->desc('Get Bucket Usage') + ->groups(['api', 'storage']) + ->label('scope', 'storage.read') + ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) + ->label('sdk.namespace', 'storage') + ->label('sdk.method', 'getUsage') + ->param('bucketId', '', new UID(), 'Bucket unique ID.') + ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) + ->inject('response') + ->inject('dbForInternal') + ->inject('register') + ->action(function ($bucketId, $range, $response, $dbForInternal) { + /** @var Appwrite\Utopia\Response $response */ + /** @var Utopia\Database\Database $dbForInternal */ + + $stats = []; + if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { + $period = [ + '24h' => [ + 'period' => '30m', + 'limit' => 48, + ], + '7d' => [ + 'period' => '1d', + 'limit' => 7, + ], + '30d' => [ + 'period' => '1d', + 'limit' => 30, + ], + '90d' => [ + 'period' => '1d', + 'limit' => 90, + ], + ]; + + Authorization::disable(); + + $metrics = [ + "storage.buckets.$bucketId.files.create", + "storage.buckets.$bucketId.files.read", + "storage.buckets.$bucketId.files.update", + "storage.buckets.$bucketId.files.delete" + ]; + + foreach ($metrics as $metric) { + $requestDocs = $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, [$metric]), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + + $stats[$metric] = array_reverse($stats[$metric]); + } + } + + Authorization::reset(); + + $create = $stats["storage.buckets.$bucketId.files.create"] ?? []; + $read = $stats["storage.buckets.$bucketId.files.read"] ?? []; + $update = $stats["storage.buckets.$bucketId.files.update"] ?? []; + $delete = $stats["storage.buckets.$bucketId.files.delete"] ?? []; + + $response->json([ + 'range' => $range, + 'create' => [ + 'data' => $create, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $create)), + ], + 'read' => [ + 'data' => $read, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $read)), + ], + 'update' => [ + 'data' => $update, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $update)), + ], + 'delete' => [ + 'data' => $delete, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $delete)), + ], + ]); }); \ No newline at end of file From 070b7d827013a39a42bf8a09edbc6b40f1ea01c4 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 20 Aug 2021 14:30:39 +0530 Subject: [PATCH 056/206] feat(usage): added usage endpoint to storage api --- app/controllers/api/storage.php | 72 +++++++++++++++++---------------- 1 file changed, 37 insertions(+), 35 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index daecfb64e6..d167ee88b7 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -706,40 +706,42 @@ App::get('/v1/storage/:bucketId/usage') $stats[$metric] = array_reverse($stats[$metric]); } + + Authorization::reset(); + + $create = $stats["storage.buckets.$bucketId.files.create"] ?? []; + $read = $stats["storage.buckets.$bucketId.files.read"] ?? []; + $update = $stats["storage.buckets.$bucketId.files.update"] ?? []; + $delete = $stats["storage.buckets.$bucketId.files.delete"] ?? []; + + $response->json([ + 'range' => $range, + 'create' => [ + 'data' => $create, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $create)), + ], + 'read' => [ + 'data' => $read, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $read)), + ], + 'update' => [ + 'data' => $update, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $update)), + ], + 'delete' => [ + 'data' => $delete, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $delete)), + ], + ]); + } else { + $response->json([]); } - - Authorization::reset(); - - $create = $stats["storage.buckets.$bucketId.files.create"] ?? []; - $read = $stats["storage.buckets.$bucketId.files.read"] ?? []; - $update = $stats["storage.buckets.$bucketId.files.update"] ?? []; - $delete = $stats["storage.buckets.$bucketId.files.delete"] ?? []; - - $response->json([ - 'range' => $range, - 'create' => [ - 'data' => $create, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $create)), - ], - 'read' => [ - 'data' => $read, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $read)), - ], - 'update' => [ - 'data' => $update, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $update)), - ], - 'delete' => [ - 'data' => $delete, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $delete)), - ], - ]); }); \ No newline at end of file From aefbdb05974274eb8217e2ff14b09847af2454c4 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 20 Aug 2021 15:27:46 +0530 Subject: [PATCH 057/206] feat(usage): added usage endpoint to users api --- app/controllers/api/users.php | 127 ++++++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 09d9d1bcdf..45d4eb03ed 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -2,6 +2,7 @@ use Appwrite\Auth\Auth; use Appwrite\Auth\Validator\Password; +use Appwrite\Database\Validator\Authorization; use Appwrite\Utopia\Response; use Utopia\App; use Utopia\Exception; @@ -17,6 +18,8 @@ use Utopia\Database\Exception\Duplicate; use Utopia\Database\Validator\UID; use DeviceDetector\DeviceDetector; use Appwrite\Database\Validator\CustomId; +use Utopia\Database\Database; +use Utopia\Database\Query; App::post('/v1/users') ->desc('Create User') @@ -605,3 +608,127 @@ App::delete('/v1/users/:userId') ; $response->noContent(); }); + +App::get('/v1/users/usage') + ->desc('Get Users Usage') + ->groups(['api', 'users']) + ->label('scope', 'users.read') + ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) + ->label('sdk.namespace', 'users') + ->label('sdk.method', 'getUsage') + ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) + ->inject('response') + ->inject('dbForInternal') + ->inject('register') + ->action(function ($range, $response, $dbForInternal) { + /** @var Appwrite\Utopia\Response $response */ + /** @var Utopia\Database\Database $dbForInternal */ + + $stats = []; + if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { + $period = [ + '24h' => [ + 'period' => '30m', + 'limit' => 48, + ], + '7d' => [ + 'period' => '1d', + 'limit' => 7, + ], + '30d' => [ + 'period' => '1d', + 'limit' => 30, + ], + '90d' => [ + 'period' => '1d', + 'limit' => 90, + ], + ]; + + Authorization::disable(); + + $metrics = [ + "users.create", + "users.read", + "users.update", + "users.delete", + "users.sessions.create", + "users.sessions.delete" + ]; + + foreach ($metrics as $metric) { + $requestDocs = $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, [$metric]), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + + $stats[$metric] = array_reverse($stats[$metric]); + } + + $usersCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['users.count'])], 0, ['time'], [Database::ORDER_DESC]); + $usersTotal = $usersCount ? $usersCount->getAttribute('value', 0) : 0; + + Authorization::reset(); + + $create = $stats["users.create"] ?? []; + $read = $stats["users.read"] ?? []; + $update = $stats["users.update"] ?? []; + $delete = $stats["users.delete"] ?? []; + $sessionsCreate = $stats["users.sessions.create"] ?? []; + $sessionsDelete = $stats["users.sessions.delete"] ?? []; + + $response->json([ + 'range' => $range, + 'users' => [ + 'data' => [], + 'total' => $usersTotal, + ], + 'create' => [ + 'data' => $create, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $create)), + ], + 'read' => [ + 'data' => $read, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $read)), + ], + 'update' => [ + 'data' => $update, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $update)), + ], + 'delete' => [ + 'data' => $delete, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $delete)), + ], + 'sessionsCreate' => [ + 'data' => $sessionsCreate, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $sessionsCreate)), + ], + 'sessionsDelete' => [ + 'data' => $sessionsDelete, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $sessionsDelete)), + ] + ]); + } else { + $response->json([]); + } + }); \ No newline at end of file From f3074bc024b64b043d658146d0be9132b724e64a Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 20 Aug 2021 15:34:57 +0530 Subject: [PATCH 058/206] feat(usage): some refactoring --- app/controllers/api/database.php | 58 ++++++++++++++++--------------- app/controllers/api/functions.php | 2 ++ 2 files changed, 32 insertions(+), 28 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 86f4029cdb..f2bf0329ca 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -316,35 +316,37 @@ App::get('/v1/database/usage') $stats[$metric] = array_reverse($stats[$metric]); } + + $documentsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['database.documents.count'])], 0, ['time'], [Database::ORDER_DESC]); + $documentsTotal = $documentsCount ? $documentsCount->getAttribute('value', 0) : 0; + + $collectionsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['n + '])], 0, ['time'], [Database::ORDER_DESC]); + $collectionsTotal = $collectionsCount ? $collectionsCount->getAttribute('value', 0) : 0; + + Authorization::reset(); + + $response->json([ + 'range' => $range, + 'stats' => $stats, + 'requests' => [ + 'data' => $stats['requests'] ?? [], + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $stats['requests'] ?? [])), + ], + 'documents' => [ + 'data' => [], + 'total' => $documentsTotal, + ], + 'collections' => [ + 'data' => [], + 'total' => $collectionsTotal, + ] + ]); + } else { + $response->json([]); } - - $documentsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['database.documents.count'])], 0, ['time'], [Database::ORDER_DESC]); - $documentsTotal = $documentsCount ? $documentsCount->getAttribute('value', 0) : 0; - - $collectionsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['n - '])], 0, ['time'], [Database::ORDER_DESC]); - $collectionsTotal = $collectionsCount ? $collectionsCount->getAttribute('value', 0) : 0; - - Authorization::reset(); - - $response->json([ - 'range' => $range, - 'stats' => $stats, - 'requests' => [ - 'data' => $stats['requests'] ?? [], - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $stats['requests'] ?? [])), - ], - 'documents' => [ - 'data' => [], - 'total' => $documentsTotal, - ], - 'collections' => [ - 'data' => [], - 'total' => $collectionsTotal, - ] - ]); }); diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 0f7a4f9a99..ee3088ce18 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -323,6 +323,8 @@ App::get('/v1/functions/:functionId/usage2') $stats[$metric] = array_reverse($stats[$metric]); } + + Authorization::reset(); $executions = $stats["functions.$functionId.executions"] ?? []; $failures = $stats["functions.$functionId.failures"] ?? []; From 8b393b11653e0f8cf8ff86050be2890f28cdc2eb Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 20 Aug 2021 16:53:33 +0530 Subject: [PATCH 059/206] feat(usage): added usage endpoint for database --- app/controllers/api/database.php | 65 ++++++++++++++++++++++++++++---- 1 file changed, 57 insertions(+), 8 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index f2bf0329ca..b4238d284a 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -320,26 +320,75 @@ App::get('/v1/database/usage') $documentsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['database.documents.count'])], 0, ['time'], [Database::ORDER_DESC]); $documentsTotal = $documentsCount ? $documentsCount->getAttribute('value', 0) : 0; - $collectionsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['n - '])], 0, ['time'], [Database::ORDER_DESC]); + $collectionsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['database.collections.count'])], 0, ['time'], [Database::ORDER_DESC]); $collectionsTotal = $collectionsCount ? $collectionsCount->getAttribute('value', 0) : 0; Authorization::reset(); + $documentsCreate = $stats["database.documents.create"] ?? []; + $documentsRead = $stats["database.documents.read"] ?? []; + $documentsUpdate = $stats["database.documents.update"] ?? []; + $documentsDelete = $stats["database.documents.delete"] ?? []; + $collectionsCreate = $stats["database.collections.create"] ?? []; + $collectionsRead = $stats["database.collections.read"] ?? []; + $collectionsUpdate = $stats["database.collections.update"] ?? []; + $collectionsDelete = $stats["database.collections.delete"] ?? []; + $response->json([ 'range' => $range, - 'stats' => $stats, - 'requests' => [ - 'data' => $stats['requests'] ?? [], + 'documents.create' => [ + 'data' => $documentsCreate, 'total' => \array_sum(\array_map(function ($item) { return $item['value']; - }, $stats['requests'] ?? [])), + }, $documentsCreate)), ], - 'documents' => [ + 'documents.read' => [ + 'data' => $documentsRead, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $documentsRead)), + ], + 'documents.update' => [ + 'data' => $documentsUpdate, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $documentsUpdate)), + ], + 'documents.delete' => [ + 'data' => $documentsDelete, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $documentsDelete)), + ], + 'collections.create' => [ + 'data' => $collectionsCreate, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $collectionsCreate)), + ], + 'collections.read' => [ + 'data' => $collectionsRead, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $collectionsRead)), + ], + 'collections.update' => [ + 'data' => $collectionsUpdate, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $collectionsUpdate)), + ], + 'collections.delete' => [ + 'data' => $collectionsDelete, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $collectionsDelete)), + ], + 'documentCount' => [ 'data' => [], 'total' => $documentsTotal, ], - 'collections' => [ + 'collectionCount' => [ 'data' => [], 'total' => $collectionsTotal, ] From 2468e6e97d5a1c5a03f88487e227c9bfe0cc38e4 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 20 Aug 2021 17:08:57 +0530 Subject: [PATCH 060/206] feat(usage): added usage endpoint for collections --- app/controllers/api/database.php | 124 +++++++++++++++++++++++++++++-- 1 file changed, 117 insertions(+), 7 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index b4238d284a..5b41fbcf6f 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -257,10 +257,9 @@ App::get('/v1/database/usage') ->label('sdk.method', 'getUsage') ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) ->inject('response') - ->inject('dbForConsole') ->inject('dbForInternal') ->inject('register') - ->action(function ($range, $response, $dbForConsole, $dbForInternal, $register) { + ->action(function ($range, $response, $dbForInternal) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForConsole */ /** @var Utopia\Database\Database $dbForInternal */ @@ -399,11 +398,122 @@ App::get('/v1/database/usage') }); -// :collectionId/usage -// 'reads', -// 'writes', -// 'updates', -// 'delete' +App::get('/v1/database/:collectionId/usage') + ->desc('Get Database Usage') + ->groups(['api', 'database']) + ->label('scope', 'collections.read') + ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) + ->label('sdk.namespace', 'database') + ->label('sdk.method', 'getUsage') + ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) + ->param('collectionId', '', new UID(), 'Collection unique ID.') + ->inject('response') + ->inject('dbForInternal') + ->inject('dbForExternal') + ->action(function ($range, $collectionId, $response, $dbForInternal, $dbForExternal) { + /** @var Appwrite\Utopia\Response $response */ + /** @var Utopia\Database\Database $dbForConsole */ + /** @var Utopia\Database\Database $dbForInternal */ + /** @var Utopia\Registry\Registry $register */ + + $collection = $dbForExternal->getCollection($collectionId); + + if ($collection->isEmpty()) { + throw new Exception('Collection not found', 404); + } + + $stats = []; + if(App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { + $period = [ + '24h' => [ + 'period' => '30m', + 'limit' => 48, + ], + '7d' => [ + 'period' => '1d', + 'limit' => 7, + ], + '30d' => [ + 'period' => '1d', + 'limit' => 30, + ], + '90d' => [ + 'period' => '1d', + 'limit' => 90, + ], + ]; + + Authorization::disable(); + + $metrics = [ + "database.collections.$collectionId.documents.create", + "database.collections.$collectionId.documents.read", + "database.collections.$collectionId.documents.update", + "database.collections.$collectionId.documents.delete", + ]; + + foreach ($metrics as $metric) { + $requestDocs = $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, [$metric]), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + + $stats[$metric] = array_reverse($stats[$metric]); + } + + $documentsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ["database.collections.$collectionId.documents.count"])], 0, ['time'], [Database::ORDER_DESC]); + $documentsTotal = $documentsCount ? $documentsCount->getAttribute('value', 0) : 0; + + Authorization::reset(); + + $create = $stats["database.collections.$collectionId.documents.create"] ?? []; + $read = $stats["database.collections.$collectionId.documents.read"] ?? []; + $update = $stats["database.collections.$collectionId.documents.update"] ?? []; + $delete = $stats["database.collections.$collectionId.documents.delete"] ?? []; + + $response->json([ + 'range' => $range, + 'create' => [ + 'data' => $create, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $create)), + ], + 'read' => [ + 'data' => $read, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $read)), + ], + 'update' => [ + 'data' => $update, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $update)), + ], + 'delete' => [ + 'data' => $delete, + 'total' => \array_sum(\array_map(function ($item) { + return $item['value']; + }, $delete)), + ], + 'documentCount' => [ + 'data' => [], + 'total' => $documentsTotal, + ], + ]); + } else { + $response->json([]); + } + }); App::get('/v1/database/collections/:collectionId/logs') ->desc('List Collection Logs') From ae030ebd4f383f86983e01be5981c6f116e28446 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 20 Aug 2021 17:10:35 +0530 Subject: [PATCH 061/206] feat(usage): doc fix --- app/controllers/api/database.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 5b41fbcf6f..e65d6929fd 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -399,7 +399,7 @@ App::get('/v1/database/usage') App::get('/v1/database/:collectionId/usage') - ->desc('Get Database Usage') + ->desc('Get Database Usage for a collection') ->groups(['api', 'database']) ->label('scope', 'collections.read') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) From c36b5b8b30716bfdae32fb6c1364a329db38981a Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 20 Aug 2021 17:11:13 +0530 Subject: [PATCH 062/206] feat(usage): cleanup comments --- app/controllers/api/database.php | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index e65d6929fd..c167c2f6dd 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -1712,16 +1712,4 @@ App::delete('/v1/database/collections/:collectionId/documents/:documentId') ; $response->noContent(); - }); - - -// Refactor usage endpoint in Functions API - - -// Create usage endpoint in the Database API - - -// Create usage endpoint in the users API - - -// Create usage endpoint in the Storage API \ No newline at end of file + }); \ No newline at end of file From 24925d2eb259ea003f945c8cf7d6a7f5a0bf6bb8 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 20 Aug 2021 17:40:52 +0530 Subject: [PATCH 063/206] feat(usage): cleanup comments --- app/controllers/api/database.php | 5 ++--- app/controllers/api/storage.php | 4 +++- app/controllers/api/users.php | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index c167c2f6dd..ddbbe7e39b 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -249,7 +249,7 @@ App::get('/v1/database/collections/:collectionId') }); App::get('/v1/database/usage') - ->desc('Get Database Usage') + ->desc('Get usage stats for the database') ->groups(['api', 'database']) ->label('scope', 'collections.read') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) @@ -258,7 +258,6 @@ App::get('/v1/database/usage') ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) ->inject('response') ->inject('dbForInternal') - ->inject('register') ->action(function ($range, $response, $dbForInternal) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForConsole */ @@ -399,7 +398,7 @@ App::get('/v1/database/usage') App::get('/v1/database/:collectionId/usage') - ->desc('Get Database Usage for a collection') + ->desc('Get usage stats for a collection') ->groups(['api', 'database']) ->label('scope', 'collections.read') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index d167ee88b7..0800f2ac65 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -645,7 +645,7 @@ App::delete('/v1/storage/files/:fileId') }); App::get('/v1/storage/:bucketId/usage') - ->desc('Get Bucket Usage') + ->desc('Get usage stats for a storage bucket') ->groups(['api', 'storage']) ->label('scope', 'storage.read') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) @@ -660,6 +660,8 @@ App::get('/v1/storage/:bucketId/usage') /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + // TODO: Check is the storage bucket exists else throw 404 + $stats = []; if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $period = [ diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 45d4eb03ed..6d82c4f472 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -610,7 +610,7 @@ App::delete('/v1/users/:userId') }); App::get('/v1/users/usage') - ->desc('Get Users Usage') + ->desc('Get usage stats for the users API') ->groups(['api', 'users']) ->label('scope', 'users.read') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) From 9ebc5b891b29654f699cb518bfafa017ee4a142e Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 20 Aug 2021 17:47:19 +0530 Subject: [PATCH 064/206] feat(usage): added usage endpoint for storage --- app/controllers/api/storage.php | 62 +++++++++++++++++++++++++++++++-- 1 file changed, 59 insertions(+), 3 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 0800f2ac65..3b433988e0 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -644,6 +644,64 @@ App::delete('/v1/storage/files/:fileId') $response->noContent(); }); +App::get('/v1/storage/usage') + ->desc('Get usage stats for storage') + ->groups(['api', 'storage']) + ->label('scope', 'storage.read') + ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) + ->label('sdk.namespace', 'storage') + ->label('sdk.method', 'getUsage') + ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) + ->inject('response') + ->inject('dbForInternal') + ->action(function ($range, $response, $dbForInternal) { + /** @var Appwrite\Utopia\Response $response */ + /** @var Utopia\Database\Database $dbForInternal */ + + if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { + $period = [ + '24h' => [ + 'period' => '30m', + 'limit' => 48, + ], + '7d' => [ + 'period' => '1d', + 'limit' => 7, + ], + '30d' => [ + 'period' => '1d', + 'limit' => 30, + ], + '90d' => [ + 'period' => '1d', + 'limit' => 90, + ], + ]; + + Authorization::disable(); + + $storageTotal = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['storage.total'])], 0, ['time'], [Database::ORDER_DESC]); + $storage = $storageTotal ? $storageTotal->getAttribute('value', 0) : 0; + + $filesCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['storage.files.count'])], 0, ['time'], [Database::ORDER_DESC]); + $filesTotal = $filesCount ? $filesCount->getAttribute('value', 0) : 0; + + Authorization::reset(); + + $response->json([ + 'range' => $range, + 'storage' => [ + 'total' => $storage, + ], + 'files' => [ + 'total' => $filesTotal, + ], + ]); + } else { + $response->json([]); + } + }); + App::get('/v1/storage/:bucketId/usage') ->desc('Get usage stats for a storage bucket') ->groups(['api', 'storage']) @@ -655,13 +713,11 @@ App::get('/v1/storage/:bucketId/usage') ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) ->inject('response') ->inject('dbForInternal') - ->inject('register') ->action(function ($bucketId, $range, $response, $dbForInternal) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ - // TODO: Check is the storage bucket exists else throw 404 - + // TODO: Check if the storage bucket exists else throw 404 $stats = []; if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $period = [ From e7eb0e7cdfdad9cbd1c9f0f5c02c2b8b82746141 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 20 Aug 2021 17:47:56 +0530 Subject: [PATCH 065/206] feat(usage): added usage endpoint for storage --- app/controllers/api/projects.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index ba9d5627b5..45b7d7b010 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -217,7 +217,7 @@ App::get('/v1/projects/:projectId') }); App::get('/v1/projects/:projectId/usage') - ->desc('Get Project Usage') + ->desc('Get usage stats for a project') ->groups(['api', 'projects']) ->label('scope', 'projects.read') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) From a2a73d87e25a3efd14000d81984cb221613ea81d Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 20 Aug 2021 17:52:04 +0530 Subject: [PATCH 066/206] feat(usage): added usage endpoint for database --- app/controllers/api/database.php | 149 ++++++++++++++++++------------- 1 file changed, 87 insertions(+), 62 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index ddbbe7e39b..59083546a1 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -61,7 +61,7 @@ $attributesCallback = function ($attribute, $response, $dbForExternal, $database if (!$validator->isValid($default)) { throw new Exception('Length of default attribute exceeds attribute size', 400); } - } + } if (!\is_null($format)) { $name = \json_decode($format, true)['name']; @@ -111,14 +111,16 @@ $attributesCallback = function ($attribute, $response, $dbForExternal, $database $database ->setParam('type', DATABASE_TYPE_CREATE_ATTRIBUTE) - ->setParam('document', $attribute); + ->setParam('document', $attribute) + ; $usage->setParam('database.collections.update', 1); $audits ->setParam('event', 'database.attributes.create') - ->setParam('resource', 'database/collection/' . $collection->getId()) - ->setParam('data', $attribute); + ->setParam('resource', 'database/collection/'.$collection->getId()) + ->setParam('data', $attribute) + ; $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($attribute, Response::MODEL_ATTRIBUTE); @@ -166,8 +168,9 @@ App::post('/v1/database/collections') $audits ->setParam('event', 'database.collections.create') - ->setParam('resource', 'database/collection/' . $collection->getId()) - ->setParam('data', $collection->getArrayCopy()); + ->setParam('resource', 'database/collection/'.$collection->getId()) + ->setParam('data', $collection->getArrayCopy()) + ; $usage->setParam('database.collections.create', 1); @@ -248,7 +251,7 @@ App::get('/v1/database/collections/:collectionId') $response->dynamic($collection, Response::MODEL_COLLECTION); }); -App::get('/v1/database/usage') + App::get('/v1/database/usage') ->desc('Get usage stats for the database') ->groups(['api', 'database']) ->label('scope', 'collections.read') @@ -396,7 +399,6 @@ App::get('/v1/database/usage') } }); - App::get('/v1/database/:collectionId/usage') ->desc('Get usage stats for a collection') ->groups(['api', 'database']) @@ -547,7 +549,7 @@ App::get('/v1/database/collections/:collectionId/logs') $audit = new Audit($dbForInternal); - $logs = $audit->getLogsByResource('database/collection/' . $collection->getId()); + $logs = $audit->getLogsByResource('database/collection/'.$collection->getId()); $output = []; @@ -599,8 +601,8 @@ App::get('/v1/database/collections/:collectionId/logs') $record = $geodb->get($log['ip']); if ($record) { - $output[$i]['countryCode'] = $locale->getText('countries.' . strtolower($record['country']['iso_code']), false) ? \strtolower($record['country']['iso_code']) : '--'; - $output[$i]['countryName'] = $locale->getText('countries.' . strtolower($record['country']['iso_code']), $locale->getText('locale.country.unknown')); + $output[$i]['countryCode'] = $locale->getText('countries.'.strtolower($record['country']['iso_code']), false) ? \strtolower($record['country']['iso_code']) : '--'; + $output[$i]['countryName'] = $locale->getText('countries.'.strtolower($record['country']['iso_code']), $locale->getText('locale.country.unknown')); } else { $output[$i]['countryCode'] = '--'; $output[$i]['countryName'] = $locale->getText('locale.country.unknown'); @@ -654,15 +656,16 @@ App::put('/v1/database/collections/:collectionId') } catch (AuthorizationException $exception) { throw new Exception('Unauthorized permissions', 401); } catch (StructureException $exception) { - throw new Exception('Bad structure. ' . $exception->getMessage(), 400); - } + throw new Exception('Bad structure. '.$exception->getMessage(), 400); + } $usage->setParam('database.collections.update', 1); $audits ->setParam('event', 'database.collections.update') - ->setParam('resource', 'database/collection/' . $collection->getId()) - ->setParam('data', $collection->getArrayCopy()); + ->setParam('resource', 'database/collection/'.$collection->getId()) + ->setParam('data', $collection->getArrayCopy()) + ; $response->dynamic($collection, Response::MODEL_COLLECTION); }); @@ -703,12 +706,14 @@ App::delete('/v1/database/collections/:collectionId') $usage->setParam('database.collections.delete', 1); $events - ->setParam('eventData', $response->output($collection, Response::MODEL_COLLECTION)); + ->setParam('eventData', $response->output($collection, Response::MODEL_COLLECTION)) + ; $audits ->setParam('event', 'database.collections.delete') - ->setParam('resource', 'database/collection/' . $collection->getId()) - ->setParam('data', $collection->getArrayCopy()); + ->setParam('resource', 'database/collection/'.$collection->getId()) + ->setParam('data', $collection->getArrayCopy()) + ; $response->noContent(); }); @@ -791,7 +796,7 @@ App::post('/v1/database/collections/:collectionId/attributes/email') 'required' => $required, 'default' => $default, 'array' => $array, - 'format' => \json_encode(['name' => 'email']), + 'format' => \json_encode(['name'=>'email']), ]), $response, $dbForExternal, $database, $audits, $usage); }); @@ -832,7 +837,7 @@ App::post('/v1/database/collections/:collectionId/attributes/ip') 'required' => $required, 'default' => $default, 'array' => $array, - 'format' => \json_encode(['name' => 'ip']), + 'format' => \json_encode(['name'=>'ip']), ]), $response, $dbForExternal, $database, $audits, $usage); }); @@ -874,7 +879,7 @@ App::post('/v1/database/collections/:collectionId/attributes/url') 'required' => $required, 'default' => $default, 'array' => $array, - 'format' => \json_encode(['name' => 'url']), + 'format' => \json_encode(['name'=>'url']), ]), $response, $dbForExternal, $database, $audits, $usage); }); @@ -909,7 +914,7 @@ App::post('/v1/database/collections/:collectionId/attributes/integer') /** @var Appwrite\Event\Event $audits */ /** @var Appwrite\Stats\Stats $usage */ - + return $attributesCallback(new Document([ '$collection' => $collectionId, '$id' => $attributeId, @@ -919,7 +924,7 @@ App::post('/v1/database/collections/:collectionId/attributes/integer') 'default' => $default, 'array' => $array, 'format' => \json_encode([ - 'name' => 'int-range', + 'name'=>'int-range', 'min' => $min, 'max' => $max, ]), @@ -966,7 +971,7 @@ App::post('/v1/database/collections/:collectionId/attributes/float') 'default' => $default, 'array' => $array, 'format' => \json_encode([ - 'name' => 'float-range', + 'name'=>'float-range', 'min' => $min, 'max' => $max, ]), @@ -1096,7 +1101,7 @@ App::get('/v1/database/collections/:collectionId/attributes/:attributeId') ])]); $usage->setParam('database.collections.read', 1); - + $response->dynamic($attribute, Response::MODEL_ATTRIBUTE); }); @@ -1151,17 +1156,20 @@ App::delete('/v1/database/collections/:collectionId/attributes/:attributeId') $database ->setParam('type', DATABASE_TYPE_DELETE_ATTRIBUTE) - ->setParam('document', $attribute); + ->setParam('document', $attribute) + ; $usage->setParam('database.collections.update', 1); $events - ->setParam('payload', $response->output($attribute, Response::MODEL_ATTRIBUTE)); + ->setParam('payload', $response->output($attribute, Response::MODEL_ATTRIBUTE)) + ; $audits ->setParam('event', 'database.attributes.delete') - ->setParam('resource', 'database/collection/' . $collection->getId()) - ->setParam('data', $attribute->getArrayCopy()); + ->setParam('resource', 'database/collection/'.$collection->getId()) + ->setParam('data', $attribute->getArrayCopy()) + ; $response->noContent(); }); @@ -1242,17 +1250,20 @@ App::post('/v1/database/collections/:collectionId/indexes') $database ->setParam('type', DATABASE_TYPE_CREATE_INDEX) - ->setParam('document', $index); + ->setParam('document', $index) + ; $usage->setParam('database.collections.update', 1); $audits ->setParam('event', 'database.indexes.create') - ->setParam('resource', 'database/collection/' . $collection->getId()) - ->setParam('data', $index->getArrayCopy()); + ->setParam('resource', 'database/collection/'.$collection->getId()) + ->setParam('data', $index->getArrayCopy()) + ; $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($index, Response::MODEL_INDEX); + }); App::get('/v1/database/collections/:collectionId/indexes') @@ -1338,7 +1349,7 @@ App::get('/v1/database/collections/:collectionId/indexes/:indexId') ])]); $usage->setParam('database.collections.read', 1); - + $response->dynamic($index, Response::MODEL_INDEX); }); @@ -1379,7 +1390,7 @@ App::delete('/v1/database/collections/:collectionId/indexes/:indexId') $indexes = $collection->getAttribute('indexes'); // find attribute in collection - $index = null; + $index= null; foreach ($indexes as $i) { if ($i->getId() === $indexId) { $index = $i->setAttribute('$collection', $collectionId); // set the collectionId @@ -1393,17 +1404,20 @@ App::delete('/v1/database/collections/:collectionId/indexes/:indexId') $database ->setParam('type', DATABASE_TYPE_DELETE_INDEX) - ->setParam('document', $index); + ->setParam('document', $index) + ; $usage->setParam('database.collections.update', 1); $events - ->setParam('payload', $response->output($index, Response::MODEL_INDEX)); + ->setParam('payload', $response->output($index, Response::MODEL_INDEX)) + ; $audits ->setParam('event', 'database.indexes.delete') - ->setParam('resource', 'database/collection/' . $collection->getId()) - ->setParam('data', $index->getArrayCopy()); + ->setParam('resource', 'database/collection/'.$collection->getId()) + ->setParam('data', $index->getArrayCopy()) + ; $response->noContent(); }); @@ -1436,7 +1450,7 @@ App::post('/v1/database/collections/:collectionId/documents') /** @var Utopia\Database\Document $user */ /** @var Appwrite\Event\Event $audits */ /** @var Appwrite\Stats\Stats $usage */ - + $data = (\is_string($data)) ? \json_decode($data, true) : $data; // Cast to JSON array if (empty($data)) { @@ -1446,7 +1460,7 @@ App::post('/v1/database/collections/:collectionId/documents') if (isset($data['$id'])) { throw new Exception('$id is not allowed for creating new documents, try update instead', 400); } - + $collection = $dbForExternal->getCollection($collectionId); if ($collection->isEmpty()) { @@ -1455,23 +1469,26 @@ App::post('/v1/database/collections/:collectionId/documents') $data['$collection'] = $collection->getId(); // Adding this param to make API easier for developers $data['$id'] = $documentId == 'unique()' ? $dbForExternal->getId() : $documentId; - $data['$read'] = (is_null($read) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $read ?? []; // By default set read permissions for user - $data['$write'] = (is_null($write) && !$user->isEmpty()) ? ['user:' . $user->getId()] : $write ?? []; // By default set write permissions for user + $data['$read'] = (is_null($read) && !$user->isEmpty()) ? ['user:'.$user->getId()] : $read ?? []; // By default set read permissions for user + $data['$write'] = (is_null($write) && !$user->isEmpty()) ? ['user:'.$user->getId()] : $write ?? []; // By default set write permissions for user try { $document = $dbForExternal->createDocument($collectionId, new Document($data)); - } catch (StructureException $exception) { + } + catch (StructureException $exception) { throw new Exception($exception->getMessage(), 400); } $usage ->setParam('database.documents.create', 1) - ->setParam('collectionId', $collectionId); + ->setParam('collectionId', $collectionId) + ; $audits ->setParam('event', 'database.documents.create') - ->setParam('resource', 'database/document/' . $document->getId()) - ->setParam('data', $document->getArrayCopy()); + ->setParam('resource', 'database/document/'.$document->getId()) + ->setParam('data', $document->getArrayCopy()) + ; $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($document, Response::MODEL_DOCUMENT); @@ -1535,8 +1552,9 @@ App::get('/v1/database/collections/:collectionId/documents') $usage ->setParam('database.documents.read', 1) - ->setParam('collectionId', $collectionId); - + ->setParam('collectionId', $collectionId) + ; + $response->dynamic(new Document([ 'sum' => $dbForExternal->count($collectionId, $queries, APP_LIMIT_COUNT), 'documents' => $dbForExternal->find($collectionId, $queries, $limit, $offset, $orderAttributes, $orderTypes, $afterDocument ?? null), @@ -1577,7 +1595,8 @@ App::get('/v1/database/collections/:collectionId/documents/:documentId') $usage ->setParam('database.documents.read', 1) - ->setParam('collectionId', $collectionId); + ->setParam('collectionId', $collectionId) + ; $response->dynamic($document, Response::MODEL_DOCUMENT); }); @@ -1626,7 +1645,7 @@ App::patch('/v1/database/collections/:collectionId/documents/:documentId') if (empty($data)) { throw new Exception('Missing payload', 400); } - + if (!\is_array($data)) { throw new Exception('Data param should be a valid JSON object', 400); } @@ -1640,20 +1659,24 @@ App::patch('/v1/database/collections/:collectionId/documents/:documentId') try { $document = $dbForExternal->updateDocument($collection->getId(), $document->getId(), new Document($data)); - } catch (AuthorizationException $exception) { - throw new Exception('Unauthorized permissions', 401); - } catch (StructureException $exception) { - throw new Exception('Bad structure. ' . $exception->getMessage(), 400); } - + catch (AuthorizationException $exception) { + throw new Exception('Unauthorized permissions', 401); + } + catch (StructureException $exception) { + throw new Exception('Bad structure. '.$exception->getMessage(), 400); + } + $usage ->setParam('database.documents.update', 1) - ->setParam('collectionId', $collectionId); + ->setParam('collectionId', $collectionId) + ; $audits ->setParam('event', 'database.documents.update') - ->setParam('resource', 'database/document/' . $document->getId()) - ->setParam('data', $document->getArrayCopy()); + ->setParam('resource', 'database/document/'.$document->getId()) + ->setParam('data', $document->getArrayCopy()) + ; $response->dynamic($document, Response::MODEL_DOCUMENT); }); @@ -1699,16 +1722,18 @@ App::delete('/v1/database/collections/:collectionId/documents/:documentId') $usage ->setParam('database.documents.delete', 1) - ->setParam('collectionId', $collectionId); + ->setParam('collectionId', $collectionId) + ; $events - ->setParam('eventData', $response->output($document, Response::MODEL_DOCUMENT)); + ->setParam('eventData', $response->output($document, Response::MODEL_DOCUMENT)) + ; $audits ->setParam('event', 'database.documents.delete') - ->setParam('resource', 'database/document/' . $document->getId()) + ->setParam('resource', 'database/document/'.$document->getId()) ->setParam('data', $document->getArrayCopy()) // Audit document in case of malicious or disastrous action ; $response->noContent(); - }); \ No newline at end of file + }); From cd3939781ecd0c2cb136094f3d0df9244a5b1e51 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 24 Aug 2021 14:21:46 +0545 Subject: [PATCH 067/206] fix session provider usage --- app/tasks/usage.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index a6c9f3edac..fdbf1d3eb2 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -41,6 +41,7 @@ use Utopia\Database\Validator\Authorization; * users.update * users.delete * users.sessions.create + * users.sessions.{provider}.create * users.sessions.delete * * Functions @@ -163,6 +164,9 @@ $cli ], 'users.sessions.create' => [ 'table' => 'appwrite_usage_users_sessions_create', + ], + 'users.sessions.provider.create' => [ + 'table' => 'appwrite_usage_users_sessions_create', 'groupBy' => 'provider', ], 'users.sessions.delete' => [ From 218b332b0f92fef229f0970c6961faac8992adff Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 24 Aug 2021 15:36:45 +0545 Subject: [PATCH 068/206] fix get url --- app/controllers/shared/api.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index fc59d100ef..b7953da491 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -104,7 +104,7 @@ App::init(function ($utopia, $request, $response, $project, $user, $events, $aud ->setParam('httpRequest', 1) ->setParam('httpUrl', $request->getHostname().$request->getURI()) ->setParam('httpMethod', $request->getMethod()) - ->setParam('httpPath', $route->getURL()) + ->setParam('httpPath', $route->getPath()) ->setParam('networkRequestSize', 0) ->setParam('networkResponseSize', 0) ->setParam('storage', 0) From 80fffe0ed49d85ffefc224c4458273c134a857a4 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 24 Aug 2021 16:15:24 +0545 Subject: [PATCH 069/206] collections count --- app/controllers/api/projects.php | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 7d8f4ed54c..7042281679 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -290,6 +290,9 @@ App::get('/v1/projects/:projectId/usage') $documentsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['database.documents.count'])], 0, ['time'], [Database::ORDER_DESC]); $documentsTotal = $documentsCount ? $documentsCount->getAttribute('value', 0) : 0; + + $collectionsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['database.collections.count'])], 0, ['time'], [Database::ORDER_DESC]); + $collectionsTotal = $collectionsCount ? $collectionsCount->getAttribute('value', 0) : 0; $storageTotal = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['storage.total'])], 0, ['time'], [Database::ORDER_DESC]); $storage = $storageTotal ? $storageTotal->getAttribute('value', 0) : 0; @@ -320,6 +323,10 @@ App::get('/v1/projects/:projectId/usage') 'data' => [], 'total' => $documentsTotal, ], + 'collections' => [ + 'data' => [], + 'total' => $collectionsTotal, + ], 'users' => [ 'data' => [], 'total' => $usersTotal, From 1af4b02034da811417ab5a551d91d8e50579ad2b Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 25 Aug 2021 11:44:12 +0545 Subject: [PATCH 070/206] fix warning --- src/Appwrite/Stats/Stats.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Stats/Stats.php b/src/Appwrite/Stats/Stats.php index e7720088f5..0cdd19fee3 100644 --- a/src/Appwrite/Stats/Stats.php +++ b/src/Appwrite/Stats/Stats.php @@ -175,7 +175,7 @@ class Stats } if ($storage >= 1) { - $tags = ",projectId={$projectId},bucketId={($this->params['bucketId'] ?? '')}"; + $tags = ",projectId={$projectId},bucketId=" . ($this->params['bucketId'] ?? ''); $this->statsd->count('storage.all' . $tags, $storage); } From 95a20159a7684fe4a2cedbe354cf90cc1157ddb6 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 25 Aug 2021 12:09:02 +0545 Subject: [PATCH 071/206] fix merge errors --- app/controllers/api/database.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 9dd7e712e8..d403a85084 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -186,7 +186,6 @@ App::get('/v1/database/collections') ->param('after', '', new UID(), 'ID of the collection used as the starting point for the query, excluding the collection itself. Should be used for efficient pagination when working with large sets of data.', true) ->param('orderType', 'ASC', new WhiteList(['ASC', 'DESC'], true), 'Order result by ASC or DESC order.', true) ->inject('response') - ->inject('dbForExternal') ->inject('dbForInternal') ->inject('usage') ->action(function ($search, $limit, $offset, $after, $orderType, $response, $dbForInternal, $usage) { @@ -228,7 +227,6 @@ App::get('/v1/database/collections/:collectionId') ->label('sdk.response.model', Response::MODEL_COLLECTION) ->param('collectionId', '', new UID(), 'Collection unique ID.') ->inject('response') - ->inject('dbForExternal') ->inject('dbForInternal') ->inject('usage') ->action(function ($collectionId, $response, $dbForInternal, $usage) { From 7775daa478a4d1f30cff6f3252d6d949f14ad568 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 26 Aug 2021 16:30:03 +0530 Subject: [PATCH 072/206] feat(usage): added new response model --- app/controllers/api/database.php | 2 - composer.json | 2 +- composer.lock | 14 ++--- docker-compose.yml | 4 +- src/Appwrite/Utopia/Response.php | 1 + src/Appwrite/Utopia/Response/Model/Metric.php | 53 +++++++++++++++++++ 6 files changed, 64 insertions(+), 12 deletions(-) create mode 100644 src/Appwrite/Utopia/Response/Model/Metric.php diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 2a85899a25..edbb9aa5a6 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -1,8 +1,6 @@ addRule('message', [ + 'type' => self::TYPE_STRING, + 'description' => 'Error message.', + 'default' => '', + 'example' => 'Not found', + ]) + ->addRule('code', [ + 'type' => self::TYPE_STRING, + 'description' => 'Error code.', + 'default' => '', + 'example' => '404', + ]) + ->addRule('version', [ + 'type' => self::TYPE_STRING, + 'description' => 'Server version number.', + 'default' => '', + 'example' => '1.0', + ]) + ; + } + + /** + * Get Name + * + * @return string + */ + public function getName():string + { + return 'Metric'; + } + + /** + * Get Collection + * + * @return string + */ + public function getType():string + { + return Response::MODEL_METRIC; + } +} \ No newline at end of file From 3d512e74bf61119ba4bc9aa2f2c2cb1c2fc7bc05 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 26 Aug 2021 23:45:36 +0530 Subject: [PATCH 073/206] feat(usage): added response models for database api --- app/controllers/api/database.php | 157 +++++------------- app/controllers/api/storage.php | 27 ++- src/Appwrite/Utopia/Response.php | 7 + .../Utopia/Response/Model/CollectionUsage.php | 77 +++++++++ .../Utopia/Response/Model/DatabaseUsage.php | 112 +++++++++++++ src/Appwrite/Utopia/Response/Model/Metric.php | 26 ++- 6 files changed, 269 insertions(+), 137 deletions(-) create mode 100644 src/Appwrite/Utopia/Response/Model/CollectionUsage.php create mode 100644 src/Appwrite/Utopia/Response/Model/DatabaseUsage.php diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index edbb9aa5a6..a18c2e885c 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -245,13 +245,16 @@ App::get('/v1/database/collections/:collectionId') $response->dynamic($collection, Response::MODEL_COLLECTION); }); - App::get('/v1/database/usage') +App::get('/v1/database/usage') ->desc('Get usage stats for the database') ->groups(['api', 'database']) ->label('scope', 'collections.read') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'database') ->label('sdk.method', 'getUsage') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_DATABASE_USAGE) ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) ->inject('response') ->inject('dbForInternal') @@ -261,11 +264,11 @@ App::get('/v1/database/collections/:collectionId') /** @var Utopia\Database\Database $dbForInternal */ /** @var Utopia\Registry\Registry $register */ - $stats = []; + $usage = []; if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $period = [ '24h' => [ - 'period' => '30m', + 'period' => '15m', 'limit' => 48, ], '7d' => [ @@ -285,6 +288,8 @@ App::get('/v1/database/collections/:collectionId') Authorization::disable(); $metrics = [ + 'database.documents.count', + 'database.collections.count', 'database.collections.create', 'database.collections.read', 'database.collections.update', @@ -295,6 +300,7 @@ App::get('/v1/database/collections/:collectionId') 'database.documents.delete' ]; + $stats = []; foreach ($metrics as $metric) { $requestDocs = $dbForInternal->find('stats', [ new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), @@ -312,85 +318,24 @@ App::get('/v1/database/collections/:collectionId') $stats[$metric] = array_reverse($stats[$metric]); } - $documentsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['database.documents.count'])], 0, ['time'], [Database::ORDER_DESC]); - $documentsTotal = $documentsCount ? $documentsCount->getAttribute('value', 0) : 0; - - $collectionsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['database.collections.count'])], 0, ['time'], [Database::ORDER_DESC]); - $collectionsTotal = $collectionsCount ? $collectionsCount->getAttribute('value', 0) : 0; - Authorization::reset(); - $documentsCreate = $stats["database.documents.create"] ?? []; - $documentsRead = $stats["database.documents.read"] ?? []; - $documentsUpdate = $stats["database.documents.update"] ?? []; - $documentsDelete = $stats["database.documents.delete"] ?? []; - $collectionsCreate = $stats["database.collections.create"] ?? []; - $collectionsRead = $stats["database.collections.read"] ?? []; - $collectionsUpdate = $stats["database.collections.update"] ?? []; - $collectionsDelete = $stats["database.collections.delete"] ?? []; - - $response->json([ + $usage = new Document([ 'range' => $range, - 'documents.create' => [ - 'data' => $documentsCreate, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $documentsCreate)), - ], - 'documents.read' => [ - 'data' => $documentsRead, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $documentsRead)), - ], - 'documents.update' => [ - 'data' => $documentsUpdate, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $documentsUpdate)), - ], - 'documents.delete' => [ - 'data' => $documentsDelete, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $documentsDelete)), - ], - 'collections.create' => [ - 'data' => $collectionsCreate, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $collectionsCreate)), - ], - 'collections.read' => [ - 'data' => $collectionsRead, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $collectionsRead)), - ], - 'collections.update' => [ - 'data' => $collectionsUpdate, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $collectionsUpdate)), - ], - 'collections.delete' => [ - 'data' => $collectionsDelete, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $collectionsDelete)), - ], - 'documentCount' => [ - 'data' => [], - 'total' => $documentsTotal, - ], - 'collectionCount' => [ - 'data' => [], - 'total' => $collectionsTotal, - ] + 'documents.count' => $stats["database.documents.count"], + 'collections.count' => $stats["database.collections.count"], + 'documents.create' => $stats["database.documents.create"], + 'documents.read' => $stats["database.documents.read"], + 'documents.update' => $stats["database.documents.update"], + 'documents.delete' => $stats["database.documents.delete"], + 'collections.create' => $stats["database.collections.create"], + 'collections.read' => $stats["database.collections.read"], + 'collections.update' => $stats["database.collections.update"], + 'collections.delete' => $stats["database.collections.delete"], ]); - } else { - $response->json([]); } + + $response->dynamic($usage, Response::MODEL_DATABASE_USAGE); }); App::get('/v1/database/:collectionId/usage') @@ -400,6 +345,9 @@ App::get('/v1/database/:collectionId/usage') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'database') ->label('sdk.method', 'getUsage') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_COLLECTION_USAGE) ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) ->param('collectionId', '', new UID(), 'Collection unique ID.') ->inject('response') @@ -417,11 +365,11 @@ App::get('/v1/database/:collectionId/usage') throw new Exception('Collection not found', 404); } - $stats = []; + $usage = []; if(App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $period = [ '24h' => [ - 'period' => '30m', + 'period' => '15m', 'limit' => 48, ], '7d' => [ @@ -441,12 +389,14 @@ App::get('/v1/database/:collectionId/usage') Authorization::disable(); $metrics = [ + "database.collections.$collectionId.documents.count", "database.collections.$collectionId.documents.create", "database.collections.$collectionId.documents.read", "database.collections.$collectionId.documents.update", "database.collections.$collectionId.documents.delete", ]; + $stats = []; foreach ($metrics as $metric) { $requestDocs = $dbForInternal->find('stats', [ new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), @@ -460,54 +410,22 @@ App::get('/v1/database/:collectionId/usage') 'date' => $requestDoc->getAttribute('time'), ]; } - $stats[$metric] = array_reverse($stats[$metric]); } - - $documentsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ["database.collections.$collectionId.documents.count"])], 0, ['time'], [Database::ORDER_DESC]); - $documentsTotal = $documentsCount ? $documentsCount->getAttribute('value', 0) : 0; Authorization::reset(); - - $create = $stats["database.collections.$collectionId.documents.create"] ?? []; - $read = $stats["database.collections.$collectionId.documents.read"] ?? []; - $update = $stats["database.collections.$collectionId.documents.update"] ?? []; - $delete = $stats["database.collections.$collectionId.documents.delete"] ?? []; - $response->json([ + $usage = new Document([ 'range' => $range, - 'create' => [ - 'data' => $create, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $create)), - ], - 'read' => [ - 'data' => $read, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $read)), - ], - 'update' => [ - 'data' => $update, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $update)), - ], - 'delete' => [ - 'data' => $delete, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $delete)), - ], - 'documentCount' => [ - 'data' => [], - 'total' => $documentsTotal, - ], + 'documents.count' => $stats["database.collections.$collectionId.documents.count"], + 'documents.create' => $stats["database.collections.$collectionId.documents.create"], + 'documents.read' => $stats["database.collections.$collectionId.documents.read"], + 'documents.update' => $stats["database.collections.$collectionId.documents.update"], + 'documents.delete' => $stats["database.collections.$collectionId.documents.delete"] ]); - } else { - $response->json([]); } + + $response->dynamic($usage, Response::MODEL_COLLECTION_USAGE); }); App::get('/v1/database/collections/:collectionId/logs') @@ -1458,6 +1376,7 @@ App::post('/v1/database/collections/:collectionId/documents') $data['$read'] = (is_null($read) && !$user->isEmpty()) ? ['user:'.$user->getId()] : $read ?? []; // By default set read permissions for user $data['$write'] = (is_null($write) && !$user->isEmpty()) ? ['user:'.$user->getId()] : $write ?? []; // By default set write permissions for user + var_dump($collectionId); try { $document = $dbForExternal->createDocument($collectionId, new Document($data)); } diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 3b433988e0..5589dbe95f 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -647,7 +647,7 @@ App::delete('/v1/storage/files/:fileId') App::get('/v1/storage/usage') ->desc('Get usage stats for storage') ->groups(['api', 'storage']) - ->label('scope', 'storage.read') + ->label('scope', 'files.read') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'storage') ->label('sdk.method', 'getUsage') @@ -705,7 +705,7 @@ App::get('/v1/storage/usage') App::get('/v1/storage/:bucketId/usage') ->desc('Get usage stats for a storage bucket') ->groups(['api', 'storage']) - ->label('scope', 'storage.read') + ->label('scope', 'files.read') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'storage') ->label('sdk.method', 'getUsage') @@ -772,6 +772,29 @@ App::get('/v1/storage/:bucketId/usage') $update = $stats["storage.buckets.$bucketId.files.update"] ?? []; $delete = $stats["storage.buckets.$bucketId.files.delete"] ?? []; + + // 'name' => [ + // [ + // [ + // 'value' => '', + // 'date' => 'unix timestamp' + // ] + // ] + // ] + + // $res = [ + // 'range' => '', + // 'name' => [ + // [ + // 'value' => , + // 'date' => + // ] + // ], + // 'nameTotal' => [ + + // ] + // ]; + $response->json([ 'range' => $range, 'create' => [ diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index f7217a5155..44799fdaa2 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -13,9 +13,11 @@ use Appwrite\Utopia\Response\Model\Any; use Appwrite\Utopia\Response\Model\Attribute; use Appwrite\Utopia\Response\Model\BaseList; use Appwrite\Utopia\Response\Model\Collection; +use Appwrite\Utopia\Response\Model\CollectionUsage; use Appwrite\Utopia\Response\Model\Continent; use Appwrite\Utopia\Response\Model\Country; use Appwrite\Utopia\Response\Model\Currency; +use Appwrite\Utopia\Response\Model\DatabaseUsage; use Appwrite\Utopia\Response\Model\Document as ModelDocument; use Appwrite\Utopia\Response\Model\Domain; use Appwrite\Utopia\Response\Model\Error; @@ -57,6 +59,7 @@ class Response extends SwooleResponse const MODEL_LOG_LIST = 'logList'; const MODEL_ERROR = 'error'; const MODEL_METRIC = 'metric'; + const MODEL_METRIC_LIST = 'metricList'; const MODEL_ERROR_DEV = 'errorDev'; const MODEL_BASE_LIST = 'baseList'; @@ -69,6 +72,8 @@ class Response extends SwooleResponse const MODEL_INDEX_LIST = 'indexList'; const MODEL_DOCUMENT = 'document'; const MODEL_DOCUMENT_LIST = 'documentList'; + const MODEL_DATABASE_USAGE = 'databaseUsage'; + const MODEL_COLLECTION_USAGE = 'collectionUsage'; // Users const MODEL_USER = 'user'; @@ -183,6 +188,8 @@ class Response extends SwooleResponse ->setModel(new Attribute()) ->setModel(new Index()) ->setModel(new ModelDocument()) + ->setModel(new DatabaseUsage()) + ->setModel(new CollectionUsage()) ->setModel(new Log()) ->setModel(new User()) ->setModel(new Preferences()) diff --git a/src/Appwrite/Utopia/Response/Model/CollectionUsage.php b/src/Appwrite/Utopia/Response/Model/CollectionUsage.php new file mode 100644 index 0000000000..2281b3d032 --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/CollectionUsage.php @@ -0,0 +1,77 @@ +addRule('range', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'The value of this metric at a timestamp.', + 'default' => 0, + 'example' => 1, + ]) + ->addRule('documents.count', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for total number of documents.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('documents.create', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for documents created.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('documents.read', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for documents read.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('documents.update', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for documents updated.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('documents.delete', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for documents deleted.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ; + } + + /** + * Get Name + * + * @return string + */ + public function getName():string + { + return 'CollectionUsage'; + } + + /** + * Get Collection + * + * @return string + */ + public function getType():string + { + return Response::MODEL_COLLECTION_USAGE; + } +} \ No newline at end of file diff --git a/src/Appwrite/Utopia/Response/Model/DatabaseUsage.php b/src/Appwrite/Utopia/Response/Model/DatabaseUsage.php new file mode 100644 index 0000000000..58d9575d70 --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/DatabaseUsage.php @@ -0,0 +1,112 @@ +addRule('range', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'The value of this metric at a timestamp.', + 'default' => 0, + 'example' => 1, + ]) + ->addRule('documents.count', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for total number of documents.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('collections.count', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for total number of collections.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('documents.create', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for documents created.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('documents.read', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for documents read.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('documents.update', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for documents updated.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('documents.delete', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for documents deleted.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('collections.create', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for collections created.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('collections.read', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for collections read.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('collections.update', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for collections updated.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('collections.delete', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for collections delete.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ; + } + + /** + * Get Name + * + * @return string + */ + public function getName():string + { + return 'DatabaseUsage'; + } + + /** + * Get Collection + * + * @return string + */ + public function getType():string + { + return Response::MODEL_DATABASE_USAGE; + } +} \ No newline at end of file diff --git a/src/Appwrite/Utopia/Response/Model/Metric.php b/src/Appwrite/Utopia/Response/Model/Metric.php index 73625eb1f3..26038af081 100644 --- a/src/Appwrite/Utopia/Response/Model/Metric.php +++ b/src/Appwrite/Utopia/Response/Model/Metric.php @@ -10,23 +10,17 @@ class Metric extends Model public function __construct() { $this - ->addRule('message', [ - 'type' => self::TYPE_STRING, - 'description' => 'Error message.', - 'default' => '', - 'example' => 'Not found', + ->addRule('value', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'The value of this metric at the timestamp.', + 'default' => -1, + 'example' => 1, ]) - ->addRule('code', [ - 'type' => self::TYPE_STRING, - 'description' => 'Error code.', - 'default' => '', - 'example' => '404', - ]) - ->addRule('version', [ - 'type' => self::TYPE_STRING, - 'description' => 'Server version number.', - 'default' => '', - 'example' => '1.0', + ->addRule('timestamp', [ + 'type' => self::TYPE_INTEGER, + 'description' => 'The UNIX timestamp at which this metric was aggregated.', + 'default' => 0, + 'example' => 1592981250 ]) ; } From 6d6b0ee02418fe6ea7d70eb8b562ef46d256f023 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 27 Aug 2021 00:14:32 +0530 Subject: [PATCH 074/206] feat(usage): added response models for storage API --- app/controllers/api/storage.php | 212 ++++++++---------- src/Appwrite/Utopia/Response.php | 6 + .../Utopia/Response/Model/BucketsUsage.php | 70 ++++++ .../Utopia/Response/Model/CollectionUsage.php | 8 +- .../Utopia/Response/Model/DatabaseUsage.php | 8 +- .../Utopia/Response/Model/StorageUsage.php | 56 +++++ 6 files changed, 230 insertions(+), 130 deletions(-) create mode 100644 src/Appwrite/Utopia/Response/Model/BucketsUsage.php create mode 100644 src/Appwrite/Utopia/Response/Model/StorageUsage.php diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 5589dbe95f..69a7c18708 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -651,6 +651,9 @@ App::get('/v1/storage/usage') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'storage') ->label('sdk.method', 'getUsage') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_STORAGE_USAGE) ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) ->inject('response') ->inject('dbForInternal') @@ -658,71 +661,11 @@ App::get('/v1/storage/usage') /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ + $usage = []; if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $period = [ '24h' => [ - 'period' => '30m', - 'limit' => 48, - ], - '7d' => [ - 'period' => '1d', - 'limit' => 7, - ], - '30d' => [ - 'period' => '1d', - 'limit' => 30, - ], - '90d' => [ - 'period' => '1d', - 'limit' => 90, - ], - ]; - - Authorization::disable(); - - $storageTotal = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['storage.total'])], 0, ['time'], [Database::ORDER_DESC]); - $storage = $storageTotal ? $storageTotal->getAttribute('value', 0) : 0; - - $filesCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['storage.files.count'])], 0, ['time'], [Database::ORDER_DESC]); - $filesTotal = $filesCount ? $filesCount->getAttribute('value', 0) : 0; - - Authorization::reset(); - - $response->json([ - 'range' => $range, - 'storage' => [ - 'total' => $storage, - ], - 'files' => [ - 'total' => $filesTotal, - ], - ]); - } else { - $response->json([]); - } - }); - -App::get('/v1/storage/:bucketId/usage') - ->desc('Get usage stats for a storage bucket') - ->groups(['api', 'storage']) - ->label('scope', 'files.read') - ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) - ->label('sdk.namespace', 'storage') - ->label('sdk.method', 'getUsage') - ->param('bucketId', '', new UID(), 'Bucket unique ID.') - ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) - ->inject('response') - ->inject('dbForInternal') - ->action(function ($bucketId, $range, $response, $dbForInternal) { - /** @var Appwrite\Utopia\Response $response */ - /** @var Utopia\Database\Database $dbForInternal */ - - // TODO: Check if the storage bucket exists else throw 404 - $stats = []; - if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { - $period = [ - '24h' => [ - 'period' => '30m', + 'period' => '15m', 'limit' => 48, ], '7d' => [ @@ -742,12 +685,11 @@ App::get('/v1/storage/:bucketId/usage') Authorization::disable(); $metrics = [ - "storage.buckets.$bucketId.files.create", - "storage.buckets.$bucketId.files.read", - "storage.buckets.$bucketId.files.update", - "storage.buckets.$bucketId.files.delete" + "storage.total", + "storage.files.count" ]; + $stats = []; foreach ($metrics as $metric) { $requestDocs = $dbForInternal->find('stats', [ new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), @@ -761,68 +703,94 @@ App::get('/v1/storage/:bucketId/usage') 'date' => $requestDoc->getAttribute('time'), ]; } - $stats[$metric] = array_reverse($stats[$metric]); } + $usage = new Document([ + 'range' => $range, + 'storage' => $stats['storage.total'], + 'files' => $stats['storage.files.count'] + ]); + } + + $response->dynamic($usage, Response::MODEL_STORAGE_USAGE); + }); + +App::get('/v1/storage/:bucketId/usage') + ->desc('Get usage stats for a storage bucket') + ->groups(['api', 'storage']) + ->label('scope', 'files.read') + ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) + ->label('sdk.namespace', 'storage') + ->label('sdk.method', 'getUsage') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_BUCKETS_USAGE) + ->param('bucketId', '', new UID(), 'Bucket unique ID.') + ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) + ->inject('response') + ->inject('dbForInternal') + ->action(function ($bucketId, $range, $response, $dbForInternal) { + /** @var Appwrite\Utopia\Response $response */ + /** @var Utopia\Database\Database $dbForInternal */ + + // TODO: Check if the storage bucket exists else throw 404 + + $usage = []; + if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { + $period = [ + '24h' => [ + 'period' => '30m', + 'limit' => 48, + ], + '7d' => [ + 'period' => '1d', + 'limit' => 7, + ], + '30d' => [ + 'period' => '1d', + 'limit' => 30, + ], + '90d' => [ + 'period' => '1d', + 'limit' => 90, + ], + ]; + + Authorization::disable(); + $metrics = [ + "storage.buckets.$bucketId.files.create", + "storage.buckets.$bucketId.files.read", + "storage.buckets.$bucketId.files.update", + "storage.buckets.$bucketId.files.delete" + ]; + + $stats = []; + foreach ($metrics as $metric) { + $requestDocs = $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, [$metric]), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + $stats[$metric] = array_reverse($stats[$metric]); + } Authorization::reset(); - $create = $stats["storage.buckets.$bucketId.files.create"] ?? []; - $read = $stats["storage.buckets.$bucketId.files.read"] ?? []; - $update = $stats["storage.buckets.$bucketId.files.update"] ?? []; - $delete = $stats["storage.buckets.$bucketId.files.delete"] ?? []; - - - // 'name' => [ - // [ - // [ - // 'value' => '', - // 'date' => 'unix timestamp' - // ] - // ] - // ] - - // $res = [ - // 'range' => '', - // 'name' => [ - // [ - // 'value' => , - // 'date' => - // ] - // ], - // 'nameTotal' => [ - - // ] - // ]; - - $response->json([ + $usage = new Document([ 'range' => $range, - 'create' => [ - 'data' => $create, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $create)), - ], - 'read' => [ - 'data' => $read, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $read)), - ], - 'update' => [ - 'data' => $update, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $update)), - ], - 'delete' => [ - 'data' => $delete, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $delete)), - ], + 'files.create' => $stats["storage.buckets.$bucketId.files.create"], + 'files.read' => $stats["storage.buckets.$bucketId.files.read"], + 'files.update' => $stats["storage.buckets.$bucketId.files.update"], + 'files.delete' => $stats["storage.buckets.$bucketId.files.delete"] ]); - } else { - $response->json([]); } + + $response->dynamic($usage, Response::MODEL_BUCKETS_USAGE); }); \ No newline at end of file diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index 44799fdaa2..811f07cb55 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -12,6 +12,7 @@ use Appwrite\Utopia\Response\Model\None; use Appwrite\Utopia\Response\Model\Any; use Appwrite\Utopia\Response\Model\Attribute; use Appwrite\Utopia\Response\Model\BaseList; +use Appwrite\Utopia\Response\Model\BucketsUsage; use Appwrite\Utopia\Response\Model\Collection; use Appwrite\Utopia\Response\Model\CollectionUsage; use Appwrite\Utopia\Response\Model\Continent; @@ -45,6 +46,7 @@ use Appwrite\Utopia\Response\Model\Token; use Appwrite\Utopia\Response\Model\Webhook; use Appwrite\Utopia\Response\Model\Preferences; use Appwrite\Utopia\Response\Model\Mock; // Keep last +use Appwrite\Utopia\Response\Model\StorageUsage; use stdClass; /** @@ -88,6 +90,8 @@ class Response extends SwooleResponse const MODEL_FILE = 'file'; const MODEL_FILE_LIST = 'fileList'; const MODEL_BUCKET = 'bucket'; // - Missing + const MODEL_BUCKETS_USAGE = 'bucketsUsage'; + const MODEL_STORAGE_USAGE = 'storageUsage'; // Locale const MODEL_LOCALE = 'locale'; @@ -198,6 +202,8 @@ class Response extends SwooleResponse ->setModel(new JWT()) ->setModel(new Locale()) ->setModel(new File()) + ->setModel(new StorageUsage()) + ->setModel(new BucketsUsage()) ->setModel(new Team()) ->setModel(new Membership()) ->setModel(new Func()) diff --git a/src/Appwrite/Utopia/Response/Model/BucketsUsage.php b/src/Appwrite/Utopia/Response/Model/BucketsUsage.php new file mode 100644 index 0000000000..0fb923e147 --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/BucketsUsage.php @@ -0,0 +1,70 @@ +addRule('range', [ + 'type' => self::TYPE_STRING, + 'description' => 'The time range of the usage stats.', + 'default' => '', + 'example' => '30d', + ]) + ->addRule('files.create', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for files created.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('files.read', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for files read.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('files.update', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for files updated.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('files.delete', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for files deleted.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ; + } + + /** + * Get Name + * + * @return string + */ + public function getName():string + { + return 'BucketsUsage'; + } + + /** + * Get Collection + * + * @return string + */ + public function getType():string + { + return Response::MODEL_BUCKETS_USAGE; + } +} \ No newline at end of file diff --git a/src/Appwrite/Utopia/Response/Model/CollectionUsage.php b/src/Appwrite/Utopia/Response/Model/CollectionUsage.php index 2281b3d032..a0b0a51c9b 100644 --- a/src/Appwrite/Utopia/Response/Model/CollectionUsage.php +++ b/src/Appwrite/Utopia/Response/Model/CollectionUsage.php @@ -12,10 +12,10 @@ class CollectionUsage extends Model { $this ->addRule('range', [ - 'type' => self::TYPE_INTEGER, - 'description' => 'The value of this metric at a timestamp.', - 'default' => 0, - 'example' => 1, + 'type' => self::TYPE_STRING, + 'description' => 'The time range of the usage stats.', + 'default' => '', + 'example' => '30d', ]) ->addRule('documents.count', [ 'type' => Response::MODEL_METRIC_LIST, diff --git a/src/Appwrite/Utopia/Response/Model/DatabaseUsage.php b/src/Appwrite/Utopia/Response/Model/DatabaseUsage.php index 58d9575d70..1c24625004 100644 --- a/src/Appwrite/Utopia/Response/Model/DatabaseUsage.php +++ b/src/Appwrite/Utopia/Response/Model/DatabaseUsage.php @@ -12,10 +12,10 @@ class DatabaseUsage extends Model { $this ->addRule('range', [ - 'type' => self::TYPE_INTEGER, - 'description' => 'The value of this metric at a timestamp.', - 'default' => 0, - 'example' => 1, + 'type' => self::TYPE_STRING, + 'description' => 'The time range of the usage stats.', + 'default' => '', + 'example' => '30d', ]) ->addRule('documents.count', [ 'type' => Response::MODEL_METRIC_LIST, diff --git a/src/Appwrite/Utopia/Response/Model/StorageUsage.php b/src/Appwrite/Utopia/Response/Model/StorageUsage.php new file mode 100644 index 0000000000..e26e0d4b3d --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/StorageUsage.php @@ -0,0 +1,56 @@ +addRule('range', [ + 'type' => self::TYPE_STRING, + 'description' => 'The time range of the usage stats.', + 'default' => '', + 'example' => '30d', + ]) + ->addRule('storage', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for the occupied storage size.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('files', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for total number of files.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ; + } + + /** + * Get Name + * + * @return string + */ + public function getName():string + { + return 'StorageUsage'; + } + + /** + * Get Collection + * + * @return string + */ + public function getType():string + { + return Response::MODEL_STORAGE_USAGE; + } +} \ No newline at end of file From b893190ce52c7859bcf2d22fb17fefe4e5f8b275 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 27 Aug 2021 00:23:55 +0530 Subject: [PATCH 075/206] feat(usage): added response models for users API --- app/controllers/api/users.php | 77 ++++------------ src/Appwrite/Utopia/Response.php | 3 + .../Utopia/Response/Model/UsersUsage.php | 91 +++++++++++++++++++ 3 files changed, 114 insertions(+), 57 deletions(-) create mode 100644 src/Appwrite/Utopia/Response/Model/UsersUsage.php diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 6d82c4f472..5a0cb03209 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -18,6 +18,7 @@ use Utopia\Database\Exception\Duplicate; use Utopia\Database\Validator\UID; use DeviceDetector\DeviceDetector; use Appwrite\Database\Validator\CustomId; +use Appwrite\Utopia\Response\Model; use Utopia\Database\Database; use Utopia\Database\Query; @@ -616,6 +617,9 @@ App::get('/v1/users/usage') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'users') ->label('sdk.method', 'getUsage') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_USERS_USAGE) ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) ->inject('response') ->inject('dbForInternal') @@ -624,11 +628,11 @@ App::get('/v1/users/usage') /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ - $stats = []; + $usage = []; if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $period = [ '24h' => [ - 'period' => '30m', + 'period' => '15m', 'limit' => 48, ], '7d' => [ @@ -646,8 +650,8 @@ App::get('/v1/users/usage') ]; Authorization::disable(); - $metrics = [ + "users.count", "users.create", "users.read", "users.update", @@ -656,6 +660,7 @@ App::get('/v1/users/usage') "users.sessions.delete" ]; + $stats = []; foreach ($metrics as $metric) { $requestDocs = $dbForInternal->find('stats', [ new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), @@ -672,63 +677,21 @@ App::get('/v1/users/usage') $stats[$metric] = array_reverse($stats[$metric]); } - - $usersCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['users.count'])], 0, ['time'], [Database::ORDER_DESC]); - $usersTotal = $usersCount ? $usersCount->getAttribute('value', 0) : 0; - + Authorization::reset(); - $create = $stats["users.create"] ?? []; - $read = $stats["users.read"] ?? []; - $update = $stats["users.update"] ?? []; - $delete = $stats["users.delete"] ?? []; - $sessionsCreate = $stats["users.sessions.create"] ?? []; - $sessionsDelete = $stats["users.sessions.delete"] ?? []; - - $response->json([ + $usage = new Document([ 'range' => $range, - 'users' => [ - 'data' => [], - 'total' => $usersTotal, - ], - 'create' => [ - 'data' => $create, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $create)), - ], - 'read' => [ - 'data' => $read, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $read)), - ], - 'update' => [ - 'data' => $update, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $update)), - ], - 'delete' => [ - 'data' => $delete, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $delete)), - ], - 'sessionsCreate' => [ - 'data' => $sessionsCreate, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $sessionsCreate)), - ], - 'sessionsDelete' => [ - 'data' => $sessionsDelete, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $sessionsDelete)), - ] + 'users.count' => $stats["users.count"], + 'users.create' => $stats["users.create"], + 'users.read' => $stats["users.read"], + 'users.update' => $stats["users.update"], + 'users.delete' => $stats["users.delete"], + 'sessions.create' => $stats["users.sessions.create"], + 'sessions.delete' => $stats["users.sessions.delete"] ]); - } else { - $response->json([]); + } + + $response->dynamic($usage, Response::MODEL_USERS_USAGE); }); \ No newline at end of file diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index 811f07cb55..728e52b9e0 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -47,6 +47,7 @@ use Appwrite\Utopia\Response\Model\Webhook; use Appwrite\Utopia\Response\Model\Preferences; use Appwrite\Utopia\Response\Model\Mock; // Keep last use Appwrite\Utopia\Response\Model\StorageUsage; +use Appwrite\Utopia\Response\Model\UsersUsage; use stdClass; /** @@ -80,6 +81,7 @@ class Response extends SwooleResponse // Users const MODEL_USER = 'user'; const MODEL_USER_LIST = 'userList'; + const MODEL_USERS_USAGE = 'usersUsage'; const MODEL_SESSION = 'session'; const MODEL_SESSION_LIST = 'sessionList'; const MODEL_TOKEN = 'token'; @@ -196,6 +198,7 @@ class Response extends SwooleResponse ->setModel(new CollectionUsage()) ->setModel(new Log()) ->setModel(new User()) + ->setModel(new UsersUsage()) ->setModel(new Preferences()) ->setModel(new Session()) ->setModel(new Token()) diff --git a/src/Appwrite/Utopia/Response/Model/UsersUsage.php b/src/Appwrite/Utopia/Response/Model/UsersUsage.php new file mode 100644 index 0000000000..bb3db30b26 --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/UsersUsage.php @@ -0,0 +1,91 @@ +addRule('range', [ + 'type' => self::TYPE_STRING, + 'description' => 'The time range of the usage stats.', + 'default' => '', + 'example' => '30d', + ]) + ->addRule('users.count', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for total number of users.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('users.create', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for users created.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('users.read', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for users read.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('users.update', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for users updated.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('users.delete', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for users deleted.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('sessions.create', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for sessions created.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('sessions.delete', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for sessions deleted.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ; + } + + /** + * Get Name + * + * @return string + */ + public function getName():string + { + return 'UsersUsage'; + } + + /** + * Get Type + * + * @return string + */ + public function getType():string + { + return Response::MODEL_USERS_USAGE; + } +} \ No newline at end of file From 1e30cdba4b8270f4ec9869ba576e32d574981dae Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 27 Aug 2021 00:42:36 +0530 Subject: [PATCH 076/206] feat(usage): added response models for functions API --- app/controllers/api/functions.php | 165 ++---------------- src/Appwrite/Utopia/Response.php | 3 + .../Utopia/Response/Model/FunctionsUsage.php | 63 +++++++ 3 files changed, 84 insertions(+), 147 deletions(-) create mode 100644 src/Appwrite/Utopia/Response/Model/FunctionsUsage.php diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index ee3088ce18..9359d267f5 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -146,6 +146,9 @@ App::get('/v1/functions/:functionId/usage') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'functions') ->label('sdk.method', 'getUsage') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_FUNCTIONS_USAGE) ->param('functionId', '', new UID(), 'Function unique ID.') ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d']), 'Date range.', true) ->inject('response') @@ -164,130 +167,11 @@ App::get('/v1/functions/:functionId/usage') throw new Exception('Function not found', 404); } + $usage = []; if(App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $period = [ '24h' => [ - 'start' => DateTime::createFromFormat('U', \strtotime('-24 hours')), - 'end' => DateTime::createFromFormat('U', \strtotime('+1 hour')), - 'group' => '30m', - ], - '7d' => [ - 'start' => DateTime::createFromFormat('U', \strtotime('-7 days')), - 'end' => DateTime::createFromFormat('U', \strtotime('now')), - 'group' => '1d', - ], - '30d' => [ - 'start' => DateTime::createFromFormat('U', \strtotime('-30 days')), - 'end' => DateTime::createFromFormat('U', \strtotime('now')), - 'group' => '1d', - ], - '90d' => [ - 'start' => DateTime::createFromFormat('U', \strtotime('-90 days')), - 'end' => DateTime::createFromFormat('U', \strtotime('now')), - 'group' => '1d', - ], - ]; - - $client = $register->get('influxdb'); - - $executions = []; - $failures = []; - $compute = []; - - if ($client) { - $start = $period[$range]['start']->format(DateTime::RFC3339); - $end = $period[$range]['end']->format(DateTime::RFC3339); - $database = $client->selectDB('telegraf'); - - // Executions - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_executions_all" WHERE time > \''.$start.'\' AND time < \''.$end.'\' AND "metric_type"=\'counter\' AND "project"=\''.$project->getId().'\' AND "functionId"=\''.$function->getId().'\' GROUP BY time('.$period[$range]['group'].') FILL(null)'); - $points = $result->getPoints(); - - foreach ($points as $point) { - $executions[] = [ - 'value' => (!empty($point['value'])) ? $point['value'] : 0, - 'date' => \strtotime($point['time']), - ]; - } - - // Failures - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_executions_all" WHERE time > \''.$start.'\' AND time < \''.$end.'\' AND "metric_type"=\'counter\' AND "project"=\''.$project->getId().'\' AND "functionId"=\''.$function->getId().'\' AND "functionStatus"=\'failed\' GROUP BY time('.$period[$range]['group'].') FILL(null)'); - $points = $result->getPoints(); - - foreach ($points as $point) { - $failures[] = [ - 'value' => (!empty($point['value'])) ? $point['value'] : 0, - 'date' => \strtotime($point['time']), - ]; - } - - // Compute - $result = $database->query('SELECT sum(value) AS "value" FROM "appwrite_usage_executions_time" WHERE time > \''.$start.'\' AND time < \''.$end.'\' AND "metric_type"=\'counter\' AND "project"=\''.$project->getId().'\' AND "functionId"=\''.$function->getId().'\' GROUP BY time('.$period[$range]['group'].') FILL(null)'); - $points = $result->getPoints(); - - foreach ($points as $point) { - $compute[] = [ - 'value' => round((!empty($point['value'])) ? $point['value'] / 1000 : 0, 2), // minutes - 'date' => \strtotime($point['time']), - ]; - } - } - - $response->json([ - 'range' => $range, - 'executions' => [ - 'data' => $executions, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $executions)), - ], - 'failures' => [ - 'data' => $failures, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $failures)), - ], - 'compute' => [ - 'data' => $compute, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $compute)), - ], - ]); - } else { - $response->json([]); - } - }); - -App::get('/v1/functions/:functionId/usage2') - ->desc('Get Function Usage') - ->groups(['api', 'functions']) - ->label('scope', 'functions.read') - ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) - ->label('sdk.namespace', 'functions') - ->label('sdk.method', 'getUsage') - ->param('functionId', '', new UID(), 'Function unique ID.') - ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d']), 'Date range.', true) - ->inject('response') - ->inject('project') - ->inject('dbForInternal') - ->inject('register') - ->action(function ($functionId, $range, $response, $project, $dbForInternal, $register) { - /** @var Appwrite\Utopia\Response $response */ - /** @var Utopia\Database\Document $project */ - /** @var Utopia\Database\Database $dbForInternal */ - /** @var Utopia\Registry\Registry $register */ - - $function = $dbForInternal->getDocument('functions', $functionId); - - if ($function->isEmpty()) { - throw new Exception('Function not found', 404); - } - - if(App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { - $period = [ - '24h' => [ - 'period' => '30m', + 'period' => '15m', 'limit' => 48, ], '7d' => [ @@ -306,7 +190,13 @@ App::get('/v1/functions/:functionId/usage2') Authorization::disable(); - $metrics = ["functions.$functionId.executions", "functions.$functionId.failures", "functions.$functionId.compute"]; + $metrics = [ + "functions.$functionId.executions", + "functions.$functionId.failures", + "functions.$functionId.compute" + ]; + + $stats = []; foreach ($metrics as $metric) { $requestDocs = $dbForInternal->find('stats', [ new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), @@ -325,35 +215,16 @@ App::get('/v1/functions/:functionId/usage2') } Authorization::reset(); - - $executions = $stats["functions.$functionId.executions"] ?? []; - $failures = $stats["functions.$functionId.failures"] ?? []; - $compute = $stats["functions.$functionId.compute"] ?? []; - $response->json([ + $usage = new Document([ 'range' => $range, - 'executions' => [ - 'data' => $executions, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $executions)), - ], - 'failures' => [ - 'data' => $failures, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $failures)), - ], - 'compute' => [ - 'data' => $compute, - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $compute)), - ], + 'functions.executions' => $stats["functions.$functionId.executions"], + 'functions.failures' => $stats["functions.$functionId.failures"], + 'functions.compute' => $stats["functions.$functionId.compute"] ]); - } else { - $response->json([]); } + + $response->dynamic($usage, Response::MODEL_FUNCTIONS_USAGE); }); App::put('/v1/functions/:functionId') diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index 728e52b9e0..a1d0d404ed 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -26,6 +26,7 @@ use Appwrite\Utopia\Response\Model\ErrorDev; use Appwrite\Utopia\Response\Model\Execution; use Appwrite\Utopia\Response\Model\File; use Appwrite\Utopia\Response\Model\Func; +use Appwrite\Utopia\Response\Model\FunctionsUsage; use Appwrite\Utopia\Response\Model\Index; use Appwrite\Utopia\Response\Model\JWT; use Appwrite\Utopia\Response\Model\Key; @@ -117,6 +118,7 @@ class Response extends SwooleResponse // Functions const MODEL_FUNCTION = 'function'; const MODEL_FUNCTION_LIST = 'functionList'; + const MODEL_FUNCTIONS_USAGE = 'functionsUsage'; const MODEL_TAG = 'tag'; const MODEL_TAG_LIST = 'tagList'; const MODEL_EXECUTION = 'execution'; @@ -210,6 +212,7 @@ class Response extends SwooleResponse ->setModel(new Team()) ->setModel(new Membership()) ->setModel(new Func()) + ->setModel(new FunctionsUsage()) ->setModel(new Tag()) ->setModel(new Execution()) ->setModel(new Project()) diff --git a/src/Appwrite/Utopia/Response/Model/FunctionsUsage.php b/src/Appwrite/Utopia/Response/Model/FunctionsUsage.php new file mode 100644 index 0000000000..60f017705e --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/FunctionsUsage.php @@ -0,0 +1,63 @@ +addRule('range', [ + 'type' => self::TYPE_STRING, + 'description' => 'The time range of the usage stats.', + 'default' => '', + 'example' => '30d', + ]) + ->addRule('functions.executions', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for function executions.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('functions.failures', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for function execution failures.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('functions.compute', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for function execution duration.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ; + } + + /** + * Get Name + * + * @return string + */ + public function getName():string + { + return 'FunctionsUsage'; + } + + /** + * Get Collection + * + * @return string + */ + public function getType():string + { + return Response::MODEL_FUNCTIONS_USAGE; + } +} \ No newline at end of file From 038aeb01fb39d41b3886c81452174f83f4c0f788 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 27 Aug 2021 01:01:37 +0530 Subject: [PATCH 077/206] feat(usage): added response models for projects API --- app/controllers/api/projects.php | 85 +++++++---------- src/Appwrite/Utopia/Response.php | 3 + .../Utopia/Response/Model/ProjectUsage.php | 91 +++++++++++++++++++ 3 files changed, 125 insertions(+), 54 deletions(-) create mode 100644 src/Appwrite/Utopia/Response/Model/ProjectUsage.php diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index d4a8f86846..e8551c29ce 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -223,6 +223,9 @@ App::get('/v1/projects/:projectId/usage') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'projects') ->label('sdk.method', 'getUsage') + ->label('sdk.response.code', Response::STATUS_CODE_OK) + ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) + ->label('sdk.response.model', Response::MODEL_PROJECT_USAGE) ->param('projectId', '', new UID(), 'Project unique ID.') ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) ->inject('response') @@ -241,11 +244,11 @@ App::get('/v1/projects/:projectId/usage') throw new Exception('Project not found', 404); } - $stats = []; + $usage = []; if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $period = [ '24h' => [ - 'period' => '30m', + 'period' => '15m', 'limit' => 48, ], '7d' => [ @@ -266,7 +269,17 @@ App::get('/v1/projects/:projectId/usage') Authorization::disable(); - $metrics = ['requests', 'network', 'executions']; + $metrics = [ + 'requests', + 'network', + 'executions', + 'users.count', + 'database.documents.count', + 'database.collections.count', + 'storage.total' + ]; + + $stats = []; foreach ($metrics as $metric) { $requestDocs = $dbForInternal->find('stats', [ new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), @@ -282,59 +295,23 @@ App::get('/v1/projects/:projectId/usage') } $stats[$metric] = array_reverse($stats[$metric]); - } + } + + Authorization::reset(); + + $usage = new Document([ + 'range' => $range, + 'requests' => $stats['requests'], + 'network' => $stats['network'], + 'functions' => $stats['executions'], + 'documents' => $stats['database.documents.count'], + 'collections' => $stats['database.collections.count'], + 'users' => $stats['users.count'], + 'storage' => $stats['storage.total'] + ]); } - $usersCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['users.count'])], 0, ['time'], [Database::ORDER_DESC]); - $usersTotal = $usersCount ? $usersCount->getAttribute('value', 0) : 0; - - $documentsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['database.documents.count'])], 0, ['time'], [Database::ORDER_DESC]); - $documentsTotal = $documentsCount ? $documentsCount->getAttribute('value', 0) : 0; - - $collectionsCount = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['database.collections.count'])], 0, ['time'], [Database::ORDER_DESC]); - $collectionsTotal = $collectionsCount ? $collectionsCount->getAttribute('value', 0) : 0; - - $storageTotal = $dbForInternal->findOne('stats', [new Query('metric', Query::TYPE_EQUAL, ['storage.total'])], 0, ['time'], [Database::ORDER_DESC]); - $storage = $storageTotal ? $storageTotal->getAttribute('value', 0) : 0; - - Authorization::reset(); - - $response->json([ - 'range' => $range, - 'requests' => [ - 'data' => $stats['requests'] ?? [], - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $stats['requests'] ?? [])), - ], - 'network' => [ - 'data' => \array_map(function ($value) {return ['value' => \round($value['value'] / 1000000, 2), 'date' => $value['date']];}, $stats['network'] ?? []), // convert bytes to mb - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $stats['network'] ?? [])), - ], - 'functions' => [ - 'data' => $stats['executions'] ?? [], - 'total' => \array_sum(\array_map(function ($item) { - return $item['value']; - }, $stats['executions'] ?? [])), - ], - 'documents' => [ - 'data' => [], - 'total' => $documentsTotal, - ], - 'collections' => [ - 'data' => [], - 'total' => $collectionsTotal, - ], - 'users' => [ - 'data' => [], - 'total' => $usersTotal, - ], - 'storage' => [ - 'total' => $storage, - ], - ]); + $response->dynamic($usage, Response::MODEL_PROJECT_USAGE); }); App::patch('/v1/projects/:projectId') diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index a1d0d404ed..c87c924807 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -47,6 +47,7 @@ use Appwrite\Utopia\Response\Model\Token; use Appwrite\Utopia\Response\Model\Webhook; use Appwrite\Utopia\Response\Model\Preferences; use Appwrite\Utopia\Response\Model\Mock; // Keep last +use Appwrite\Utopia\Response\Model\ProjectUsage; use Appwrite\Utopia\Response\Model\StorageUsage; use Appwrite\Utopia\Response\Model\UsersUsage; use stdClass; @@ -127,6 +128,7 @@ class Response extends SwooleResponse // Project const MODEL_PROJECT = 'project'; const MODEL_PROJECT_LIST = 'projectList'; + const MODEL_PROJECT_USAGE = 'projectUsage'; const MODEL_WEBHOOK = 'webhook'; const MODEL_WEBHOOK_LIST = 'webhookList'; const MODEL_KEY = 'key'; @@ -216,6 +218,7 @@ class Response extends SwooleResponse ->setModel(new Tag()) ->setModel(new Execution()) ->setModel(new Project()) + ->setModel(new ProjectUsage()) ->setModel(new Webhook()) ->setModel(new Key()) ->setModel(new Domain()) diff --git a/src/Appwrite/Utopia/Response/Model/ProjectUsage.php b/src/Appwrite/Utopia/Response/Model/ProjectUsage.php new file mode 100644 index 0000000000..deee3b78b1 --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/ProjectUsage.php @@ -0,0 +1,91 @@ +addRule('range', [ + 'type' => self::TYPE_STRING, + 'description' => 'The time range of the usage stats.', + 'default' => '', + 'example' => '30d', + ]) + ->addRule('requests', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for number of requests.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('network', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for consumed bandwidth.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('functions', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for function executions.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('documents', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for number of documents.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('collections', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for number of collections.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('users', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for number of users.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ->addRule('functions', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for the occupied storage size.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) + ; + } + + /** + * Get Name + * + * @return string + */ + public function getName():string + { + return 'ProjectUsage'; + } + + /** + * Get Collection + * + * @return string + */ + public function getType():string + { + return Response::MODEL_PROJECT_USAGE; + } +} \ No newline at end of file From d009b8ab89274e368fe9c87cc4284342921447ee Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 27 Aug 2021 17:22:52 +0545 Subject: [PATCH 078/206] delete older usage stats --- app/init.php | 1 + app/tasks/maintenance.php | 11 +++++++++++ app/workers/deletes.php | 26 ++++++++++++++++++++++++++ 3 files changed, 38 insertions(+) diff --git a/app/init.php b/app/init.php index c8261ffd33..475bb3e40d 100644 --- a/app/init.php +++ b/app/init.php @@ -89,6 +89,7 @@ const DELETE_TYPE_EXECUTIONS = 'executions'; const DELETE_TYPE_AUDIT = 'audit'; const DELETE_TYPE_ABUSE = 'abuse'; const DELETE_TYPE_CERTIFICATES = 'certificates'; +const DELETE_TYPE_USAGE_STATS = 'certificates'; // Mail Worker Types const MAIL_TYPE_VERIFICATION = 'verification'; const MAIL_TYPE_RECOVERY = 'recovery'; diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index eccdf61b17..a5cf63f821 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -39,11 +39,22 @@ $cli ]); } + function notifyDeleteUsageStats(int $interval30m, int $interval1d) + { + Resque::enqueue(Event::DELETE_QUEUE_NAME, Event::DELETE_CLASS_NAME, [ + 'type' => DELETE_TYPE_USAGE_STATS, + 'timestamp1d' => $interval1d, + 'timestamp30m' => $interval30m, + ]); + } + // # of days in seconds (1 day = 86400s) $interval = (int) App::getEnv('_APP_MAINTENANCE_INTERVAL', '86400'); $executionLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_EXECUTION', '1209600'); $auditLogRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_AUDIT', '1209600'); $abuseLogsRetention = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_ABUSE', '86400'); + $usageStatsRetention30m = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_30M', '129600');//36 hours + $usageStatsRetention1d = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_1D', '8640000'); // 100 days Console::loop(function() use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention){ $time = date('d-m-Y H:i:s', time()); diff --git a/app/workers/deletes.php b/app/workers/deletes.php index f0e66d2b46..fe0037056c 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -72,6 +72,9 @@ class DeletesV1 extends Worker $document = new Document($this->args['document']); $this->deleteCertificates($document); break; + + case DELETE_TYPE_USAGE_STATS: + $this->deleteUsageStats($this->args['timestamp1d'], $this->args['timestamp30m']); default: Console::error('No delete operation for type: '.$type); @@ -82,6 +85,29 @@ class DeletesV1 extends Worker public function shutdown(): void { } + + /** + * @param int $timestamp1d + * @param int $timestamp30m + */ + protected function deleteUsageStats(int $timestamp1d, int $timestamp30m) { + $this->deleteForProjectIds(function($projectId) use ($timestamp1d, $timestamp30m) { + if (!($dbForInternal = $this->getInternalDB($projectId))) { + throw new Exception('Failed to get projectDB for project '.$projectId); + } + + // Delete Usage stats + $this->deleteByGroup('stats', [ + new Query('time', Query::TYPE_LESSER, [$timestamp1d]), + new Query('period', Query::TYPE_EQUAL, ['1d', '15m']), + ], $dbForInternal); + + $this->deleteByGroup('stats', [ + new Query('time', Query::TYPE_LESSER, [$timestamp30m]), + new Query('period', Query::TYPE_EQUAL, ['30m']), + ], $dbForInternal); + }); + } /** * @param Document $document teams document From ad99f8bd2404a2f505125c0998b8bdab32ba8834 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 27 Aug 2021 22:15:08 +0530 Subject: [PATCH 079/206] feat(usage): rename all usage response models --- app/controllers/api/database.php | 1 - docker-compose.yml | 2 +- src/Appwrite/Utopia/Response.php | 41 ++++++++++--------- .../{BucketsUsage.php => UsageBuckets.php} | 8 ++-- ...ollectionUsage.php => UsageCollection.php} | 8 ++-- .../{DatabaseUsage.php => UsageDatabase.php} | 8 ++-- ...{FunctionsUsage.php => UsageFunctions.php} | 8 ++-- .../{ProjectUsage.php => UsageProject.php} | 8 ++-- .../{StorageUsage.php => UsageStorage.php} | 6 +-- .../Model/{UsersUsage.php => UsageUsers.php} | 6 +-- 10 files changed, 48 insertions(+), 48 deletions(-) rename src/Appwrite/Utopia/Response/Model/{BucketsUsage.php => UsageBuckets.php} (93%) rename src/Appwrite/Utopia/Response/Model/{CollectionUsage.php => UsageCollection.php} (93%) rename src/Appwrite/Utopia/Response/Model/{DatabaseUsage.php => UsageDatabase.php} (96%) rename src/Appwrite/Utopia/Response/Model/{FunctionsUsage.php => UsageFunctions.php} (91%) rename src/Appwrite/Utopia/Response/Model/{ProjectUsage.php => UsageProject.php} (95%) rename src/Appwrite/Utopia/Response/Model/{StorageUsage.php => UsageStorage.php} (92%) rename src/Appwrite/Utopia/Response/Model/{UsersUsage.php => UsageUsers.php} (96%) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index a18c2e885c..86e8304d8b 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -1376,7 +1376,6 @@ App::post('/v1/database/collections/:collectionId/documents') $data['$read'] = (is_null($read) && !$user->isEmpty()) ? ['user:'.$user->getId()] : $read ?? []; // By default set read permissions for user $data['$write'] = (is_null($write) && !$user->isEmpty()) ? ['user:'.$user->getId()] : $write ?? []; // By default set write permissions for user - var_dump($collectionId); try { $document = $dbForExternal->createDocument($collectionId, new Document($data)); } diff --git a/docker-compose.yml b/docker-compose.yml index 910c07f094..12bc51b802 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -63,7 +63,7 @@ services: - ./psalm.xml:/usr/src/code/psalm.xml - ./tests:/usr/src/code/tests - ./app:/usr/src/code/app - - ./vendor/utopia-php/database:/usr/src/code/vendor/utopia-php/database + # - ./vendor/utopia-php/database:/usr/src/code/vendor/utopia-php/database - ./docs:/usr/src/code/docs - ./public:/usr/src/code/public - ./src:/usr/src/code/src diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index c87c924807..0c4c6bf0aa 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -12,13 +12,10 @@ use Appwrite\Utopia\Response\Model\None; use Appwrite\Utopia\Response\Model\Any; use Appwrite\Utopia\Response\Model\Attribute; use Appwrite\Utopia\Response\Model\BaseList; -use Appwrite\Utopia\Response\Model\BucketsUsage; use Appwrite\Utopia\Response\Model\Collection; -use Appwrite\Utopia\Response\Model\CollectionUsage; use Appwrite\Utopia\Response\Model\Continent; use Appwrite\Utopia\Response\Model\Country; use Appwrite\Utopia\Response\Model\Currency; -use Appwrite\Utopia\Response\Model\DatabaseUsage; use Appwrite\Utopia\Response\Model\Document as ModelDocument; use Appwrite\Utopia\Response\Model\Domain; use Appwrite\Utopia\Response\Model\Error; @@ -26,7 +23,6 @@ use Appwrite\Utopia\Response\Model\ErrorDev; use Appwrite\Utopia\Response\Model\Execution; use Appwrite\Utopia\Response\Model\File; use Appwrite\Utopia\Response\Model\Func; -use Appwrite\Utopia\Response\Model\FunctionsUsage; use Appwrite\Utopia\Response\Model\Index; use Appwrite\Utopia\Response\Model\JWT; use Appwrite\Utopia\Response\Model\Key; @@ -47,8 +43,13 @@ use Appwrite\Utopia\Response\Model\Token; use Appwrite\Utopia\Response\Model\Webhook; use Appwrite\Utopia\Response\Model\Preferences; use Appwrite\Utopia\Response\Model\Mock; // Keep last -use Appwrite\Utopia\Response\Model\ProjectUsage; -use Appwrite\Utopia\Response\Model\StorageUsage; +use Appwrite\Utopia\Response\Model\UsageBuckets; +use Appwrite\Utopia\Response\Model\UsageCollection; +use Appwrite\Utopia\Response\Model\UsageDatabase; +use Appwrite\Utopia\Response\Model\UsageFunctions; +use Appwrite\Utopia\Response\Model\UsageProject; +use Appwrite\Utopia\Response\Model\UsageStorage; +use Appwrite\Utopia\Response\Model\UsageUsers; use Appwrite\Utopia\Response\Model\UsersUsage; use stdClass; @@ -67,6 +68,13 @@ class Response extends SwooleResponse const MODEL_METRIC_LIST = 'metricList'; const MODEL_ERROR_DEV = 'errorDev'; const MODEL_BASE_LIST = 'baseList'; + const MODEL_USAGE_DATABASE = 'usageDatabase'; + const MODEL_USAGE_COLLECTION = 'usageCollection'; + const MODEL_USAGE_USERS = 'usageUsers'; + const MODEL_USAGE_BUCKETS = 'usageBuckets'; + const MODEL_USAGE_STORAGE = 'usageStorage'; + const MODEL_USAGE_FUNCTIONS = 'usageFunctions'; + const MODEL_USAGE_PROJECT = 'usageProject'; // Database const MODEL_COLLECTION = 'collection'; @@ -77,13 +85,10 @@ class Response extends SwooleResponse const MODEL_INDEX_LIST = 'indexList'; const MODEL_DOCUMENT = 'document'; const MODEL_DOCUMENT_LIST = 'documentList'; - const MODEL_DATABASE_USAGE = 'databaseUsage'; - const MODEL_COLLECTION_USAGE = 'collectionUsage'; // Users const MODEL_USER = 'user'; const MODEL_USER_LIST = 'userList'; - const MODEL_USERS_USAGE = 'usersUsage'; const MODEL_SESSION = 'session'; const MODEL_SESSION_LIST = 'sessionList'; const MODEL_TOKEN = 'token'; @@ -94,8 +99,6 @@ class Response extends SwooleResponse const MODEL_FILE = 'file'; const MODEL_FILE_LIST = 'fileList'; const MODEL_BUCKET = 'bucket'; // - Missing - const MODEL_BUCKETS_USAGE = 'bucketsUsage'; - const MODEL_STORAGE_USAGE = 'storageUsage'; // Locale const MODEL_LOCALE = 'locale'; @@ -119,7 +122,6 @@ class Response extends SwooleResponse // Functions const MODEL_FUNCTION = 'function'; const MODEL_FUNCTION_LIST = 'functionList'; - const MODEL_FUNCTIONS_USAGE = 'functionsUsage'; const MODEL_TAG = 'tag'; const MODEL_TAG_LIST = 'tagList'; const MODEL_EXECUTION = 'execution'; @@ -128,7 +130,6 @@ class Response extends SwooleResponse // Project const MODEL_PROJECT = 'project'; const MODEL_PROJECT_LIST = 'projectList'; - const MODEL_PROJECT_USAGE = 'projectUsage'; const MODEL_WEBHOOK = 'webhook'; const MODEL_WEBHOOK_LIST = 'webhookList'; const MODEL_KEY = 'key'; @@ -198,27 +199,20 @@ class Response extends SwooleResponse ->setModel(new Attribute()) ->setModel(new Index()) ->setModel(new ModelDocument()) - ->setModel(new DatabaseUsage()) - ->setModel(new CollectionUsage()) ->setModel(new Log()) ->setModel(new User()) - ->setModel(new UsersUsage()) ->setModel(new Preferences()) ->setModel(new Session()) ->setModel(new Token()) ->setModel(new JWT()) ->setModel(new Locale()) ->setModel(new File()) - ->setModel(new StorageUsage()) - ->setModel(new BucketsUsage()) ->setModel(new Team()) ->setModel(new Membership()) ->setModel(new Func()) - ->setModel(new FunctionsUsage()) ->setModel(new Tag()) ->setModel(new Execution()) ->setModel(new Project()) - ->setModel(new ProjectUsage()) ->setModel(new Webhook()) ->setModel(new Key()) ->setModel(new Domain()) @@ -228,6 +222,13 @@ class Response extends SwooleResponse ->setModel(new Language()) ->setModel(new Currency()) ->setModel(new Phone()) + ->setModel(new UsageDatabase()) + ->setModel(new UsageCollection()) + ->setModel(new UsageUsers()) + ->setModel(new UsageStorage()) + ->setModel(new UsageBuckets()) + ->setModel(new UsageFunctions()) + ->setModel(new UsageProject()) // Verification // Recovery // Tests (keep last) diff --git a/src/Appwrite/Utopia/Response/Model/BucketsUsage.php b/src/Appwrite/Utopia/Response/Model/UsageBuckets.php similarity index 93% rename from src/Appwrite/Utopia/Response/Model/BucketsUsage.php rename to src/Appwrite/Utopia/Response/Model/UsageBuckets.php index 0fb923e147..11fb2e2f8c 100644 --- a/src/Appwrite/Utopia/Response/Model/BucketsUsage.php +++ b/src/Appwrite/Utopia/Response/Model/UsageBuckets.php @@ -6,7 +6,7 @@ use Appwrite\Utopia\Response; use Appwrite\Utopia\Response\Model; use stdClass; -class BucketsUsage extends Model +class UsageBuckets extends Model { public function __construct() { @@ -55,16 +55,16 @@ class BucketsUsage extends Model */ public function getName():string { - return 'BucketsUsage'; + return 'UsageBuckets'; } /** - * Get Collection + * Get Type * * @return string */ public function getType():string { - return Response::MODEL_BUCKETS_USAGE; + return Response::MODEL_USAGE_BUCKETS; } } \ No newline at end of file diff --git a/src/Appwrite/Utopia/Response/Model/CollectionUsage.php b/src/Appwrite/Utopia/Response/Model/UsageCollection.php similarity index 93% rename from src/Appwrite/Utopia/Response/Model/CollectionUsage.php rename to src/Appwrite/Utopia/Response/Model/UsageCollection.php index a0b0a51c9b..f2815831a2 100644 --- a/src/Appwrite/Utopia/Response/Model/CollectionUsage.php +++ b/src/Appwrite/Utopia/Response/Model/UsageCollection.php @@ -6,7 +6,7 @@ use Appwrite\Utopia\Response; use Appwrite\Utopia\Response\Model; use stdClass; -class CollectionUsage extends Model +class UsageCollection extends Model { public function __construct() { @@ -62,16 +62,16 @@ class CollectionUsage extends Model */ public function getName():string { - return 'CollectionUsage'; + return 'UsageCollection'; } /** - * Get Collection + * Get Type * * @return string */ public function getType():string { - return Response::MODEL_COLLECTION_USAGE; + return Response::MODEL_USAGE_COLLECTION; } } \ No newline at end of file diff --git a/src/Appwrite/Utopia/Response/Model/DatabaseUsage.php b/src/Appwrite/Utopia/Response/Model/UsageDatabase.php similarity index 96% rename from src/Appwrite/Utopia/Response/Model/DatabaseUsage.php rename to src/Appwrite/Utopia/Response/Model/UsageDatabase.php index 1c24625004..2ebe0b64c0 100644 --- a/src/Appwrite/Utopia/Response/Model/DatabaseUsage.php +++ b/src/Appwrite/Utopia/Response/Model/UsageDatabase.php @@ -6,7 +6,7 @@ use Appwrite\Utopia\Response; use Appwrite\Utopia\Response\Model; use stdClass; -class DatabaseUsage extends Model +class UsageDatabase extends Model { public function __construct() { @@ -97,16 +97,16 @@ class DatabaseUsage extends Model */ public function getName():string { - return 'DatabaseUsage'; + return 'UsageDatabase'; } /** - * Get Collection + * Get Type * * @return string */ public function getType():string { - return Response::MODEL_DATABASE_USAGE; + return Response::MODEL_USAGE_DATABASE; } } \ No newline at end of file diff --git a/src/Appwrite/Utopia/Response/Model/FunctionsUsage.php b/src/Appwrite/Utopia/Response/Model/UsageFunctions.php similarity index 91% rename from src/Appwrite/Utopia/Response/Model/FunctionsUsage.php rename to src/Appwrite/Utopia/Response/Model/UsageFunctions.php index 60f017705e..625bdab71c 100644 --- a/src/Appwrite/Utopia/Response/Model/FunctionsUsage.php +++ b/src/Appwrite/Utopia/Response/Model/UsageFunctions.php @@ -6,7 +6,7 @@ use Appwrite\Utopia\Response; use Appwrite\Utopia\Response\Model; use stdClass; -class FunctionsUsage extends Model +class UsageFunctions extends Model { public function __construct() { @@ -48,16 +48,16 @@ class FunctionsUsage extends Model */ public function getName():string { - return 'FunctionsUsage'; + return 'UsageFunctions'; } /** - * Get Collection + * Get Type * * @return string */ public function getType():string { - return Response::MODEL_FUNCTIONS_USAGE; + return Response::MODEL_USAGE_FUNCTIONS; } } \ No newline at end of file diff --git a/src/Appwrite/Utopia/Response/Model/ProjectUsage.php b/src/Appwrite/Utopia/Response/Model/UsageProject.php similarity index 95% rename from src/Appwrite/Utopia/Response/Model/ProjectUsage.php rename to src/Appwrite/Utopia/Response/Model/UsageProject.php index deee3b78b1..95bb3cd42d 100644 --- a/src/Appwrite/Utopia/Response/Model/ProjectUsage.php +++ b/src/Appwrite/Utopia/Response/Model/UsageProject.php @@ -6,7 +6,7 @@ use Appwrite\Utopia\Response; use Appwrite\Utopia\Response\Model; use stdClass; -class ProjectUsage extends Model +class UsageProject extends Model { public function __construct() { @@ -76,16 +76,16 @@ class ProjectUsage extends Model */ public function getName():string { - return 'ProjectUsage'; + return 'UsageProject'; } /** - * Get Collection + * Get Type * * @return string */ public function getType():string { - return Response::MODEL_PROJECT_USAGE; + return Response::MODEL_USAGE_PROJECT; } } \ No newline at end of file diff --git a/src/Appwrite/Utopia/Response/Model/StorageUsage.php b/src/Appwrite/Utopia/Response/Model/UsageStorage.php similarity index 92% rename from src/Appwrite/Utopia/Response/Model/StorageUsage.php rename to src/Appwrite/Utopia/Response/Model/UsageStorage.php index e26e0d4b3d..66eaed602c 100644 --- a/src/Appwrite/Utopia/Response/Model/StorageUsage.php +++ b/src/Appwrite/Utopia/Response/Model/UsageStorage.php @@ -6,7 +6,7 @@ use Appwrite\Utopia\Response; use Appwrite\Utopia\Response\Model; use stdClass; -class StorageUsage extends Model +class UsageStorage extends Model { public function __construct() { @@ -45,12 +45,12 @@ class StorageUsage extends Model } /** - * Get Collection + * Get Type * * @return string */ public function getType():string { - return Response::MODEL_STORAGE_USAGE; + return Response::MODEL_USAGE_STORAGE; } } \ No newline at end of file diff --git a/src/Appwrite/Utopia/Response/Model/UsersUsage.php b/src/Appwrite/Utopia/Response/Model/UsageUsers.php similarity index 96% rename from src/Appwrite/Utopia/Response/Model/UsersUsage.php rename to src/Appwrite/Utopia/Response/Model/UsageUsers.php index bb3db30b26..db8056f5c6 100644 --- a/src/Appwrite/Utopia/Response/Model/UsersUsage.php +++ b/src/Appwrite/Utopia/Response/Model/UsageUsers.php @@ -6,7 +6,7 @@ use Appwrite\Utopia\Response; use Appwrite\Utopia\Response\Model; use stdClass; -class UsersUsage extends Model +class UsageUsers extends Model { public function __construct() { @@ -76,7 +76,7 @@ class UsersUsage extends Model */ public function getName():string { - return 'UsersUsage'; + return 'UsageUsers'; } /** @@ -86,6 +86,6 @@ class UsersUsage extends Model */ public function getType():string { - return Response::MODEL_USERS_USAGE; + return Response::MODEL_USAGE_USERS; } } \ No newline at end of file From 6835124af8f387f0df364991d39f0ef2e76db473 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Fri, 27 Aug 2021 23:04:43 +0530 Subject: [PATCH 080/206] feat(usage): rename all usage response models --- app/controllers/api/database.php | 8 ++++---- app/controllers/api/functions.php | 4 ++-- app/controllers/api/projects.php | 4 ++-- app/controllers/api/storage.php | 8 ++++---- app/controllers/api/users.php | 4 ++-- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 86e8304d8b..1ff2e9b76e 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -254,7 +254,7 @@ App::get('/v1/database/usage') ->label('sdk.method', 'getUsage') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) - ->label('sdk.response.model', Response::MODEL_DATABASE_USAGE) + ->label('sdk.response.model', Response::MODEL_USAGE_DATABASE) ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) ->inject('response') ->inject('dbForInternal') @@ -335,7 +335,7 @@ App::get('/v1/database/usage') ]); } - $response->dynamic($usage, Response::MODEL_DATABASE_USAGE); + $response->dynamic($usage, Response::MODEL_USAGE_DATABASE); }); App::get('/v1/database/:collectionId/usage') @@ -347,7 +347,7 @@ App::get('/v1/database/:collectionId/usage') ->label('sdk.method', 'getUsage') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) - ->label('sdk.response.model', Response::MODEL_COLLECTION_USAGE) + ->label('sdk.response.model', Response::MODEL_USAGE_COLLECTION) ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) ->param('collectionId', '', new UID(), 'Collection unique ID.') ->inject('response') @@ -425,7 +425,7 @@ App::get('/v1/database/:collectionId/usage') ]); } - $response->dynamic($usage, Response::MODEL_COLLECTION_USAGE); + $response->dynamic($usage, Response::MODEL_USAGE_COLLECTION); }); App::get('/v1/database/collections/:collectionId/logs') diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 9359d267f5..dec4fa92bf 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -148,7 +148,7 @@ App::get('/v1/functions/:functionId/usage') ->label('sdk.method', 'getUsage') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) - ->label('sdk.response.model', Response::MODEL_FUNCTIONS_USAGE) + ->label('sdk.response.model', Response::MODEL_USAGE_FUNCTIONS) ->param('functionId', '', new UID(), 'Function unique ID.') ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d']), 'Date range.', true) ->inject('response') @@ -224,7 +224,7 @@ App::get('/v1/functions/:functionId/usage') ]); } - $response->dynamic($usage, Response::MODEL_FUNCTIONS_USAGE); + $response->dynamic($usage, Response::MODEL_USAGE_FUNCTIONS); }); App::put('/v1/functions/:functionId') diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index e8551c29ce..2d8311845a 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -225,7 +225,7 @@ App::get('/v1/projects/:projectId/usage') ->label('sdk.method', 'getUsage') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) - ->label('sdk.response.model', Response::MODEL_PROJECT_USAGE) + ->label('sdk.response.model', Response::MODEL_USAGE_PROJECT) ->param('projectId', '', new UID(), 'Project unique ID.') ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) ->inject('response') @@ -311,7 +311,7 @@ App::get('/v1/projects/:projectId/usage') ]); } - $response->dynamic($usage, Response::MODEL_PROJECT_USAGE); + $response->dynamic($usage, Response::MODEL_USAGE_PROJECT); }); App::patch('/v1/projects/:projectId') diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 69a7c18708..c3804b2f60 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -653,7 +653,7 @@ App::get('/v1/storage/usage') ->label('sdk.method', 'getUsage') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) - ->label('sdk.response.model', Response::MODEL_STORAGE_USAGE) + ->label('sdk.response.model', Response::MODEL_USAGE_STORAGE) ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) ->inject('response') ->inject('dbForInternal') @@ -713,7 +713,7 @@ App::get('/v1/storage/usage') ]); } - $response->dynamic($usage, Response::MODEL_STORAGE_USAGE); + $response->dynamic($usage, Response::MODEL_USAGE_STORAGE); }); App::get('/v1/storage/:bucketId/usage') @@ -725,7 +725,7 @@ App::get('/v1/storage/:bucketId/usage') ->label('sdk.method', 'getUsage') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) - ->label('sdk.response.model', Response::MODEL_BUCKETS_USAGE) + ->label('sdk.response.model', Response::MODEL_USAGE_BUCKETS) ->param('bucketId', '', new UID(), 'Bucket unique ID.') ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) ->inject('response') @@ -792,5 +792,5 @@ App::get('/v1/storage/:bucketId/usage') ]); } - $response->dynamic($usage, Response::MODEL_BUCKETS_USAGE); + $response->dynamic($usage, Response::MODEL_USAGE_BUCKETS); }); \ No newline at end of file diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 5a0cb03209..1895098bf2 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -619,7 +619,7 @@ App::get('/v1/users/usage') ->label('sdk.method', 'getUsage') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) - ->label('sdk.response.model', Response::MODEL_USERS_USAGE) + ->label('sdk.response.model', Response::MODEL_USAGE_USERS) ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) ->inject('response') ->inject('dbForInternal') @@ -693,5 +693,5 @@ App::get('/v1/users/usage') } - $response->dynamic($usage, Response::MODEL_USERS_USAGE); + $response->dynamic($usage, Response::MODEL_USAGE_USERS); }); \ No newline at end of file From 5a1ab3b059cea6dccf8a8f5ff9ddc40c939eace7 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Wed, 11 Aug 2021 21:05:19 -0400 Subject: [PATCH 081/206] Add enforce param to collections --- app/controllers/api/database.php | 41 ++++++++++++++++++++++++++++++++ composer.lock | 12 +++++----- 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index dd275c9c2e..c02d8d1087 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -15,6 +15,7 @@ use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Query; use Utopia\Database\Adapter\MariaDB; +use Utopia\Database\Validator\Authorization; use Utopia\Database\Validator\Key; use Utopia\Database\Validator\Permissions; use Utopia\Database\Validator\QueryValidator; @@ -1138,6 +1139,14 @@ App::post('/v1/database/collections/:collectionId/documents') throw new Exception('Collection not found', 404); } + // Check collection permissions when enforced + if ($collection->getAttribute('enforce') === 'collection') { + $validator = new Authorization($collection, 'write'); + if (!$validator->isValid($collection->getWrite())) { + throw new Exception('Unauthorized permissions', 401); + } + } + $data['$collection'] = $collection->getId(); // Adding this param to make API easier for developers $data['$id'] = $documentId == 'unique()' ? $dbForExternal->getId() : $documentId; $data['$read'] = (is_null($read) && !$user->isEmpty()) ? ['user:'.$user->getId()] : $read ?? []; // By default set read permissions for user @@ -1195,6 +1204,14 @@ App::get('/v1/database/collections/:collectionId/documents') throw new Exception('Collection not found', 404); } + // Check collection permissions when enforced + if ($collection->getAttribute('enforce') === 'collection') { + $validator = new Authorization($collection, 'read'); + if (!$validator->isValid($collection->getRead())) { + throw new Exception('Unauthorized permissions', 401); + } + } + $queries = \array_map(function ($query) { return Query::parse($query); }, $queries); @@ -1247,6 +1264,14 @@ App::get('/v1/database/collections/:collectionId/documents/:documentId') throw new Exception('Collection not found', 404); } + // Check collection permissions when enforced + if ($collection->getAttribute('enforce') === 'collection') { + $validator = new Authorization($collection, 'read'); + if (!$validator->isValid($collection->getRead())) { + throw new Exception('Unauthorized permissions', 401); + } + } + $document = $dbForExternal->getDocument($collectionId, $documentId); if ($document->isEmpty()) { @@ -1289,6 +1314,14 @@ App::patch('/v1/database/collections/:collectionId/documents/:documentId') throw new Exception('Collection not found', 404); } + // Check collection permissions when enforced + if ($collection->getAttribute('enforce') === 'collection') { + $validator = new Authorization($collection, 'write'); + if (!$validator->isValid($collection->getWrite())) { + throw new Exception('Unauthorized permissions', 401); + } + } + $document = $dbForExternal->getDocument($collectionId, $documentId); if ($document->isEmpty()) { @@ -1361,6 +1394,14 @@ App::delete('/v1/database/collections/:collectionId/documents/:documentId') throw new Exception('Collection not found', 404); } + // Check collection permissions when enforced + if ($collection->getAttribute('enforce') === 'collection') { + $validator = new Authorization($collection, 'write'); + if (!$validator->isValid($collection->getWrite())) { + throw new Exception('Unauthorized permissions', 401); + } + } + $document = $dbForExternal->getDocument($collectionId, $documentId); if ($document->isEmpty()) { diff --git a/composer.lock b/composer.lock index a5f2a3e062..8dac7bcc6d 100644 --- a/composer.lock +++ b/composer.lock @@ -355,16 +355,16 @@ }, { "name": "composer/package-versions-deprecated", - "version": "1.11.99.2", + "version": "1.11.99.3", "source": { "type": "git", "url": "https://github.com/composer/package-versions-deprecated.git", - "reference": "c6522afe5540d5fc46675043d3ed5a45a740b27c" + "reference": "fff576ac850c045158a250e7e27666e146e78d18" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/composer/package-versions-deprecated/zipball/c6522afe5540d5fc46675043d3ed5a45a740b27c", - "reference": "c6522afe5540d5fc46675043d3ed5a45a740b27c", + "url": "https://api.github.com/repos/composer/package-versions-deprecated/zipball/fff576ac850c045158a250e7e27666e146e78d18", + "reference": "fff576ac850c045158a250e7e27666e146e78d18", "shasum": "" }, "require": { @@ -408,7 +408,7 @@ "description": "Composer plugin that provides efficient querying for installed package versions (no runtime IO)", "support": { "issues": "https://github.com/composer/package-versions-deprecated/issues", - "source": "https://github.com/composer/package-versions-deprecated/tree/1.11.99.2" + "source": "https://github.com/composer/package-versions-deprecated/tree/1.11.99.3" }, "funding": [ { @@ -424,7 +424,7 @@ "type": "tidelift" } ], - "time": "2021-05-24T07:46:03+00:00" + "time": "2021-08-17T13:49:14+00:00" }, { "name": "dragonmantank/cron-expression", From 7db7e39f78f9ea30cab29bd357862750f34a0d45 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Wed, 11 Aug 2021 21:07:46 -0400 Subject: [PATCH 082/206] Test enforce collection permissions --- tests/e2e/Services/Database/DatabaseBase.php | 63 ++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/tests/e2e/Services/Database/DatabaseBase.php b/tests/e2e/Services/Database/DatabaseBase.php index 3e109ca33b..f62e9d902d 100644 --- a/tests/e2e/Services/Database/DatabaseBase.php +++ b/tests/e2e/Services/Database/DatabaseBase.php @@ -1089,4 +1089,67 @@ trait DatabaseBase return $data; } + + public function testEnforceCollectionPermissions() + { + $user = 'user:' . $this->getUser()['$id']; + $collection = $this->client->call(Client::METHOD_POST, '/database/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => 'unique()', + 'name' => 'enforceCollectionPermissions', + 'enforce' => 'collection', + 'read' => [$user], + 'write' => [$user] + ]); + + var_dump($collection); + + $this->assertEquals($collection['headers']['status-code'], 201); + $this->assertEquals($collection['body']['name'], 'enforceCollectionPermissions'); + $this->assertEquals($collection['body']['enforce'], 'collection'); + + $collectionId = $collection['body']['$id']; + + $attribute = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/attributes/string', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'attributeId' => 'attribute', + 'size' => 64, + 'required' => true, + ]); + + $this->assertEquals($attribute['headers']['status-code'], 201); + $this->assertEquals($attribute['body']['$id'], 'attribute'); + + // wait for db to add attribute + sleep(3); + + $document = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'documentId' => 'unique()', + 'data' => [ + 'attribute' => 'documen1', + ], + 'read' => [], + 'write' => [], + ]); + + $this->assertEquals($document['headers']['status-code'], 201); + + $documents = $this->client->call(Client::METHOD_GET, '/database/collections/' . $collectionId . '/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + // var_dump($documents); + + $this->assertCount(1, $documents['body']['documents']); + } } \ No newline at end of file From 0e68b4a1e4d0307f78ef05e8818edcabcdd9360e Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Wed, 11 Aug 2021 21:25:51 -0400 Subject: [PATCH 083/206] Remove var dumps --- tests/e2e/Services/Database/DatabaseBase.php | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/e2e/Services/Database/DatabaseBase.php b/tests/e2e/Services/Database/DatabaseBase.php index f62e9d902d..3d1041272d 100644 --- a/tests/e2e/Services/Database/DatabaseBase.php +++ b/tests/e2e/Services/Database/DatabaseBase.php @@ -1105,8 +1105,6 @@ trait DatabaseBase 'write' => [$user] ]); - var_dump($collection); - $this->assertEquals($collection['headers']['status-code'], 201); $this->assertEquals($collection['body']['name'], 'enforceCollectionPermissions'); $this->assertEquals($collection['body']['enforce'], 'collection'); @@ -1148,8 +1146,6 @@ trait DatabaseBase 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders())); - // var_dump($documents); - $this->assertCount(1, $documents['body']['documents']); } } \ No newline at end of file From 2b057c06176fbd1ea43fa71657909ba0bbe088c8 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Wed, 11 Aug 2021 21:26:31 -0400 Subject: [PATCH 084/206] Skip authorization on document routes if collection permissions are met --- app/controllers/api/database.php | 49 ++++++++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 6 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index c02d8d1087..9b62bc85b7 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -1153,7 +1153,14 @@ App::post('/v1/database/collections/:collectionId/documents') $data['$write'] = (is_null($write) && !$user->isEmpty()) ? ['user:'.$user->getId()] : $write ?? []; // By default set write permissions for user try { - $document = $dbForExternal->createDocument($collectionId, new Document($data)); + if ($collection->getAttribute('enforce') === 'collection') { + /** @var Document $document */ + $document = Authorization::skip(function() use ($dbForExternal, $collectionId, $data) { + return $dbForExternal->createDocument($collectionId, new Document($data)); + }); + } else { + $document = $dbForExternal->createDocument($collectionId, new Document($data)); + } } catch (StructureException $exception) { throw new Exception($exception->getMessage(), 400); @@ -1227,13 +1234,22 @@ App::get('/v1/database/collections/:collectionId/documents') $afterDocument = $dbForExternal->getDocument($collectionId, $after); if ($afterDocument->isEmpty()) { - throw new Exception("Document '{$after}' for the 'after' value not found.", 400); + throw new Exception("Document \'{$after}\' for the \'after\' value not found.", 400); } } + if ($collection->getAttribute('enforce') === 'collection') { + /** @var Document[] $documents */ + $documents = Authorization::skip(function() use ($dbForExternal, $collectionId, $queries, $limit, $offset, $orderAttributes, $orderTypes, $afterDocument) { + return $dbForExternal->find($collectionId, $queries, $limit, $offset, $orderAttributes, $orderTypes, $afterDocument ?? null); + }); + } else { + $documents = $dbForExternal->find($collectionId, $queries, $limit, $offset, $orderAttributes, $orderTypes, $afterDocument ?? null); + } + $response->dynamic(new Document([ 'sum' => $dbForExternal->count($collectionId, $queries, APP_LIMIT_COUNT), - 'documents' => $dbForExternal->find($collectionId, $queries, $limit, $offset, $orderAttributes, $orderTypes, $afterDocument ?? null), + 'documents' => $documents, ]), Response::MODEL_DOCUMENT_LIST); }); @@ -1272,7 +1288,14 @@ App::get('/v1/database/collections/:collectionId/documents/:documentId') } } - $document = $dbForExternal->getDocument($collectionId, $documentId); + if ($collection->getAttribute('enforce') === 'collection') { + /** @var Document $document */ + $document = Authorization::skip(function() use ($dbForExternal, $collectionId, $documentId) { + return $dbForExternal->getDocument($collectionId, $documentId); + }); + } else { + $document = $dbForExternal->getDocument($collectionId, $documentId); + } if ($document->isEmpty()) { throw new Exception('No document found', 404); @@ -1346,7 +1369,14 @@ App::patch('/v1/database/collections/:collectionId/documents/:documentId') $data['$write'] = (is_null($write)) ? ($document->getWrite() ?? []) : $write; // By default inherit write permissions try { - $document = $dbForExternal->updateDocument($collection->getId(), $document->getId(), new Document($data)); + if ($collection->getAttribute('enforce') === 'collection') { + /** @var Document $document */ + $document = Authorization::skip(function() use ($dbForExternal, $collection, $document, $data) { + return $dbForExternal->updateDocument($collection->getId(), $document->getId(), new Document($data)); + }); + } else { + $document = $dbForExternal->updateDocument($collection->getId(), $document->getId(), new Document($data)); + } } catch (AuthorizationException $exception) { throw new Exception('Unauthorized permissions', 401); @@ -1402,7 +1432,14 @@ App::delete('/v1/database/collections/:collectionId/documents/:documentId') } } - $document = $dbForExternal->getDocument($collectionId, $documentId); + if ($collection->getAttribute('enforce') === 'collection') { + /** @var Document $document */ + $document = Authorization::skip(function() use ($dbForExternal, $collectionId, $documentId) { + return $dbForExternal->getDocument($collectionId, $documentId); + }); + } else { + $document = $dbForExternal->getDocument($collectionId, $documentId); + } if ($document->isEmpty()) { throw new Exception('No document found', 404); From 95f505b61657ff66a177c20b8fed55f3aa2ecfbe Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Fri, 13 Aug 2021 16:20:54 -0400 Subject: [PATCH 085/206] Add tests for collection permissions --- tests/e2e/Services/Database/DatabaseBase.php | 106 +++++++++++++++++-- 1 file changed, 98 insertions(+), 8 deletions(-) diff --git a/tests/e2e/Services/Database/DatabaseBase.php b/tests/e2e/Services/Database/DatabaseBase.php index 3d1041272d..43d7f51b0a 100644 --- a/tests/e2e/Services/Database/DatabaseBase.php +++ b/tests/e2e/Services/Database/DatabaseBase.php @@ -654,6 +654,8 @@ trait DatabaseBase 'required' => false, ]); + sleep(2); + $ip = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/attributes/ip', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], @@ -663,6 +665,8 @@ trait DatabaseBase 'required' => false, ]); + sleep(2); + $url = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/attributes/url', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], @@ -673,6 +677,8 @@ trait DatabaseBase 'required' => false, ]); + sleep(2); + $range = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/attributes/integer', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], @@ -684,6 +690,8 @@ trait DatabaseBase 'max' => 10, ]); + sleep(2); + // TODO@kodumbeats min and max are rounded in error message $floatRange = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/attributes/float', array_merge([ 'content-type' => 'application/json', @@ -696,6 +704,8 @@ trait DatabaseBase 'max' => 1.4, ]); + sleep(2); + // TODO@kodumbeats float validator rejects 0.0 and 1.0 as floats // $probability = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/attributes/float', array_merge([ // 'content-type' => 'application/json', @@ -718,6 +728,8 @@ trait DatabaseBase 'max' => 10, ]); + sleep(2); + $lowerBound = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/attributes/integer', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], @@ -1111,6 +1123,8 @@ trait DatabaseBase $collectionId = $collection['body']['$id']; + sleep(2); + $attribute = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/attributes/string', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], @@ -1121,31 +1135,107 @@ trait DatabaseBase 'required' => true, ]); - $this->assertEquals($attribute['headers']['status-code'], 201); - $this->assertEquals($attribute['body']['$id'], 'attribute'); + $this->assertEquals(201, $attribute['headers']['status-code'], 201); + $this->assertEquals('attribute', $attribute['body']['$id']); // wait for db to add attribute - sleep(3); + sleep(2); - $document = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/documents', array_merge([ + $index = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/indexes', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'indexId' => 'key_attribute', + 'type' => 'key', + 'attributes' => [$attribute['body']['$id']], + ]); + + $this->assertEquals(201, $index['headers']['status-code']); + $this->assertEquals('key_attribute', $index['body']['$id']); + + // wait for db to add attribute + sleep(2); + + $document1 = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/documents', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ 'documentId' => 'unique()', 'data' => [ - 'attribute' => 'documen1', + 'attribute' => 'one', ], - 'read' => [], - 'write' => [], + 'read' => [$user], + 'write' => [$user], ]); - $this->assertEquals($document['headers']['status-code'], 201); + $this->assertEquals(201, $document1['headers']['status-code']); $documents = $this->client->call(Client::METHOD_GET, '/database/collections/' . $collectionId . '/documents', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders())); + $this->assertEquals(1, $documents['body']['sum']); $this->assertCount(1, $documents['body']['documents']); + + /* + * Test for Failure + */ + + // Remove write permission + $collection = $this->client->call(Client::METHOD_PUT, '/database/collections/' . $collectionId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'name' => 'enforceCollectionPermissions', + 'enforce' => 'collection', + 'read' => [$user], + 'write' => [] + ]); + + $this->assertEquals(200, $collection['headers']['status-code']); + + $badDocument = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'documentId' => 'unique()', + 'data' => [ + 'attribute' => 'bad', + ], + 'read' => [$user], + 'write' => [$user], + ]); + + if($this->getSide() == 'client') { + $this->assertEquals(401, $badDocument['headers']['status-code']); + } + + if($this->getSide() == 'server') { + $this->assertEquals(201, $badDocument['headers']['status-code']); + } + + // Remove read permission + $collection = $this->client->call(Client::METHOD_PUT, '/database/collections/' . $collectionId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'name' => 'enforceCollectionPermissions', + 'enforce' => 'collection', + 'read' => [], + 'write' => [] + ]); + + $this->assertEquals(200, $collection['headers']['status-code']); + + $documents = $this->client->call(Client::METHOD_GET, '/database/collections/' . $collectionId . '/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ])); + + $this->assertEquals(401, $documents['headers']['status-code']); } } \ No newline at end of file From a33faab21393ceb2c3ab6cafb8fc5a931c1bb2a3 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Fri, 27 Aug 2021 21:45:15 -0400 Subject: [PATCH 086/206] Enforce attribute refactored to permission --- app/controllers/api/database.php | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 9b62bc85b7..14feb5df37 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -1140,7 +1140,7 @@ App::post('/v1/database/collections/:collectionId/documents') } // Check collection permissions when enforced - if ($collection->getAttribute('enforce') === 'collection') { + if ($collection->getAttribute('permission') === 'collection') { $validator = new Authorization($collection, 'write'); if (!$validator->isValid($collection->getWrite())) { throw new Exception('Unauthorized permissions', 401); @@ -1153,7 +1153,7 @@ App::post('/v1/database/collections/:collectionId/documents') $data['$write'] = (is_null($write) && !$user->isEmpty()) ? ['user:'.$user->getId()] : $write ?? []; // By default set write permissions for user try { - if ($collection->getAttribute('enforce') === 'collection') { + if ($collection->getAttribute('permission') === 'collection') { /** @var Document $document */ $document = Authorization::skip(function() use ($dbForExternal, $collectionId, $data) { return $dbForExternal->createDocument($collectionId, new Document($data)); @@ -1212,7 +1212,7 @@ App::get('/v1/database/collections/:collectionId/documents') } // Check collection permissions when enforced - if ($collection->getAttribute('enforce') === 'collection') { + if ($collection->getAttribute('permission') === 'collection') { $validator = new Authorization($collection, 'read'); if (!$validator->isValid($collection->getRead())) { throw new Exception('Unauthorized permissions', 401); @@ -1238,7 +1238,7 @@ App::get('/v1/database/collections/:collectionId/documents') } } - if ($collection->getAttribute('enforce') === 'collection') { + if ($collection->getAttribute('permission') === 'collection') { /** @var Document[] $documents */ $documents = Authorization::skip(function() use ($dbForExternal, $collectionId, $queries, $limit, $offset, $orderAttributes, $orderTypes, $afterDocument) { return $dbForExternal->find($collectionId, $queries, $limit, $offset, $orderAttributes, $orderTypes, $afterDocument ?? null); @@ -1281,14 +1281,14 @@ App::get('/v1/database/collections/:collectionId/documents/:documentId') } // Check collection permissions when enforced - if ($collection->getAttribute('enforce') === 'collection') { + if ($collection->getAttribute('permission') === 'collection') { $validator = new Authorization($collection, 'read'); if (!$validator->isValid($collection->getRead())) { throw new Exception('Unauthorized permissions', 401); } } - if ($collection->getAttribute('enforce') === 'collection') { + if ($collection->getAttribute('permission') === 'collection') { /** @var Document $document */ $document = Authorization::skip(function() use ($dbForExternal, $collectionId, $documentId) { return $dbForExternal->getDocument($collectionId, $documentId); @@ -1338,7 +1338,7 @@ App::patch('/v1/database/collections/:collectionId/documents/:documentId') } // Check collection permissions when enforced - if ($collection->getAttribute('enforce') === 'collection') { + if ($collection->getAttribute('permission') === 'collection') { $validator = new Authorization($collection, 'write'); if (!$validator->isValid($collection->getWrite())) { throw new Exception('Unauthorized permissions', 401); @@ -1369,7 +1369,7 @@ App::patch('/v1/database/collections/:collectionId/documents/:documentId') $data['$write'] = (is_null($write)) ? ($document->getWrite() ?? []) : $write; // By default inherit write permissions try { - if ($collection->getAttribute('enforce') === 'collection') { + if ($collection->getAttribute('permission') === 'collection') { /** @var Document $document */ $document = Authorization::skip(function() use ($dbForExternal, $collection, $document, $data) { return $dbForExternal->updateDocument($collection->getId(), $document->getId(), new Document($data)); @@ -1425,14 +1425,14 @@ App::delete('/v1/database/collections/:collectionId/documents/:documentId') } // Check collection permissions when enforced - if ($collection->getAttribute('enforce') === 'collection') { + if ($collection->getAttribute('permission') === 'collection') { $validator = new Authorization($collection, 'write'); if (!$validator->isValid($collection->getWrite())) { throw new Exception('Unauthorized permissions', 401); } } - if ($collection->getAttribute('enforce') === 'collection') { + if ($collection->getAttribute('permission') === 'collection') { /** @var Document $document */ $document = Authorization::skip(function() use ($dbForExternal, $collectionId, $documentId) { return $dbForExternal->getDocument($collectionId, $documentId); From 4af81db28072263f651a14a6aa7d7f0daace9f00 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Fri, 27 Aug 2021 21:45:42 -0400 Subject: [PATCH 087/206] Authorization validator only accepts one argument --- app/controllers/api/database.php | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 14feb5df37..517651c3e3 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -1141,7 +1141,7 @@ App::post('/v1/database/collections/:collectionId/documents') // Check collection permissions when enforced if ($collection->getAttribute('permission') === 'collection') { - $validator = new Authorization($collection, 'write'); + $validator = new Authorization('write'); if (!$validator->isValid($collection->getWrite())) { throw new Exception('Unauthorized permissions', 401); } @@ -1213,7 +1213,7 @@ App::get('/v1/database/collections/:collectionId/documents') // Check collection permissions when enforced if ($collection->getAttribute('permission') === 'collection') { - $validator = new Authorization($collection, 'read'); + $validator = new Authorization('read'); if (!$validator->isValid($collection->getRead())) { throw new Exception('Unauthorized permissions', 401); } @@ -1234,7 +1234,7 @@ App::get('/v1/database/collections/:collectionId/documents') $afterDocument = $dbForExternal->getDocument($collectionId, $after); if ($afterDocument->isEmpty()) { - throw new Exception("Document \'{$after}\' for the \'after\' value not found.", 400); + throw new Exception("Document '{$after}' for the 'after' value not found.", 400); } } @@ -1282,7 +1282,7 @@ App::get('/v1/database/collections/:collectionId/documents/:documentId') // Check collection permissions when enforced if ($collection->getAttribute('permission') === 'collection') { - $validator = new Authorization($collection, 'read'); + $validator = new Authorization('read'); if (!$validator->isValid($collection->getRead())) { throw new Exception('Unauthorized permissions', 401); } @@ -1339,7 +1339,7 @@ App::patch('/v1/database/collections/:collectionId/documents/:documentId') // Check collection permissions when enforced if ($collection->getAttribute('permission') === 'collection') { - $validator = new Authorization($collection, 'write'); + $validator = new Authorization('write'); if (!$validator->isValid($collection->getWrite())) { throw new Exception('Unauthorized permissions', 401); } @@ -1426,7 +1426,7 @@ App::delete('/v1/database/collections/:collectionId/documents/:documentId') // Check collection permissions when enforced if ($collection->getAttribute('permission') === 'collection') { - $validator = new Authorization($collection, 'write'); + $validator = new Authorization('write'); if (!$validator->isValid($collection->getWrite())) { throw new Exception('Unauthorized permissions', 401); } From e9541a9269af49c63d01e454b41dce8e4d8f7282 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Fri, 27 Aug 2021 21:46:33 -0400 Subject: [PATCH 088/206] Fix tests --- app/controllers/api/database.php | 1 + tests/e2e/Services/Database/DatabaseBase.php | 16 ++++++++-------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 517651c3e3..4519da1be4 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -1230,6 +1230,7 @@ App::get('/v1/database/collections/:collectionId/documents') throw new Exception($validator->getDescription(), 400); } + $afterDocument = null; if (!empty($after)) { $afterDocument = $dbForExternal->getDocument($collectionId, $after); diff --git a/tests/e2e/Services/Database/DatabaseBase.php b/tests/e2e/Services/Database/DatabaseBase.php index 43d7f51b0a..e260f1477a 100644 --- a/tests/e2e/Services/Database/DatabaseBase.php +++ b/tests/e2e/Services/Database/DatabaseBase.php @@ -1112,14 +1112,14 @@ trait DatabaseBase ]), [ 'collectionId' => 'unique()', 'name' => 'enforceCollectionPermissions', - 'enforce' => 'collection', + 'permission' => 'collection', 'read' => [$user], 'write' => [$user] ]); $this->assertEquals($collection['headers']['status-code'], 201); $this->assertEquals($collection['body']['name'], 'enforceCollectionPermissions'); - $this->assertEquals($collection['body']['enforce'], 'collection'); + $this->assertEquals($collection['body']['permission'], 'collection'); $collectionId = $collection['body']['$id']; @@ -1136,7 +1136,7 @@ trait DatabaseBase ]); $this->assertEquals(201, $attribute['headers']['status-code'], 201); - $this->assertEquals('attribute', $attribute['body']['$id']); + $this->assertEquals('attribute', $attribute['body']['key']); // wait for db to add attribute sleep(2); @@ -1148,11 +1148,11 @@ trait DatabaseBase ]), [ 'indexId' => 'key_attribute', 'type' => 'key', - 'attributes' => [$attribute['body']['$id']], + 'attributes' => [$attribute['body']['key']], ]); $this->assertEquals(201, $index['headers']['status-code']); - $this->assertEquals('key_attribute', $index['body']['$id']); + $this->assertEquals('key_attribute', $index['body']['key']); // wait for db to add attribute sleep(2); @@ -1190,7 +1190,7 @@ trait DatabaseBase 'x-appwrite-key' => $this->getProject()['apiKey'] ]), [ 'name' => 'enforceCollectionPermissions', - 'enforce' => 'collection', + 'permission' => 'collection', 'read' => [$user], 'write' => [] ]); @@ -1224,7 +1224,7 @@ trait DatabaseBase 'x-appwrite-key' => $this->getProject()['apiKey'] ]), [ 'name' => 'enforceCollectionPermissions', - 'enforce' => 'collection', + 'permission' => 'collection', 'read' => [], 'write' => [] ]); @@ -1236,6 +1236,6 @@ trait DatabaseBase 'x-appwrite-project' => $this->getProject()['$id'], ])); - $this->assertEquals(401, $documents['headers']['status-code']); + $this->assertEquals(404, $documents['headers']['status-code']); } } \ No newline at end of file From 1b074a3c38b6aa49b85c8a26dcfaa19952ea686b Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sat, 28 Aug 2021 21:38:50 +0530 Subject: [PATCH 089/206] feat(model): added better description for rule --- src/Appwrite/Utopia/Response/Model/UsageProject.php | 4 ++-- src/Appwrite/Utopia/Response/Model/UsageStorage.php | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Appwrite/Utopia/Response/Model/UsageProject.php b/src/Appwrite/Utopia/Response/Model/UsageProject.php index 95bb3cd42d..c44af4cf6d 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageProject.php +++ b/src/Appwrite/Utopia/Response/Model/UsageProject.php @@ -59,9 +59,9 @@ class UsageProject extends Model 'example' => new stdClass, 'array' => true ]) - ->addRule('functions', [ + ->addRule('storage', [ 'type' => Response::MODEL_METRIC_LIST, - 'description' => 'Aggregated stats for the occupied storage size.', + 'description' => 'Aggregated stats for the occupied storage size (in bytes).', 'default' => [], 'example' => new stdClass, 'array' => true diff --git a/src/Appwrite/Utopia/Response/Model/UsageStorage.php b/src/Appwrite/Utopia/Response/Model/UsageStorage.php index 66eaed602c..4462a219a9 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageStorage.php +++ b/src/Appwrite/Utopia/Response/Model/UsageStorage.php @@ -19,7 +19,7 @@ class UsageStorage extends Model ]) ->addRule('storage', [ 'type' => Response::MODEL_METRIC_LIST, - 'description' => 'Aggregated stats for the occupied storage size.', + 'description' => 'Aggregated stats for the occupied storage size (in bytes).', 'default' => [], 'example' => new stdClass, 'array' => true From f7d657cc38ff6418f002fb5a7f11e9dec914a388 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sat, 28 Aug 2021 21:55:48 +0530 Subject: [PATCH 090/206] feat(model): use the new authorization skip methods --- app/controllers/api/database.php | 73 +++++++++++++++---------------- app/controllers/api/functions.php | 36 +++++++-------- app/controllers/api/projects.php | 36 +++++++-------- app/controllers/api/storage.php | 68 ++++++++++++++-------------- app/controllers/api/users.php | 37 ++++++++-------- 5 files changed, 122 insertions(+), 128 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 3b3e477d90..e2af571bf7 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -1,6 +1,5 @@ find('stats', [ - new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), - new Query('metric', Query::TYPE_EQUAL, [$metric]), - ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); - $stats[$metric] = []; - foreach ($requestDocs as $requestDoc) { - $stats[$metric][] = [ - 'value' => $requestDoc->getAttribute('value'), - 'date' => $requestDoc->getAttribute('time'), - ]; - } - - $stats[$metric] = array_reverse($stats[$metric]); - } - - Authorization::reset(); + Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) { + foreach ($metrics as $metric) { + $requestDocs = $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, [$metric]), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + $stats[$metric] = array_reverse($stats[$metric]); + } + }); $usage = new Document([ 'range' => $range, @@ -387,8 +385,6 @@ App::get('/v1/database/:collectionId/usage') 'limit' => 90, ], ]; - - Authorization::disable(); $metrics = [ "database.collections.$collectionId.documents.count", @@ -399,23 +395,24 @@ App::get('/v1/database/:collectionId/usage') ]; $stats = []; - foreach ($metrics as $metric) { - $requestDocs = $dbForInternal->find('stats', [ - new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), - new Query('metric', Query::TYPE_EQUAL, [$metric]), - ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); - $stats[$metric] = []; - foreach ($requestDocs as $requestDoc) { - $stats[$metric][] = [ - 'value' => $requestDoc->getAttribute('value'), - 'date' => $requestDoc->getAttribute('time'), - ]; - } - $stats[$metric] = array_reverse($stats[$metric]); - } - - Authorization::reset(); + Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) { + foreach ($metrics as $metric) { + $requestDocs = $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, [$metric]), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + $stats[$metric] = array_reverse($stats[$metric]); + } + }); $usage = new Document([ 'range' => $range, diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index dec4fa92bf..fadc323832 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -188,8 +188,6 @@ App::get('/v1/functions/:functionId/usage') ], ]; - Authorization::disable(); - $metrics = [ "functions.$functionId.executions", "functions.$functionId.failures", @@ -197,24 +195,24 @@ App::get('/v1/functions/:functionId/usage') ]; $stats = []; - foreach ($metrics as $metric) { - $requestDocs = $dbForInternal->find('stats', [ - new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), - new Query('metric', Query::TYPE_EQUAL, [$metric]), - ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); - $stats[$metric] = []; - foreach ($requestDocs as $requestDoc) { - $stats[$metric][] = [ - 'value' => $requestDoc->getAttribute('value'), - 'date' => $requestDoc->getAttribute('time'), - ]; - } - - $stats[$metric] = array_reverse($stats[$metric]); - } - - Authorization::reset(); + Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) { + foreach ($metrics as $metric) { + $requestDocs = $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, [$metric]), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + $stats[$metric] = array_reverse($stats[$metric]); + } + }); $usage = new Document([ 'range' => $range, diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 2d8311845a..0d1c3b713a 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -267,8 +267,6 @@ App::get('/v1/projects/:projectId/usage') $dbForInternal->setNamespace('project_' . $projectId . '_internal'); - Authorization::disable(); - $metrics = [ 'requests', 'network', @@ -280,24 +278,24 @@ App::get('/v1/projects/:projectId/usage') ]; $stats = []; - foreach ($metrics as $metric) { - $requestDocs = $dbForInternal->find('stats', [ - new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), - new Query('metric', Query::TYPE_EQUAL, [$metric]), - ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); - $stats[$metric] = []; - foreach ($requestDocs as $requestDoc) { - $stats[$metric][] = [ - 'value' => $requestDoc->getAttribute('value'), - 'date' => $requestDoc->getAttribute('time'), - ]; - } - - $stats[$metric] = array_reverse($stats[$metric]); - } - - Authorization::reset(); + Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) { + foreach ($metrics as $metric) { + $requestDocs = $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, [$metric]), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + $stats[$metric] = array_reverse($stats[$metric]); + } + }); $usage = new Document([ 'range' => $range, diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index c3804b2f60..a11d50b5e3 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -10,7 +10,7 @@ use Utopia\Validator\HexColor; use Utopia\Cache\Cache; use Utopia\Cache\Adapter\Filesystem; use Appwrite\ClamAV\Network; -use Appwrite\Database\Validator\Authorization; +use Utopia\Database\Validator\Authorization; use Appwrite\Database\Validator\CustomId; use Utopia\Database\Document; use Utopia\Database\Validator\UID; @@ -682,29 +682,30 @@ App::get('/v1/storage/usage') ], ]; - Authorization::disable(); - $metrics = [ "storage.total", "storage.files.count" ]; $stats = []; - foreach ($metrics as $metric) { - $requestDocs = $dbForInternal->find('stats', [ - new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), - new Query('metric', Query::TYPE_EQUAL, [$metric]), - ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); - $stats[$metric] = []; - foreach ($requestDocs as $requestDoc) { - $stats[$metric][] = [ - 'value' => $requestDoc->getAttribute('value'), - 'date' => $requestDoc->getAttribute('time'), - ]; - } - $stats[$metric] = array_reverse($stats[$metric]); - } + Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) { + foreach ($metrics as $metric) { + $requestDocs = $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, [$metric]), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + $stats[$metric] = array_reverse($stats[$metric]); + } + }); $usage = new Document([ 'range' => $range, @@ -757,7 +758,6 @@ App::get('/v1/storage/:bucketId/usage') ], ]; - Authorization::disable(); $metrics = [ "storage.buckets.$bucketId.files.create", "storage.buckets.$bucketId.files.read", @@ -766,22 +766,24 @@ App::get('/v1/storage/:bucketId/usage') ]; $stats = []; - foreach ($metrics as $metric) { - $requestDocs = $dbForInternal->find('stats', [ - new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), - new Query('metric', Query::TYPE_EQUAL, [$metric]), - ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); - $stats[$metric] = []; - foreach ($requestDocs as $requestDoc) { - $stats[$metric][] = [ - 'value' => $requestDoc->getAttribute('value'), - 'date' => $requestDoc->getAttribute('time'), - ]; - } - $stats[$metric] = array_reverse($stats[$metric]); - } - Authorization::reset(); + Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) { + foreach ($metrics as $metric) { + $requestDocs = $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, [$metric]), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + $stats[$metric] = array_reverse($stats[$metric]); + } + }); $usage = new Document([ 'range' => $range, diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 1895098bf2..7541487ba0 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -2,7 +2,6 @@ use Appwrite\Auth\Auth; use Appwrite\Auth\Validator\Password; -use Appwrite\Database\Validator\Authorization; use Appwrite\Utopia\Response; use Utopia\App; use Utopia\Exception; @@ -21,6 +20,7 @@ use Appwrite\Database\Validator\CustomId; use Appwrite\Utopia\Response\Model; use Utopia\Database\Database; use Utopia\Database\Query; +use Utopia\Database\Validator\Authorization; App::post('/v1/users') ->desc('Create User') @@ -649,7 +649,6 @@ App::get('/v1/users/usage') ], ]; - Authorization::disable(); $metrics = [ "users.count", "users.create", @@ -661,24 +660,24 @@ App::get('/v1/users/usage') ]; $stats = []; - foreach ($metrics as $metric) { - $requestDocs = $dbForInternal->find('stats', [ - new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), - new Query('metric', Query::TYPE_EQUAL, [$metric]), - ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); - $stats[$metric] = []; - foreach ($requestDocs as $requestDoc) { - $stats[$metric][] = [ - 'value' => $requestDoc->getAttribute('value'), - 'date' => $requestDoc->getAttribute('time'), - ]; - } - - $stats[$metric] = array_reverse($stats[$metric]); - } - - Authorization::reset(); + Authorization::skip(function() use ($dbForInternal, $period, $range, $metrics, &$stats) { + foreach ($metrics as $metric) { + $requestDocs = $dbForInternal->find('stats', [ + new Query('period', Query::TYPE_EQUAL, [$period[$range]['period']]), + new Query('metric', Query::TYPE_EQUAL, [$metric]), + ], $period[$range]['limit'], 0, ['time'], [Database::ORDER_DESC]); + + $stats[$metric] = []; + foreach ($requestDocs as $requestDoc) { + $stats[$metric][] = [ + 'value' => $requestDoc->getAttribute('value'), + 'date' => $requestDoc->getAttribute('time'), + ]; + } + $stats[$metric] = array_reverse($stats[$metric]); + } + }); $usage = new Document([ 'range' => $range, From a4e2aeb5e79615f544a3f4bbd2533021624e7adf Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sun, 29 Aug 2021 11:00:37 +0530 Subject: [PATCH 091/206] feat(usage): added provider metrics to endpoints --- app/controllers/api/users.php | 5 ++++- src/Appwrite/Utopia/Response/Model/UsageUsers.php | 7 +++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 7541487ba0..7fcdffcc8b 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -621,10 +621,11 @@ App::get('/v1/users/usage') ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_USAGE_USERS) ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) + ->param('provider', '', new WhiteList(['email', 'anonymous', 'oauth2-google', 'oauth2-apple'], true), 'Provider Name.', true) ->inject('response') ->inject('dbForInternal') ->inject('register') - ->action(function ($range, $response, $dbForInternal) { + ->action(function ($range, $provider, $response, $dbForInternal) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ @@ -656,6 +657,7 @@ App::get('/v1/users/usage') "users.update", "users.delete", "users.sessions.create", + "users.sessions.$provider.create", "users.sessions.delete" ]; @@ -687,6 +689,7 @@ App::get('/v1/users/usage') 'users.update' => $stats["users.update"], 'users.delete' => $stats["users.delete"], 'sessions.create' => $stats["users.sessions.create"], + 'sessions.provider.create' => $stats["users.sessions.$provider.create"], 'sessions.delete' => $stats["users.sessions.delete"] ]); diff --git a/src/Appwrite/Utopia/Response/Model/UsageUsers.php b/src/Appwrite/Utopia/Response/Model/UsageUsers.php index db8056f5c6..913c4f33fe 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageUsers.php +++ b/src/Appwrite/Utopia/Response/Model/UsageUsers.php @@ -59,6 +59,13 @@ class UsageUsers extends Model 'example' => new stdClass, 'array' => true ]) + ->addRule('sessions.provider.create', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for sessions created for a provider ( email, anonymous or oauth2 ).', + 'default' => null, + 'example' => new stdClass, + 'array' => true + ]) ->addRule('sessions.delete', [ 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for sessions deleted.', From 58a5320cf11f04ccebd35e92a71b846bdb9bed15 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sun, 29 Aug 2021 13:20:52 +0530 Subject: [PATCH 092/206] feat(tests): added tests for database usage --- tests/e2e/Scopes/SideConsole.php | 23 +++++ .../Services/Database/DatabaseConsoleTest.php | 89 +++++++++++++++++ .../Functions/FunctionsConsoleTest.php | 97 +++++++++++++++++++ 3 files changed, 209 insertions(+) create mode 100644 tests/e2e/Scopes/SideConsole.php create mode 100644 tests/e2e/Services/Database/DatabaseConsoleTest.php create mode 100644 tests/e2e/Services/Functions/FunctionsConsoleTest.php diff --git a/tests/e2e/Scopes/SideConsole.php b/tests/e2e/Scopes/SideConsole.php new file mode 100644 index 0000000000..ada99a6e6a --- /dev/null +++ b/tests/e2e/Scopes/SideConsole.php @@ -0,0 +1,23 @@ + 'http://localhost', + 'cookie' => 'a_session_console='. $this->getRoot()['session'], + 'x-appwrite-mode' => 'admin' + ]; + } + + /** + * @return string + */ + public function getSide() + { + return 'console'; + } +} diff --git a/tests/e2e/Services/Database/DatabaseConsoleTest.php b/tests/e2e/Services/Database/DatabaseConsoleTest.php new file mode 100644 index 0000000000..0921d2c50a --- /dev/null +++ b/tests/e2e/Services/Database/DatabaseConsoleTest.php @@ -0,0 +1,89 @@ +client->call(Client::METHOD_POST, '/database/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'collectionId' => 'unique()', + 'name' => 'Movies', + 'read' => ['role:all'], + 'write' => ['role:all'], + 'permission' => 'document', + ]); + + $this->assertEquals($movies['headers']['status-code'], 201); + $this->assertEquals($movies['body']['name'], 'Movies'); + + return ['moviesId' => $movies['body']['$id']]; + } + + public function testGetDatabaseUsage() + { + /** + * Test for SUCCESS + */ + + $response = $this->client->call(Client::METHOD_GET, '/database/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h' + ]); + + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertEquals($response['body']['range'], '24h'); + $this->assertIsArray($response['body']['documents.count']); + $this->assertIsArray($response['body']['collections.count']); + $this->assertIsArray($response['body']['documents.create']); + $this->assertIsArray($response['body']['documents.read']); + $this->assertIsArray($response['body']['documents.update']); + $this->assertIsArray($response['body']['documents.delete']); + $this->assertIsArray($response['body']['collections.create']); + $this->assertIsArray($response['body']['collections.read']); + $this->assertIsArray($response['body']['collections.update']); + $this->assertIsArray($response['body']['collections.delete']); + } + + + /** + * @depends testCreateCollection + */ + public function testGetCollectionUsage(array $data) + { + /** + * Test for SUCCESS + */ + + $response = $this->client->call(Client::METHOD_GET, '/database/'.$data['moviesId'].'/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h' + ]); + + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertEquals($response['body']['range'], '24h'); + $this->assertIsArray($response['body']['documents.count']); + $this->assertIsArray($response['body']['documents.create']); + $this->assertIsArray($response['body']['documents.read']); + $this->assertIsArray($response['body']['documents.update']); + $this->assertIsArray($response['body']['documents.delete']); + } +} \ No newline at end of file diff --git a/tests/e2e/Services/Functions/FunctionsConsoleTest.php b/tests/e2e/Services/Functions/FunctionsConsoleTest.php new file mode 100644 index 0000000000..99dd4d315d --- /dev/null +++ b/tests/e2e/Services/Functions/FunctionsConsoleTest.php @@ -0,0 +1,97 @@ + 'http://localhost', + 'cookie' => 'a_session_console=' . $this->getRoot()['session'], + 'x-appwrite-mode' => 'admin' + ]; + } + + public function testCreateCollection():array + { + /** + * Test for SUCCESS + */ + $movies = $this->client->call(Client::METHOD_POST, '/database/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'collectionId' => 'unique()', + 'name' => 'Movies', + 'read' => ['role:all'], + 'write' => ['role:all'], + 'permission' => 'document', + ]); + + $this->assertEquals($movies['headers']['status-code'], 201); + $this->assertEquals($movies['body']['name'], 'Movies'); + + return ['moviesId' => $movies['body']['$id']]; + } + + public function testGetDatabaseUsage() + { + /** + * Test for SUCCESS + */ + + $response = $this->client->call(Client::METHOD_GET, '/database/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h' + ]); + + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertEquals($response['body']['range'], '24h'); + $this->assertIsArray($response['body']['documents.count']); + $this->assertIsArray($response['body']['collections.count']); + $this->assertIsArray($response['body']['documents.create']); + $this->assertIsArray($response['body']['documents.read']); + $this->assertIsArray($response['body']['documents.update']); + $this->assertIsArray($response['body']['documents.delete']); + $this->assertIsArray($response['body']['collections.create']); + $this->assertIsArray($response['body']['collections.read']); + $this->assertIsArray($response['body']['collections.update']); + $this->assertIsArray($response['body']['collections.delete']); + } + + + /** + * @depends testCreateCollection + */ + public function testGetCollectionUsage(array $data) + { + /** + * Test for SUCCESS + */ + + $response = $this->client->call(Client::METHOD_GET, '/database/'.$data['moviesId'].'/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h' + ]); + + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertEquals($response['body']['range'], '24h'); + $this->assertIsArray($response['body']['documents.count']); + $this->assertIsArray($response['body']['documents.create']); + $this->assertIsArray($response['body']['documents.read']); + $this->assertIsArray($response['body']['documents.update']); + $this->assertIsArray($response['body']['documents.delete']); + } + +} \ No newline at end of file From 94a17e9ce06b65cd408cb7ff57e80426a8506b96 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sun, 29 Aug 2021 13:29:08 +0530 Subject: [PATCH 093/206] feat(tests): added tests for functions usage --- .../Services/Database/DatabaseConsoleTest.php | 36 +++++- .../Functions/FunctionsConsoleTest.php | 109 ++++++++---------- 2 files changed, 86 insertions(+), 59 deletions(-) diff --git a/tests/e2e/Services/Database/DatabaseConsoleTest.php b/tests/e2e/Services/Database/DatabaseConsoleTest.php index 0921d2c50a..f8d50188ce 100644 --- a/tests/e2e/Services/Database/DatabaseConsoleTest.php +++ b/tests/e2e/Services/Database/DatabaseConsoleTest.php @@ -36,6 +36,19 @@ class DatabaseConsoleTest extends Scope public function testGetDatabaseUsage() { + /** + * Test for FAILURE + */ + + $response = $this->client->call(Client::METHOD_GET, '/database/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '32h' + ]); + + $this->assertEquals($response['headers']['status-code'], 400); + /** * Test for SUCCESS */ @@ -68,9 +81,30 @@ class DatabaseConsoleTest extends Scope public function testGetCollectionUsage(array $data) { /** - * Test for SUCCESS + * Test for FAILURE */ + $response = $this->client->call(Client::METHOD_GET, '/database/'.$data['moviesId'].'/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '32h' + ]); + + $this->assertEquals($response['headers']['status-code'], 400); + + $response = $this->client->call(Client::METHOD_GET, '/database/randomCollectionId/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h' + ]); + + $this->assertEquals($response['headers']['status-code'], 404); + + /** + * Test for SUCCESS + */ $response = $this->client->call(Client::METHOD_GET, '/database/'.$data['moviesId'].'/usage', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'] diff --git a/tests/e2e/Services/Functions/FunctionsConsoleTest.php b/tests/e2e/Services/Functions/FunctionsConsoleTest.php index 99dd4d315d..669a42c2ee 100644 --- a/tests/e2e/Services/Functions/FunctionsConsoleTest.php +++ b/tests/e2e/Services/Functions/FunctionsConsoleTest.php @@ -5,80 +5,75 @@ namespace Tests\E2E\Services\Database; use Tests\E2E\Scopes\Scope; use Tests\E2E\Scopes\ProjectCustom; use Tests\E2E\Client; +use Tests\E2E\Scopes\SideConsole; -class DatabaseConsoleTest extends Scope +class FunctionsConsoleTest extends Scope { use ProjectCustom; + use SideConsole; - public function getHeaders():array + public function testCreateFunction():array { - return [ - 'origin' => 'http://localhost', - 'cookie' => 'a_session_console=' . $this->getRoot()['session'], - 'x-appwrite-mode' => 'admin' - ]; - } - - public function testCreateCollection():array - { - /** - * Test for SUCCESS - */ - $movies = $this->client->call(Client::METHOD_POST, '/database/collections', array_merge([ + $function = $this->client->call(Client::METHOD_POST, '/functions', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ - 'collectionId' => 'unique()', - 'name' => 'Movies', - 'read' => ['role:all'], - 'write' => ['role:all'], - 'permission' => 'document', + 'functionId' => 'unique()', + 'name' => 'Test', + 'execute' => ['user:'.$this->getUser()['$id']], + 'runtime' => 'php-8.0', + 'vars' => [ + 'funcKey1' => 'funcValue1', + 'funcKey2' => 'funcValue2', + 'funcKey3' => 'funcValue3', + ], + 'events' => [ + 'account.create', + 'account.delete', + ], + 'schedule' => '0 0 1 1 *', + 'timeout' => 10, ]); - $this->assertEquals($movies['headers']['status-code'], 201); - $this->assertEquals($movies['body']['name'], 'Movies'); + $this->assertEquals(201, $function['headers']['status-code']); - return ['moviesId' => $movies['body']['$id']]; + return [ + 'functionId' => $function['body']['$id'] + ]; } - public function testGetDatabaseUsage() - { - /** - * Test for SUCCESS - */ - - $response = $this->client->call(Client::METHOD_GET, '/database/usage', array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'] - ], $this->getHeaders()), [ - 'range' => '24h' - ]); - - $this->assertEquals($response['headers']['status-code'], 200); - $this->assertEquals($response['body']['range'], '24h'); - $this->assertIsArray($response['body']['documents.count']); - $this->assertIsArray($response['body']['collections.count']); - $this->assertIsArray($response['body']['documents.create']); - $this->assertIsArray($response['body']['documents.read']); - $this->assertIsArray($response['body']['documents.update']); - $this->assertIsArray($response['body']['documents.delete']); - $this->assertIsArray($response['body']['collections.create']); - $this->assertIsArray($response['body']['collections.read']); - $this->assertIsArray($response['body']['collections.update']); - $this->assertIsArray($response['body']['collections.delete']); - } - - /** - * @depends testCreateCollection + * @depends testCreateFunction */ public function testGetCollectionUsage(array $data) { + /** + * Test for FAILURE + */ + + $response = $this->client->call(Client::METHOD_GET, '/functions/'.$data['functionId'].'/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '232h' + ]); + + $this->assertEquals(400, $response['headers']['status-code']); + + $response = $this->client->call(Client::METHOD_GET, '/functions/randomFunctionId/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h' + ]); + + $this->assertEquals(404, $response['headers']['status-code']); + /** * Test for SUCCESS */ - $response = $this->client->call(Client::METHOD_GET, '/database/'.$data['moviesId'].'/usage', array_merge([ + $response = $this->client->call(Client::METHOD_GET, '/functions/'.$data['functionId'].'/usage', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'] ], $this->getHeaders()), [ @@ -87,11 +82,9 @@ class DatabaseConsoleTest extends Scope $this->assertEquals($response['headers']['status-code'], 200); $this->assertEquals($response['body']['range'], '24h'); - $this->assertIsArray($response['body']['documents.count']); - $this->assertIsArray($response['body']['documents.create']); - $this->assertIsArray($response['body']['documents.read']); - $this->assertIsArray($response['body']['documents.update']); - $this->assertIsArray($response['body']['documents.delete']); + $this->assertIsArray($response['body']['functions.executions']); + $this->assertIsArray($response['body']['functions.failures']); + $this->assertIsArray($response['body']['functions.compute']); } } \ No newline at end of file From 9eea6d55f737f41d199245a6e37327d31aae8e2d Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sun, 29 Aug 2021 13:41:29 +0530 Subject: [PATCH 094/206] feat(tests): added tests for projects usage --- ...Test.php => DatabaseConsoleClientTest.php} | 4 ++- ...est.php => FunctionsConsoleClientTest.php} | 3 ++- .../Projects/ProjectsConsoleClientTest.php | 26 +++++++------------ 3 files changed, 14 insertions(+), 19 deletions(-) rename tests/e2e/Services/Database/{DatabaseConsoleTest.php => DatabaseConsoleClientTest.php} (96%) rename tests/e2e/Services/Functions/{FunctionsConsoleTest.php => FunctionsConsoleClientTest.php} (96%) diff --git a/tests/e2e/Services/Database/DatabaseConsoleTest.php b/tests/e2e/Services/Database/DatabaseConsoleClientTest.php similarity index 96% rename from tests/e2e/Services/Database/DatabaseConsoleTest.php rename to tests/e2e/Services/Database/DatabaseConsoleClientTest.php index f8d50188ce..625f26107b 100644 --- a/tests/e2e/Services/Database/DatabaseConsoleTest.php +++ b/tests/e2e/Services/Database/DatabaseConsoleClientTest.php @@ -7,7 +7,7 @@ use Tests\E2E\Scopes\ProjectCustom; use Tests\E2E\Client; use Tests\E2E\Scopes\SideConsole; -class DatabaseConsoleTest extends Scope +class DatabaseConsoleClientTest extends Scope { use ProjectCustom; use SideConsole; @@ -61,6 +61,7 @@ class DatabaseConsoleTest extends Scope ]); $this->assertEquals($response['headers']['status-code'], 200); + $this->assertEquals(count($response['body']), 11); $this->assertEquals($response['body']['range'], '24h'); $this->assertIsArray($response['body']['documents.count']); $this->assertIsArray($response['body']['collections.count']); @@ -113,6 +114,7 @@ class DatabaseConsoleTest extends Scope ]); $this->assertEquals($response['headers']['status-code'], 200); + $this->assertEquals(count($response['body']), 6); $this->assertEquals($response['body']['range'], '24h'); $this->assertIsArray($response['body']['documents.count']); $this->assertIsArray($response['body']['documents.create']); diff --git a/tests/e2e/Services/Functions/FunctionsConsoleTest.php b/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php similarity index 96% rename from tests/e2e/Services/Functions/FunctionsConsoleTest.php rename to tests/e2e/Services/Functions/FunctionsConsoleClientTest.php index 669a42c2ee..d8bdb30b65 100644 --- a/tests/e2e/Services/Functions/FunctionsConsoleTest.php +++ b/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php @@ -7,7 +7,7 @@ use Tests\E2E\Scopes\ProjectCustom; use Tests\E2E\Client; use Tests\E2E\Scopes\SideConsole; -class FunctionsConsoleTest extends Scope +class FunctionsConsoleClientTest extends Scope { use ProjectCustom; use SideConsole; @@ -81,6 +81,7 @@ class FunctionsConsoleTest extends Scope ]); $this->assertEquals($response['headers']['status-code'], 200); + $this->assertEquals(count($response['body']), 4); $this->assertEquals($response['body']['range'], '24h'); $this->assertIsArray($response['body']['functions.executions']); $this->assertIsArray($response['body']['functions.failures']); diff --git a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php index 4d83efb416..1842891492 100644 --- a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php +++ b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php @@ -215,24 +215,16 @@ class ProjectsConsoleClientTest extends Scope ], $this->getHeaders())); $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals(count($response['body']), 8); $this->assertNotEmpty($response['body']); - $this->assertArrayHasKey('collections', $response['body']); - $this->assertArrayHasKey('documents', $response['body']); - $this->assertArrayHasKey('network', $response['body']); - $this->assertArrayHasKey('requests', $response['body']); - $this->assertArrayHasKey('storage', $response['body']); - $this->assertArrayHasKey('users', $response['body']); - $this->assertIsArray($response['body']['collections']['data']); - $this->assertIsInt($response['body']['collections']['total']); - $this->assertIsArray($response['body']['documents']['data']); - $this->assertIsInt($response['body']['documents']['total']); - $this->assertIsArray($response['body']['network']['data']); - $this->assertIsInt($response['body']['network']['total']); - $this->assertIsArray($response['body']['requests']['data']); - $this->assertIsInt($response['body']['requests']['total']); - $this->assertIsInt($response['body']['storage']['total']); - $this->assertIsArray($response['body']['users']['data']); - $this->assertIsInt($response['body']['users']['total']); + $this->assertEquals('30d', $response['body']['range']); + $this->assertIsArray($response['body']['requests']); + $this->assertIsArray($response['body']['network']); + $this->assertIsArray($response['body']['functions']); + $this->assertIsArray($response['body']['documents']); + $this->assertIsArray($response['body']['collections']); + $this->assertIsArray($response['body']['users']); + $this->assertIsArray($response['body']['storage']); /** * Test for FAILURE From 4454a518b78b891917332d18f39a266a047def2f Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sun, 29 Aug 2021 13:49:25 +0530 Subject: [PATCH 095/206] feat(tests): added tests for storage usage --- .../Storage/StorageConsoleClientTest.php | 86 ++++++++++++++++++- 1 file changed, 82 insertions(+), 4 deletions(-) diff --git a/tests/e2e/Services/Storage/StorageConsoleClientTest.php b/tests/e2e/Services/Storage/StorageConsoleClientTest.php index b2c9582960..6960661777 100644 --- a/tests/e2e/Services/Storage/StorageConsoleClientTest.php +++ b/tests/e2e/Services/Storage/StorageConsoleClientTest.php @@ -2,13 +2,91 @@ namespace Tests\E2E\Services\Storage; +use Tests\E2E\Client; use Tests\E2E\Scopes\Scope; -use Tests\E2E\Scopes\ProjectConsole; -use Tests\E2E\Scopes\SideClient; +use Tests\E2E\Scopes\ProjectCustom; +use Tests\E2E\Scopes\SideConsole; class StorageConsoleClientTest extends Scope { + use SideConsole; use StorageBase; - use ProjectConsole; - use SideClient; + use ProjectCustom; + + public function testGetStorageUsage() + { + /** + * Test for FAILURE + */ + $response = $this->client->call(Client::METHOD_GET, '/storage/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '32h' + ]); + + $this->assertEquals($response['headers']['status-code'], 400); + + /** + * Test for SUCCESS + */ + + $response = $this->client->call(Client::METHOD_GET, '/storage/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h' + ]); + + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertEquals(count($response['body']), 3); + $this->assertEquals($response['body']['range'], '24h'); + $this->assertIsArray($response['body']['storage']); + $this->assertIsArray($response['body']['files']); + } + + public function testGetStorageBucketUsage() + { + /** + * Test for FAILURE + */ + + $response = $this->client->call(Client::METHOD_GET, '/storage/default/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '32h' + ]); + + $this->assertEquals($response['headers']['status-code'], 400); + + // TODO: Uncomment once we implement check for missing bucketId in the usage endpoint. + + // $response = $this->client->call(Client::METHOD_GET, '/storage/randomBucketId/usage', array_merge([ + // 'content-type' => 'application/json', + // 'x-appwrite-project' => $this->getProject()['$id'] + // ], $this->getHeaders()), [ + // 'range' => '24h' + // ]); + + // $this->assertEquals($response['headers']['status-code'], 404); + + /** + * Test for SUCCESS + */ + $response = $this->client->call(Client::METHOD_GET, '/storage/default/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h' + ]); + + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertEquals(count($response['body']), 5); + $this->assertEquals($response['body']['range'], '24h'); + $this->assertIsArray($response['body']['files.create']); + $this->assertIsArray($response['body']['files.read']); + $this->assertIsArray($response['body']['files.update']); + $this->assertIsArray($response['body']['files.delete']); + } } \ No newline at end of file From 3aedad2061d020a2fc392273c167a6b7ab4d9812 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sun, 29 Aug 2021 14:13:14 +0530 Subject: [PATCH 096/206] feat(tests): added tests for users usage --- .../Utopia/Response/Model/UsageUsers.php | 2 +- .../Functions/FunctionsConsoleClientTest.php | 2 +- .../Services/Users/UsersConsoleClientTest.php | 83 +++++++++++++++++++ 3 files changed, 85 insertions(+), 2 deletions(-) create mode 100644 tests/e2e/Services/Users/UsersConsoleClientTest.php diff --git a/src/Appwrite/Utopia/Response/Model/UsageUsers.php b/src/Appwrite/Utopia/Response/Model/UsageUsers.php index 913c4f33fe..8813615062 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageUsers.php +++ b/src/Appwrite/Utopia/Response/Model/UsageUsers.php @@ -62,7 +62,7 @@ class UsageUsers extends Model ->addRule('sessions.provider.create', [ 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for sessions created for a provider ( email, anonymous or oauth2 ).', - 'default' => null, + 'default' => [], 'example' => new stdClass, 'array' => true ]) diff --git a/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php b/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php index d8bdb30b65..ee15a81dcb 100644 --- a/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsConsoleClientTest.php @@ -1,6 +1,6 @@ client->call(Client::METHOD_GET, '/users/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '32h', + 'provider' => 'email' + ]); + + $this->assertEquals($response['headers']['status-code'], 400); + + $response = $this->client->call(Client::METHOD_GET, '/users/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h', + 'provider' => 'some-random-provider' + ]); + + $this->assertEquals($response['headers']['status-code'], 400); + + /** + * Test for SUCCESS + */ + $response = $this->client->call(Client::METHOD_GET, '/users/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h', + 'provider' => 'email' + ]); + + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertEquals(count($response['body']), 9); + $this->assertEquals($response['body']['range'], '24h'); + $this->assertIsArray($response['body']['users.count']); + $this->assertIsArray($response['body']['users.create']); + $this->assertIsArray($response['body']['users.read']); + $this->assertIsArray($response['body']['users.update']); + $this->assertIsArray($response['body']['users.delete']); + $this->assertIsArray($response['body']['sessions.create']); + $this->assertIsArray($response['body']['sessions.provider.create']); + $this->assertIsArray($response['body']['sessions.delete']); + + $response = $this->client->call(Client::METHOD_GET, '/users/usage', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'] + ], $this->getHeaders()), [ + 'range' => '24h' + ]); + + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertEquals(count($response['body']), 9); + $this->assertEquals($response['body']['range'], '24h'); + $this->assertIsArray($response['body']['users.count']); + $this->assertIsArray($response['body']['users.create']); + $this->assertIsArray($response['body']['users.read']); + $this->assertIsArray($response['body']['users.update']); + $this->assertIsArray($response['body']['users.delete']); + $this->assertIsArray($response['body']['sessions.create']); + $this->assertIsArray($response['body']['sessions.provider.create']); + $this->assertIsArray($response['body']['sessions.delete']); + } +} \ No newline at end of file From 66294445ce47a2a7010ff63dc1c75ae25f47fb17 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Sun, 29 Aug 2021 14:24:50 +0530 Subject: [PATCH 097/206] feat(tests): added tests for users usage --- app/controllers/api/storage.php | 2 ++ src/Appwrite/Utopia/Response/Model/UsageBuckets.php | 7 +++++++ tests/e2e/Services/Storage/StorageConsoleClientTest.php | 4 ++-- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index a11d50b5e3..ee6210d25d 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -759,6 +759,7 @@ App::get('/v1/storage/:bucketId/usage') ]; $metrics = [ + "storage.buckets.$bucketId.files.count", "storage.buckets.$bucketId.files.create", "storage.buckets.$bucketId.files.read", "storage.buckets.$bucketId.files.update", @@ -787,6 +788,7 @@ App::get('/v1/storage/:bucketId/usage') $usage = new Document([ 'range' => $range, + 'files.count' => $stats["storage.buckets.$bucketId.files.count"], 'files.create' => $stats["storage.buckets.$bucketId.files.create"], 'files.read' => $stats["storage.buckets.$bucketId.files.read"], 'files.update' => $stats["storage.buckets.$bucketId.files.update"], diff --git a/src/Appwrite/Utopia/Response/Model/UsageBuckets.php b/src/Appwrite/Utopia/Response/Model/UsageBuckets.php index 11fb2e2f8c..8c66ff47c4 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageBuckets.php +++ b/src/Appwrite/Utopia/Response/Model/UsageBuckets.php @@ -17,6 +17,13 @@ class UsageBuckets extends Model 'default' => '', 'example' => '30d', ]) + ->addRule('files.count', [ + 'type' => Response::MODEL_METRIC_LIST, + 'description' => 'Aggregated stats for total number of files in this bucket.', + 'default' => [], + 'example' => new stdClass, + 'array' => true + ]) ->addRule('files.create', [ 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for files created.', diff --git a/tests/e2e/Services/Storage/StorageConsoleClientTest.php b/tests/e2e/Services/Storage/StorageConsoleClientTest.php index 6960661777..4ec1a6ce9a 100644 --- a/tests/e2e/Services/Storage/StorageConsoleClientTest.php +++ b/tests/e2e/Services/Storage/StorageConsoleClientTest.php @@ -30,7 +30,6 @@ class StorageConsoleClientTest extends Scope /** * Test for SUCCESS */ - $response = $this->client->call(Client::METHOD_GET, '/storage/usage', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'] @@ -82,8 +81,9 @@ class StorageConsoleClientTest extends Scope ]); $this->assertEquals($response['headers']['status-code'], 200); - $this->assertEquals(count($response['body']), 5); + $this->assertEquals(count($response['body']), 6); $this->assertEquals($response['body']['range'], '24h'); + $this->assertIsArray($response['body']['files.count']); $this->assertIsArray($response['body']['files.create']); $this->assertIsArray($response['body']['files.read']); $this->assertIsArray($response['body']['files.update']); From c84dc0fa92c774cd4d6e94d34721a53003edf47b Mon Sep 17 00:00:00 2001 From: Eldad Fux Date: Mon, 30 Aug 2021 10:19:29 +0300 Subject: [PATCH 098/206] init --- app/tasks/usage.php | 222 ++++++++++++++++++++++++++------------------ 1 file changed, 130 insertions(+), 92 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index fdbf1d3eb2..2de5c09377 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -11,15 +11,19 @@ use Utopia\CLI\Console; use Utopia\Database\Adapter\MariaDB; use Utopia\Database\Database; use Utopia\Database\Document; -use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; /** * Metrics We collect * + * General + * * requests * network * executions + * + * Database + * * database.collections.create * database.collections.read * database.collections.update @@ -32,10 +36,16 @@ use Utopia\Database\Validator\Authorization; * database.collections.{collectionId}.documents.read * database.collections.{collectionId}.documents.update * database.collections.{collectionId}.documents.delete + * + * Storage + * * storage.buckets.{bucketId}.files.create * storage.buckets.{bucketId}.files.read * storage.buckets.{bucketId}.files.update * storage.buckets.{bucketId}.files.delete + * + * Users + * * users.create * users.read * users.update @@ -71,7 +81,7 @@ $cli Console::title('Usage Aggregation V1'); Console::success(APP_NAME . ' usage aggregation process v1 has started'); - $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); //30 seconds + $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); // 30 seconds (by default) $periods = [ [ 'key' => '30m', @@ -189,16 +199,18 @@ $cli ], ]; + // TODO Maybe move this to the setResource method, and reuse in the http.php file $attempts = 0; $max = 10; $sleep = 1; + do { // connect to db try { $attempts++; $db = $register->get('db'); $redis = $register->get('cache'); break; // leave the do-while if successful - } catch (\Exception$e) { + } catch (\Exception $e) { Console::warning("Database not ready. Retrying connection ({$attempts})..."); if ($attempts >= $max) { throw new \Exception('Failed to connect to database: ' . $e->getMessage()); @@ -207,6 +219,7 @@ $cli } } while ($attempts < $max); + // TODO use inject $cacheAdapter = new Cache(new Redis($redis)); $dbForProject = new Database(new MariaDB($db), $cacheAdapter); $dbForConsole = new Database(new MariaDB($db), $cacheAdapter); @@ -223,9 +236,13 @@ $cli $loopStart = microtime(true); + /** + * Aggregate InfluxDB every 30 seconds + */ $client = $register->get('influxdb'); if ($client) { $database = $client->selectDB('telegraf'); + // sync data foreach ($globalMetrics as $metric => $options) { //for each metrics foreach ($periods as $period) { // aggregate data for each period @@ -250,12 +267,11 @@ $cli $points = $result->getPoints(); foreach ($points as $point) { $projectId = $point['projectId']; + if (!empty($projectId) && $projectId != 'console') { $dbForProject->setNamespace('project_' . $projectId . '_internal'); - if($metric == 'functions.functionId.executions') { - var_dump($points); - } $metricUpdated = $metric; + if (!empty($groupBy)) { $groupedBy = $point[$options['groupBy']] ?? ''; if (empty($groupedBy)) { @@ -263,9 +279,11 @@ $cli } $metricUpdated = str_replace($options['groupBy'], $groupedBy, $metric); } + $time = \strtotime($point['time']); $id = \md5($time . '_' . $period['key'] . '_' . $metricUpdated); //construct unique id for each metric using time, period and metric $value = (!empty($point['value'])) ? $point['value'] : 0; + try { $document = $dbForProject->getDocument('stats', $id); if ($document->isEmpty()) { @@ -279,12 +297,11 @@ $cli ])); } else { $dbForProject->updateDocument('stats', $document->getId(), - $document->setAttribute('value', $value)); + $document->setAttribute('value', $value)); } $latestTime[$metric][$period['key']] = $time; - } catch (\Exception$e) { - // if projects are deleted this might fail - Console::warning("Failed to save data for project {$projectId} and metric {$metricUpdated}"); + } catch (\Exception $e) { // if projects are deleted this might fail + Console::warning("Failed to save data for project {$projectId} and metric {$metricUpdated}: {$e->getMessage()}"); } } } @@ -292,119 +309,140 @@ $cli } } - if ($iterations % 30 == 0) { //every 15 minutes - // aggregate number of objects in database - // get count of all the documents per collection - - // buckets will have the same + /** + * Aggregate MariaDB every 15 minutes + * Some of the queries here might contain full-table scans. + */ + if ($iterations % 30 == 0) { // Every 15 minutes + // Aggregate number of objects in database + // Get count of all the documents per collection - + // Buckets will have the same + $latestProject = null; + do { $projects = $dbForConsole->find('projects', [], 100, orderAfter:$latestProject); - if (!empty($projects)) { - $latestProject = $projects[array_key_last($projects)]; + + if (empty($projects)) { + continue; + } - foreach ($projects as $project) { - $id = $project->getId(); + $latestProject = $projects[array_key_last($projects)]; - // get total storage - $dbForProject->setNamespace('project_' . $id . '_internal'); - $storageTotal = $dbForProject->sum('files', 'sizeOriginal') + $dbForProject->sum('tags', 'size'); + foreach ($projects as $project) { + $id = $project->getId(); - $dbForProject->createDocument('stats', new Document([ - '$id' => $dbForProject->getId(), - 'period' => '15m', - 'time' => time(), - 'metric' => 'storage.total', - 'value' => $storageTotal, - 'type' => 1, - ])); + // Get total storage + $dbForProject->setNamespace('project_' . $id . '_internal'); + $storageTotal = $dbForProject->sum('files', 'sizeOriginal') + $dbForProject->sum('tags', 'size'); - $collections = [ - 'users' => [ - 'namespace' => 'internal', - ], - 'collections' => [ - 'metricPrefix' => 'database', - 'namespace' => 'internal', - 'subCollections' => [ - 'documents' => [ - 'namespace' => 'external', - ], + $dbForProject->createDocument('stats', new Document([ + '$id' => $dbForProject->getId(), + 'period' => '15m', + 'time' => time(), + 'metric' => 'storage.total', + 'value' => $storageTotal, + 'type' => 1, + ])); + + $collections = [ + 'users' => [ + 'namespace' => 'internal', + ], + 'collections' => [ + 'metricPrefix' => 'database', + 'namespace' => 'internal', + 'subCollections' => [ // TODO better document this key + 'documents' => [ + 'namespace' => 'external', ], ], - 'files' => [ - 'metricPrefix' => 'storage', - 'namespace' => 'internal', - ], - ]; - foreach ($collections as $collection => $options) { - try { + ], + 'files' => [ + 'metricPrefix' => 'storage', + 'namespace' => 'internal', + ], + ]; + + foreach ($collections as $collection => $options) { + try { + $dbForProject->setNamespace("project_{$id}_{$options['namespace']}"); + $count = $dbForProject->count($collection); + $dbForProject->setNamespace("project_{$id}_internal"); + $metricPrefix = $options['metricPrefix'] ?? ''; + $metric = empty($metricPrefix) ? "{$collection}.count" : "{$metricPrefix}.{$collection}.count"; + + $dbForProject->createDocument('stats', new Document([ + '$id' => $dbForProject->getId(), + 'time' => time(), + 'period' => '15m', + 'metric' => $metric, + 'value' => $count, + 'type' => 1, + ])); + + $subCollections = $options['subCollections'] ?? []; + + if (empty($subCollections)) { + continue; + } + + $latestParent = null; + $subCollectionCounts = []; //total project level count of sub collections + + do { $dbForProject->setNamespace("project_{$id}_{$options['namespace']}"); - $count = $dbForProject->count($collection); - $dbForProject->setNamespace("project_{$id}_internal"); - $metricPrefix = $options['metricPrefix'] ?? ''; - $metric = empty($metricPrefix) ? "{$collection}.count" : "{$metricPrefix}.{$collection}.count"; - $dbForProject->createDocument('stats', new Document([ - '$id' => $dbForProject->getId(), - 'time' => time(), - 'period' => '15m', - 'metric' => $metric, - 'value' => $count, - 'type' => 1, - ])); + $parents = $dbForProject->find($collection, [], 100, orderAfter:$latestParent); - $subCollections = $options['subCollections'] ?? []; - if (!empty($subCollections)) { - $latestParent = null; - $subCollectionCounts = []; //total project level count of sub collections - do { - $dbForProject->setNamespace("project_{$id}_{$options['namespace']}"); - $parents = $dbForProject->find($collection, [], 100, orderAfter:$latestParent); - if (!empty($parents)) { - $latestParent = $parents[array_key_last($parents)]; - foreach ($parents as $parent) { - foreach ($subCollections as $subCollection => $subOptions) { - $dbForProject->setNamespace("project_{$id}_{$subOptions['namespace']}"); - $count = $dbForProject->count($parent->getId()); - $subCollectionsCounts[$subCollection] = ($subCollectionCounts[$subCollection] ?? 0) + $count; + if (empty($parents)) { + continue; + } - $dbForProject->setNamespace("project_{$id}_internal"); - $dbForProject->createDocument('stats', new Document([ - '$id' => $dbForProject->getId(), - 'time' => time(), - 'period' => '15m', - 'metric' => empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.count" : "{$metricPrefix}.{$collection}.{$parent->getId()}.{$subCollection}.count", - 'value' => $count, - 'type' => 1, - ])); - } - } - } - } while (!empty($parents)); + $latestParent = $parents[array_key_last($parents)]; + + foreach ($parents as $parent) { + foreach ($subCollections as $subCollection => $subOptions) { + $dbForProject->setNamespace("project_{$id}_{$subOptions['namespace']}"); + $count = $dbForProject->count($parent->getId()); + $subCollectionsCounts[$subCollection] = ($subCollectionCounts[$subCollection] ?? 0) + $count; - foreach ($subCollectionsCounts as $subCollection => $count) { $dbForProject->setNamespace("project_{$id}_internal"); + $dbForProject->createDocument('stats', new Document([ '$id' => $dbForProject->getId(), 'time' => time(), 'period' => '15m', - 'metric' => empty($metricPrefix) ? "{$subCollection}.count" : "{$metricPrefix}.{$subCollection}.count", + 'metric' => empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.count" : "{$metricPrefix}.{$collection}.{$parent->getId()}.{$subCollection}.count", 'value' => $count, 'type' => 1, ])); } } - } catch (\Exception$e) { - Console::warning("Failed to save database counters data for project {$collection}"); + } while (!empty($parents)); + + foreach ($subCollectionsCounts as $subCollection => $count) { + $dbForProject->setNamespace("project_{$id}_internal"); + $dbForProject->createDocument('stats', new Document([ + '$id' => $dbForProject->getId(), + 'time' => time(), + 'period' => '15m', + 'metric' => empty($metricPrefix) ? "{$subCollection}.count" : "{$metricPrefix}.{$subCollection}.count", + 'value' => $count, + 'type' => 1, + ])); } + } catch (\Exception $e) { + Console::warning("Failed to save database counters data for project {$collection}: {$e->getMessage()}"); } } } - } while (!empty($projects)); } + $iterations++; $loopTook = microtime(true) - $loopStart; $now = date('d-m-Y H:i:s', time()); + Console::info("[{$now}] Aggregation took {$loopTook} seconds"); }, $interval); - }); + }); \ No newline at end of file From 82085df58a839a47b910a1f511196492aeccf558 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Mon, 30 Aug 2021 09:33:45 +0200 Subject: [PATCH 099/206] Migrated project platforms to it's own collection --- app/config/collections2.php | 107 ++++++++++++++++++++++++++++++- app/controllers/api/projects.php | 36 ++++++----- app/init.php | 12 ++++ 3 files changed, 138 insertions(+), 17 deletions(-) diff --git a/app/config/collections2.php b/app/config/collections2.php index 8079bdd7c8..1e4e4fb306 100644 --- a/app/config/collections2.php +++ b/app/config/collections2.php @@ -511,7 +511,7 @@ $collections = [ 'required' => false, 'default' => null, 'array' => true, - 'filters' => ['json'], + 'filters' => ['subQueryProjectPlatforms'], ], [ '$id' => 'webhooks', @@ -558,6 +558,111 @@ $collections = [ ], ], + 'projectsPlatforms' => [ + '$collection' => Database::METADATA, + '$id' => 'projectsPlatforms', + 'name' => 'projectsPlatforms', + 'attributes' => [ + [ + '$id' => 'projectId', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'type', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 16, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'name', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 256, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'key', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'store', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 256, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'hostname', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 256, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'dateCreated', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'dateUpdated', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + ], + 'indexes' => [ + [ + '$id' => '_key_project', + 'type' => Database::INDEX_KEY, + 'attributes' => ['projectId'], + 'lengths' => [Database::LENGTH_KEY], + 'orders' => [Database::ORDER_ASC], + ], + ], + ], + 'users' => [ '$collection' => Database::METADATA, '$id' => 'users', diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index b83dff6d9e..2dc5ce9ac8 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -1047,8 +1047,13 @@ App::post('/v1/projects/:projectId/platforms') throw new Exception('Project not found', 404); } + $teamId = $project->getAttribute("teamId"); + $platform = new Document([ + 'projectId' => $project->getId(), '$id' => $dbForConsole->getId(), + '$read' => ['team:' . $teamId], + '$write' => ['team:' . $teamId . '/owner', 'team:' . $teamId . '/developer'], 'type' => $type, 'name' => $name, 'key' => $key, @@ -1058,9 +1063,8 @@ App::post('/v1/projects/:projectId/platforms') 'dateUpdated' => \time(), ]); - $project = $dbForConsole->updateDocument('projects', $project->getId(), $project - ->setAttribute('platforms', $platform, Document::SET_TYPE_APPEND) - ); + $dbForConsole->createDocument("projectsPlatforms", $platform); + $dbForConsole->purgeDocument('projects', $project->getId()); $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($platform, Response::MODEL_PLATFORM); @@ -1083,13 +1087,17 @@ App::get('/v1/projects/:projectId/platforms') /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForConsole */ + // TODO: Implement pagination + $project = $dbForConsole->getDocument('projects', $projectId); if ($project->isEmpty()) { throw new Exception('Project not found', 404); } - $platforms = $project->getAttribute('platforms', []); + $platforms = $dbForConsole->find('projectsPlatforms', [ + new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) + ]); $response->dynamic(new Document([ 'platforms' => $platforms, @@ -1121,7 +1129,7 @@ App::get('/v1/projects/:projectId/platforms/:platformId') throw new Exception('Project not found', 404); } - $platform = $project->find('$id', $platformId, 'platforms'); + $platform = $dbForConsole->getDocument("projectsPlatforms", $platformId); if (empty($platform) || !$platform instanceof Document) { throw new Exception('Platform not found', 404); @@ -1158,7 +1166,7 @@ App::put('/v1/projects/:projectId/platforms/:platformId') throw new Exception('Project not found', 404); } - $platform = $project->find('$id', $platformId, 'platforms'); + $platform = $dbForConsole->getDocument("projectsPlatforms", $platformId); if (empty($platform) || !$platform instanceof Document) { throw new Exception('Platform not found', 404); @@ -1172,15 +1180,9 @@ App::put('/v1/projects/:projectId/platforms/:platformId') ->setAttribute('hostname', $hostname) ; - $project->findAndReplace('$id', $platform->getId(), $platform - ->setAttribute('name', $name) - ->setAttribute('dateUpdated', \time()) - ->setAttribute('key', $key) - ->setAttribute('store', $store) - ->setAttribute('hostname', $hostname) - , 'platforms'); + $dbForConsole->updateDocument('projectsPlatforms', $platform->getId(), $platform); - $dbForConsole->updateDocument('projects', $project->getId(), $project); + $dbForConsole->purgeDocument('projects', $project->getId()); $response->dynamic($platform, Response::MODEL_PLATFORM); }); @@ -1208,11 +1210,13 @@ App::delete('/v1/projects/:projectId/platforms/:platformId') throw new Exception('Project not found', 404); } - if (!$project->findAndRemove('$id', $platformId, 'platforms')) { + $deleteRecord = $dbForConsole->deleteDocument("projectsPlatforms", $platformId); + + if($deleteRecord == false) { throw new Exception('Platform not found', 404); } - $dbForConsole->updateDocument('projects', $project->getId(), $project); + $dbForConsole->purgeDocument('projects', $project->getId()); $response->noContent(); }); diff --git a/app/init.php b/app/init.php index 57f2dc7130..4da737ebbe 100644 --- a/app/init.php +++ b/app/init.php @@ -200,6 +200,18 @@ Database::addFilter('subQueryIndexes', } ); +Database::addFilter('subQueryProjectPlatforms', + function($value) { + return null; + }, + function($value, Document $document, Database $database) { + return $database + ->find('projectsPlatforms', [ + new Query('collectionId', Query::TYPE_EQUAL, [$document->getId()]) + ], 100, 0, []); + } +); + Database::addFilter('encrypt', function($value) { $key = App::getEnv('_APP_OPENSSL_KEY_V1'); From 723bddf95ab876521be0803e9869407b21484745 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Mon, 30 Aug 2021 13:14:45 +0530 Subject: [PATCH 100/206] feat(compose): disable debug mode --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 12bc51b802..539799aa9a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -39,7 +39,7 @@ services: build: context: . args: - - DEBUG=true + - DEBUG=false - TESTING=true - VERSION=dev ports: From 2477b80978534a797d5a69118b47a0537c805856 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 30 Aug 2021 15:20:32 +0545 Subject: [PATCH 101/206] update counters also as time series data --- app/tasks/usage.php | 250 +++++++++++++++++++++++++++++++------------- 1 file changed, 179 insertions(+), 71 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 2de5c09377..3fa90a3f43 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -17,13 +17,13 @@ use Utopia\Database\Validator\Authorization; * Metrics We collect * * General - * + * * requests * network * executions - * + * * Database - * + * * database.collections.create * database.collections.read * database.collections.update @@ -36,16 +36,16 @@ use Utopia\Database\Validator\Authorization; * database.collections.{collectionId}.documents.read * database.collections.{collectionId}.documents.update * database.collections.{collectionId}.documents.delete - * + * * Storage - * + * * storage.buckets.{bucketId}.files.create * storage.buckets.{bucketId}.files.read * storage.buckets.{bucketId}.files.update * storage.buckets.{bucketId}.files.delete - * + * * Users - * + * * users.create * users.read * users.update @@ -203,7 +203,7 @@ $cli $attempts = 0; $max = 10; $sleep = 1; - + do { // connect to db try { $attempts++; @@ -219,7 +219,7 @@ $cli } } while ($attempts < $max); - // TODO use inject + // TODO use inject $cacheAdapter = new Cache(new Redis($redis)); $dbForProject = new Database(new MariaDB($db), $cacheAdapter); $dbForConsole = new Database(new MariaDB($db), $cacheAdapter); @@ -242,7 +242,7 @@ $cli $client = $register->get('influxdb'); if ($client) { $database = $client->selectDB('telegraf'); - + // sync data foreach ($globalMetrics as $metric => $options) { //for each metrics foreach ($periods as $period) { // aggregate data for each period @@ -252,10 +252,10 @@ $cli } $end = DateTime::createFromFormat('U', \strtotime('now'))->format(DateTime::RFC3339); - $table = $options['table']; //which influxdb table to query for this metric - $groupBy = empty($options['groupBy']) ? '' : ', "' . $options['groupBy'] . '"'; //some sub level metrics may be grouped by other tags like collectionId, bucketId, etc + $table = $options['table']; //Which influxdb table to query for this metric + $groupBy = empty($options['groupBy']) ? '' : ', "' . $options['groupBy'] . '"'; //Some sub level metrics may be grouped by other tags like collectionId, bucketId, etc - $filters = $options['filters'] ?? []; + $filters = $options['filters'] ?? []; // Some metrics might have additional filters, like function's status if (!empty($filters)) { $filters = ' AND ' . implode(' AND ', array_map(function ($filter, $value) { return '"' . $filter . '"=\'' . $value . '\''; @@ -263,7 +263,7 @@ $cli } $result = $database->query('SELECT sum(value) AS "value" FROM "' . $table . '" WHERE time > \'' . $start . '\' AND time < \'' . $end . '\' AND "metric_type"=\'counter\'' . (empty($filters) ? '' : $filters) . ' GROUP BY time(' . $period['key'] . '), "projectId"' . $groupBy . ' FILL(null)'); - + $points = $result->getPoints(); foreach ($points as $point) { $projectId = $point['projectId']; @@ -271,7 +271,7 @@ $cli if (!empty($projectId) && $projectId != 'console') { $dbForProject->setNamespace('project_' . $projectId . '_internal'); $metricUpdated = $metric; - + if (!empty($groupBy)) { $groupedBy = $point[$options['groupBy']] ?? ''; if (empty($groupedBy)) { @@ -281,9 +281,9 @@ $cli } $time = \strtotime($point['time']); - $id = \md5($time . '_' . $period['key'] . '_' . $metricUpdated); //construct unique id for each metric using time, period and metric + $id = \md5($time . '_' . $period['key'] . '_' . $metricUpdated); //Construct unique id for each metric using time, period and metric $value = (!empty($point['value'])) ? $point['value'] : 0; - + try { $document = $dbForProject->getDocument('stats', $id); if ($document->isEmpty()) { @@ -310,19 +310,19 @@ $cli } /** - * Aggregate MariaDB every 15 minutes - * Some of the queries here might contain full-table scans. - */ + * Aggregate MariaDB every 15 minutes + * Some of the queries here might contain full-table scans. + */ if ($iterations % 30 == 0) { // Every 15 minutes // Aggregate number of objects in database // Get count of all the documents per collection - // Buckets will have the same $latestProject = null; - + do { $projects = $dbForConsole->find('projects', [], 100, orderAfter:$latestProject); - + if (empty($projects)) { continue; } @@ -330,20 +330,45 @@ $cli $latestProject = $projects[array_key_last($projects)]; foreach ($projects as $project) { - $id = $project->getId(); + $projectId = $project->getId(); // Get total storage - $dbForProject->setNamespace('project_' . $id . '_internal'); + $dbForProject->setNamespace('project_' . $projectId . '_internal'); $storageTotal = $dbForProject->sum('files', 'sizeOriginal') + $dbForProject->sum('tags', 'size'); - $dbForProject->createDocument('stats', new Document([ - '$id' => $dbForProject->getId(), - 'period' => '15m', - 'time' => time(), - 'metric' => 'storage.total', - 'value' => $storageTotal, - 'type' => 1, - ])); + $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes + $id = \md5($time . '_30m_storage.total'); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'period' => '30m', + 'time' => $time, + 'metric' => 'storage.total', + 'value' => $storageTotal, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $storageTotal)); + } + + $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day + $id = \md5($time . '_1d_storage.total'); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'period' => '1d', + 'time' => $time, + 'metric' => 'storage.total', + 'value' => $storageTotal, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $storageTotal)); + } $collections = [ 'users' => [ @@ -352,7 +377,7 @@ $cli 'collections' => [ 'metricPrefix' => 'database', 'namespace' => 'internal', - 'subCollections' => [ // TODO better document this key + 'subCollections' => [ // Some collections, like collections and later buckets have child collections that need counting 'documents' => [ 'namespace' => 'external', ], @@ -366,20 +391,45 @@ $cli foreach ($collections as $collection => $options) { try { - $dbForProject->setNamespace("project_{$id}_{$options['namespace']}"); + $dbForProject->setNamespace("project_{$projectId}_{$options['namespace']}"); $count = $dbForProject->count($collection); - $dbForProject->setNamespace("project_{$id}_internal"); + $dbForProject->setNamespace("project_{$projectId}_internal"); $metricPrefix = $options['metricPrefix'] ?? ''; $metric = empty($metricPrefix) ? "{$collection}.count" : "{$metricPrefix}.{$collection}.count"; - - $dbForProject->createDocument('stats', new Document([ - '$id' => $dbForProject->getId(), - 'time' => time(), - 'period' => '15m', - 'metric' => $metric, - 'value' => $count, - 'type' => 1, - ])); + + $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes + $id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'time' => $time, + 'period' => '30m', + 'metric' => $metric, + 'value' => $count, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $count)); + } + + $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day + $id = \md5($time . '_1d_' . $metric); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'time' => $time, + 'period' => '1d', + 'metric' => $metric, + 'value' => $count, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $count)); + } $subCollections = $options['subCollections'] ?? []; @@ -391,8 +441,8 @@ $cli $subCollectionCounts = []; //total project level count of sub collections do { - $dbForProject->setNamespace("project_{$id}_{$options['namespace']}"); - $parents = $dbForProject->find($collection, [], 100, orderAfter:$latestParent); + $dbForProject->setNamespace("project_{$projectId}_{$options['namespace']}"); + $parents = $dbForProject->find($collection, [], 100, orderAfter:$latestParent); // Get all the parents for the sub collections for example for documents, this will get all the collections if (empty($parents)) { continue; @@ -401,37 +451,95 @@ $cli $latestParent = $parents[array_key_last($parents)]; foreach ($parents as $parent) { - foreach ($subCollections as $subCollection => $subOptions) { - $dbForProject->setNamespace("project_{$id}_{$subOptions['namespace']}"); + foreach ($subCollections as $subCollection => $subOptions) { // Sub collection counts, like database.collections.collectionId.documents.count + $dbForProject->setNamespace("project_{$projectId}_{$subOptions['namespace']}"); $count = $dbForProject->count($parent->getId()); - $subCollectionsCounts[$subCollection] = ($subCollectionCounts[$subCollection] ?? 0) + $count; - $dbForProject->setNamespace("project_{$id}_internal"); + $subCollectionCounts[$subCollection] = ($subCollectionCounts[$subCollection] ?? 0) + $count; // Project level counts for sub collections like database.documents.count - $dbForProject->createDocument('stats', new Document([ - '$id' => $dbForProject->getId(), - 'time' => time(), - 'period' => '15m', - 'metric' => empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.count" : "{$metricPrefix}.{$collection}.{$parent->getId()}.{$subCollection}.count", - 'value' => $count, - 'type' => 1, - ])); + $dbForProject->setNamespace("project_{$projectId}_internal"); + + $metric = empty($metricPrefix) ? "{$collection}.{$parent->getId()}.{$subCollection}.count" : "{$metricPrefix}.{$collection}.{$parent->getId()}.{$subCollection}.count"; + $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes + $id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'time' => $time, + 'period' => '30m', + 'metric' => $metric, + 'value' => $count, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $count)); + } + + $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day + $id = \md5($time . '_1d_' . $metric); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'time' => $time, + 'period' => '1d', + 'metric' => $metric, + 'value' => $count, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $count)); + } } } } while (!empty($parents)); - foreach ($subCollectionsCounts as $subCollection => $count) { - $dbForProject->setNamespace("project_{$id}_internal"); - $dbForProject->createDocument('stats', new Document([ - '$id' => $dbForProject->getId(), - 'time' => time(), - 'period' => '15m', - 'metric' => empty($metricPrefix) ? "{$subCollection}.count" : "{$metricPrefix}.{$subCollection}.count", - 'value' => $count, - 'type' => 1, - ])); + /** + * Inserting project level counts for sub collections like database.documents.count + */ + foreach ($subCollectionCounts as $subCollection => $count) { + $dbForProject->setNamespace("project_{$projectId}_internal"); + + $metric = empty($metricPrefix) ? "{$subCollection}.count" : "{$metricPrefix}.{$subCollection}.count"; + + $time = (int) (floor(time() / 1800) * 1800); // Time rounded to nearest 30 minutes + $id = \md5($time . '_30m_' . $metric); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'time' => $time, + 'period' => '30m', + 'metric' => $metric, + 'value' => $count, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $count)); + } + + $time = (int) (floor(time() / 86400) * 86400); // Time rounded to nearest day + $id = \md5($time . '_1d_' . $metric); //Construct unique id for each metric using time, period and metric + $document = $dbForProject->getDocument('stats', $id); + if ($document->isEmpty()) { + $dbForProject->createDocument('stats', new Document([ + '$id' => $id, + 'time' => $time, + 'period' => '1d', + 'metric' => $metric, + 'value' => $count, + 'type' => 1, + ])); + } else { + $dbForProject->updateDocument('stats', $document->getId(), + $document->setAttribute('value', $count)); + } } - } catch (\Exception $e) { + } catch (\Exception$e) { Console::warning("Failed to save database counters data for project {$collection}: {$e->getMessage()}"); } } @@ -442,7 +550,7 @@ $cli $iterations++; $loopTook = microtime(true) - $loopStart; $now = date('d-m-Y H:i:s', time()); - + Console::info("[{$now}] Aggregation took {$loopTook} seconds"); }, $interval); - }); \ No newline at end of file + }); From 87311797f2ece141467966599175cc3ef5313e58 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 30 Aug 2021 15:23:11 +0545 Subject: [PATCH 102/206] more comments --- app/tasks/usage.php | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 3fa90a3f43..e66bec113f 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -310,17 +310,14 @@ $cli } /** - * Aggregate MariaDB every 15 minutes - * Some of the queries here might contain full-table scans. - */ - if ($iterations % 30 == 0) { // Every 15 minutes - // Aggregate number of objects in database - // Get count of all the documents per collection - - // Buckets will have the same + * Aggregate MariaDB every 15 minutes + * Some of the queries here might contain full-table scans. + */ + if ($iterations % 30 == 0) { // Every 15 minutes aggregate number of objects in database $latestProject = null; - do { + do { // Loop over all the projects $projects = $dbForConsole->find('projects', [], 100, orderAfter:$latestProject); if (empty($projects)) { @@ -440,7 +437,7 @@ $cli $latestParent = null; $subCollectionCounts = []; //total project level count of sub collections - do { + do { // Loop over all the parent collection document for each sub collection $dbForProject->setNamespace("project_{$projectId}_{$options['namespace']}"); $parents = $dbForProject->find($collection, [], 100, orderAfter:$latestParent); // Get all the parents for the sub collections for example for documents, this will get all the collections From 8009adc11d478f83507203c9f94212cf9805fd67 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Mon, 30 Aug 2021 16:24:50 +0200 Subject: [PATCH 103/206] Migrated project services, providers, domains, webhooks and keys to it's own collection --- app/config/collections2.php | 378 ++++++++++++++++++++++++++++++- app/config/services.php | 4 + app/controllers/api/projects.php | 244 ++++++++++++++------ app/init.php | 86 ++++++- 4 files changed, 629 insertions(+), 83 deletions(-) diff --git a/app/config/collections2.php b/app/config/collections2.php index 1e4e4fb306..a29a9728c8 100644 --- a/app/config/collections2.php +++ b/app/config/collections2.php @@ -478,7 +478,7 @@ $collections = [ 'required' => false, 'default' => null, 'array' => false, - 'filters' => ['json'], + 'filters' => ['subQueryProjectServices'], ], [ '$id' => 'auths', @@ -500,7 +500,7 @@ $collections = [ 'required' => false, 'default' => null, 'array' => false, - 'filters' => ['json'], + 'filters' => ['subQueryProjectProviders'], ], [ '$id' => 'platforms', @@ -510,7 +510,7 @@ $collections = [ 'signed' => true, 'required' => false, 'default' => null, - 'array' => true, + 'array' => false, 'filters' => ['subQueryProjectPlatforms'], ], [ @@ -521,8 +521,8 @@ $collections = [ 'signed' => true, 'required' => false, 'default' => null, - 'array' => true, - 'filters' => ['json'], + 'array' => false, + 'filters' => ['subQueryProjectWebhooks'], ], [ '$id' => 'keys', @@ -532,8 +532,8 @@ $collections = [ 'signed' => true, 'required' => false, 'default' => null, - 'array' => true, - 'filters' => ['json'], + 'array' => false, + 'filters' => ['subQueryProjectKeys'], ], [ '$id' => 'domains', @@ -543,8 +543,8 @@ $collections = [ 'signed' => true, 'required' => false, 'default' => null, - 'array' => true, - 'filters' => ['json'], + 'array' => false, + 'filters' => ['subQueryProjectDomains'], ], ], 'indexes' => [ @@ -663,6 +663,366 @@ $collections = [ ], ], + 'projectsServices' => [ + '$collection' => Database::METADATA, + '$id' => 'projectsServices', + 'name' => 'projectsServices', + 'attributes' => [ + [ + '$id' => 'projectId', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'key', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'status', + 'type' => Database::VAR_BOOLEAN, + 'format' => '', + 'size' => 0, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ] + ], + 'indexes' => [ + [ + '$id' => '_key_project', + 'type' => Database::INDEX_KEY, + 'attributes' => ['projectId'], + 'lengths' => [Database::LENGTH_KEY], + 'orders' => [Database::ORDER_ASC], + ], + ], + ], + + 'projectProviders' => [ + '$collection' => Database::METADATA, + '$id' => 'projectProviders', + 'name' => 'projectProviders', + 'attributes' => [ + [ + '$id' => 'projectId', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'key', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'appId', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'appSecret', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 16384, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + ], + 'indexes' => [ + [ + '$id' => '_key_project', + 'type' => Database::INDEX_KEY, + 'attributes' => ['projectId'], + 'lengths' => [Database::LENGTH_KEY], + 'orders' => [Database::ORDER_ASC], + ], + ], + ], + + 'projectDomains' => [ + '$collection' => Database::METADATA, + '$id' => 'projectDomains', + 'name' => 'projectDomains', + 'attributes' => [ + [ + '$id' => 'projectId', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'updated', + 'type' => Database::VAR_INTEGER, + 'format' => '', + 'size' => 0, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'domain', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'tld', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'registerable', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'verification', + 'type' => Database::VAR_BOOLEAN, + 'format' => '', + 'size' => 0, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'certificateId', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + ], + 'indexes' => [ + [ + '$id' => '_key_project', + 'type' => Database::INDEX_KEY, + 'attributes' => ['projectId'], + 'lengths' => [Database::LENGTH_KEY], + 'orders' => [Database::ORDER_ASC], + ], + ], + ], + + 'projectKeys' => [ + '$collection' => Database::METADATA, + '$id' => 'projectKeys', + 'name' => 'projectKeys', + 'attributes' => [ + [ + '$id' => 'projectId', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'name', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'scopes', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => true, + 'filters' => [], + ], + [ + '$id' => 'secret', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 256, // var_dump of \bin2hex(\random_bytes(128)) => string(256) + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + ], + 'indexes' => [ + [ + '$id' => '_key_project', + 'type' => Database::INDEX_KEY, + 'attributes' => ['projectId'], + 'lengths' => [Database::LENGTH_KEY], + 'orders' => [Database::ORDER_ASC], + ], + ], + ], + + 'projectWebhooks' => [ + '$collection' => Database::METADATA, + '$id' => 'projectWebhooks', + 'name' => 'projectWebhooks', + 'attributes' => [ + [ + '$id' => 'projectId', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'name', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'url', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'httpUser', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'httpPass', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'security', + 'type' => Database::VAR_BOOLEAN, + 'format' => '', + 'size' => 0, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'events', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => Database::LENGTH_KEY, + 'signed' => true, + 'required' => true, + 'default' => null, + 'array' => true, + 'filters' => [], + ], + ], + 'indexes' => [ + [ + '$id' => '_key_project', + 'type' => Database::INDEX_KEY, + 'attributes' => ['projectId'], + 'lengths' => [Database::LENGTH_KEY], + 'orders' => [Database::ORDER_ASC], + ], + ], + ], + 'users' => [ '$collection' => Database::METADATA, '$id' => 'users', diff --git a/app/config/services.php b/app/config/services.php index 09ea35f746..17375c360a 100644 --- a/app/config/services.php +++ b/app/config/services.php @@ -5,6 +5,7 @@ return [ 'key' => 'homepage', 'name' => 'Homepage', 'subtitle' => '', + 'description' => '', 'controller' => 'web/home.php', 'sdk' => false, 'docs' => false, @@ -16,6 +17,8 @@ return [ 'console' => [ 'key' => 'console', 'name' => 'Console', + 'subtitle' => '', + 'description' => '', 'controller' => 'web/console.php', 'sdk' => false, 'docs' => false, @@ -93,6 +96,7 @@ return [ 'key' => 'projects', 'name' => 'Projects', 'subtitle' => 'The Project service allows you to manage all the projects in your Appwrite server.', + 'description' => '', 'controller' => 'api/projects.php', 'sdk' => true, 'docs' => true, diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 2dc5ce9ac8..15dee15f8d 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -7,6 +7,7 @@ use Appwrite\Network\Validator\Domain as DomainValidator; use Appwrite\Network\Validator\URL; use Appwrite\Utopia\Response; use Utopia\App; +use Utopia\CLI\CLI; use Utopia\Config\Config; use Utopia\Database\Document; use Utopia\Database\Query; @@ -92,14 +93,17 @@ App::post('/v1/projects') 'legalCity' => $legalCity, 'legalAddress' => $legalAddress, 'legalTaxId' => $legalTaxId, - 'services' => new stdClass(), - 'platforms' => [], - 'webhooks' => [], - 'keys' => [], - 'domains' => [], - 'auths' => $auths, + 'services' => null, + 'platforms' => null, + 'providers' => null, + 'webhooks' => null, + 'keys' => null, + 'domains' => null, + 'auths' => null ])); + // TODO: Implement write of auths from $auths + $collections = Config::getParam('collections2', []); /** @var array $collections */ $dbForInternal->setNamespace('project_' . $project->getId() . '_internal'); @@ -468,12 +472,13 @@ App::patch('/v1/projects/:projectId/service') ->label('sdk.response.model', Response::MODEL_PROJECT) ->param('projectId', '', new UID(), 'Project unique ID.') ->param('service', '', new WhiteList(array_keys(array_filter(Config::getParam('services'), function($element) {return $element['optional'];})), true), 'Service name.') - ->param('status', null, new Boolean(), 'Service status.') + ->param('status', true, new Boolean(), 'Service status.') ->inject('response') ->inject('dbForConsole') - ->action(function ($projectId, $service, $status, $response, $dbForConsole) { + ->action(function ($projectId, $serviceKey, $status, $response, $dbForConsole) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForConsole */ + /** @var Boolean $status */ $project = $dbForConsole->getDocument('projects', $projectId); @@ -481,11 +486,37 @@ App::patch('/v1/projects/:projectId/service') throw new Exception('Project not found', 404); } - $services = $project->getAttribute('services', []); - $services[$service] = $status; + $service = $dbForConsole->findOne("projectsServices", [ + new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]), + new Query('key', Query::TYPE_EQUAL, [$serviceKey]), + ]); - $project = $dbForConsole->updateDocument('projects', $project->getId(), $project->setAttribute('services', $services)); + // TODO: Implement delete method. Do we delete for "true" or for "false"? Same for auth!!!! + if($service == false || $service->isEmpty()) { + $teamId = $project->getAttribute("teamId"); + $service = new Document([ + '$id' => $dbForConsole->getId(), + '$read' => ['team:' . $teamId], + '$write' => ['team:' . $teamId . '/owner', 'team:' . $teamId . '/developer'], + 'projectId' => $project->getId(), + 'key' => $serviceKey, + 'status' => $status, + ]); + + $dbForConsole->createDocument("projectsServices", $service); + + $dbForConsole->purgeDocument('projects', $project->getId()); + } else { + if($service->getAttribute("status") != $status) { + $service->setAttribute("status", $status); + $dbForConsole->updateDocument("projectsServices", $service->getId(), $service); + + $dbForConsole->purgeDocument('projects', $project->getId()); + } + } + + $project = $dbForConsole->getDocument('projects', $projectId); $response->dynamic($project, Response::MODEL_PROJECT); }); @@ -505,7 +536,7 @@ App::patch('/v1/projects/:projectId/oauth2') ->param('secret', '', new text(512), 'Provider secret key. Max length: 512 chars.', true) ->inject('response') ->inject('dbForConsole') - ->action(function ($projectId, $provider, $appId, $secret, $response, $dbForConsole) { + ->action(function ($projectId, $providerKey, $appId, $secret, $response, $dbForConsole) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForConsole */ @@ -515,12 +546,41 @@ App::patch('/v1/projects/:projectId/oauth2') throw new Exception('Project not found', 404); } - $providers = $project->getAttribute('providers', []); - $providers[$provider . 'Appid'] = $appId; - $providers[$provider . 'Secret'] = $secret; + $provider = $dbForConsole->findOne("projectProviders", [ + new Query('key', Query::TYPE_EQUAL, [$providerKey]), + new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]), + ]); - $project = $dbForConsole->updateDocument('projects', $project->getId(), $project->setAttribute('providers', $providers)); + if($provider && !$provider->isEmpty()) { + // Provider exists + $provider->setAttribute("appId", $appId) + ->setAttribute("appSecret", $secret); + + $dbForConsole->updateDocument("projectProviders", $provider->getId(), $provider); + + $dbForConsole->purgeDocument("projects", $project->getId()); + } else { + // Provider does not exist yet + + $teamId = $project->getAttribute("teamId"); + + $provider = new Document([ + '$id' => $dbForConsole->getId(), + '$read' => ['team:' . $teamId], + '$write' => ['team:' . $teamId . '/owner', 'team:' . $teamId . '/developer'], + 'projectId' => $project->getId(), + 'key' => $providerKey, + 'appId' => $appId, + 'appSecret' => $secret + ]); + + $dbForConsole->createDocument("projectProviders", $provider); + + $dbForConsole->purgeDocument("projects", $project->getId()); + } + + $project = $dbForConsole->getDocument('projects', $projectId); $response->dynamic($project, Response::MODEL_PROJECT); }); @@ -674,8 +734,13 @@ App::post('/v1/projects/:projectId/webhooks') $security = ($security === '1' || $security === 'true' || $security === 1 || $security === true); + $teamId = $project->getAttribute("teamId"); + $webhook = new Document([ '$id' => $dbForConsole->getId(), + '$read' => ['team:' . $teamId], + '$write' => ['team:' . $teamId . '/owner', 'team:' . $teamId . '/developer'], + 'projectId' => $project->getId(), 'name' => $name, 'events' => $events, 'url' => $url, @@ -684,9 +749,9 @@ App::post('/v1/projects/:projectId/webhooks') 'httpPass' => $httpPass, ]); - $project = $dbForConsole->updateDocument('projects', $project->getId(), $project - ->setAttribute('webhooks', $webhook, Document::SET_TYPE_APPEND) - ); + $webhook = $dbForConsole->createDocument("projectWebhooks", $webhook); + + $dbForConsole->purgeDocument("projects", $project->getId()); $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($webhook, Response::MODEL_WEBHOOK); @@ -715,7 +780,9 @@ App::get('/v1/projects/:projectId/webhooks') throw new Exception('Project not found', 404); } - $webhooks = $project->getAttribute('webhooks', []); + $webhooks = $dbForConsole->find("projectWebhooks", [ + new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) + ]); $response->dynamic(new Document([ 'webhooks' => $webhooks, @@ -747,9 +814,9 @@ App::get('/v1/projects/:projectId/webhooks/:webhookId') throw new Exception('Project not found', 404); } - $webhook = $project->find('$id', $webhookId, 'webhooks'); + $webhook = $dbForConsole->getDocument('projectWebhooks', $webhookId); - if (empty($webhook) || !$webhook instanceof Document) { + if ($webhook->isEmpty()) { throw new Exception('Webhook not found', 404); } @@ -788,22 +855,23 @@ App::put('/v1/projects/:projectId/webhooks/:webhookId') $security = ($security === '1' || $security === 'true' || $security === 1 || $security === true); - $webhook = $project->find('$id', $webhookId, 'webhooks'); + $webhook = $dbForConsole->getDocument('projectWebhooks', $webhookId); - if (empty($webhook) || !$webhook instanceof Document) { + if ($webhook->isEmpty()) { throw new Exception('Webhook not found', 404); } - $project->findAndReplace('$id', $webhook->getId(), $webhook - ->setAttribute('name', $name) - ->setAttribute('events', $events) - ->setAttribute('url', $url) - ->setAttribute('security', $security) - ->setAttribute('httpUser', $httpUser) - ->setAttribute('httpPass', $httpPass) - , 'webhooks'); + $webhook + ->setAttribute('name', $name) + ->setAttribute('events', $events) + ->setAttribute('url', $url) + ->setAttribute('security', $security) + ->setAttribute('httpUser', $httpUser) + ->setAttribute('httpPass', $httpPass); - $dbForConsole->updateDocument('projects', $project->getId(), $project); + $dbForConsole->updateDocument("projectWebhooks", $webhook->getId(), $webhook); + + $dbForConsole->purgeDocument('projects', $project->getId()); $response->dynamic($webhook, Response::MODEL_WEBHOOK); }); @@ -831,11 +899,15 @@ App::delete('/v1/projects/:projectId/webhooks/:webhookId') throw new Exception('Project not found', 404); } - if (!$project->findAndRemove('$id', $webhookId, 'webhooks')) { + $webhook = $dbForConsole->getDocument("projectWebhooks", $webhookId); + + if($webhook->isEmpty()) { throw new Exception('Webhook not found', 404); } - $dbForConsole->updateDocument('projects', $project->getId(), $project); + $dbForConsole->deleteDocument("projectWebhooks", $webhook->getId()); + + $dbForConsole->purgeDocument('projects', $project->getId()); $response->noContent(); }); @@ -867,18 +939,24 @@ App::post('/v1/projects/:projectId/keys') throw new Exception('Project not found', 404); } + $teamId = $project->getAttribute("teamId"); + $key = new Document([ '$id' => $dbForConsole->getId(), + '$read' => ['team:' . $teamId], + '$write' => ['team:' . $teamId . '/owner', 'team:' . $teamId . '/developer'], + 'projectId' => $project->getId(), 'name' => $name, 'scopes' => $scopes, 'secret' => \bin2hex(\random_bytes(128)), ]); - $project = $dbForConsole->updateDocument('projects', $project->getId(), $project - ->setAttribute('keys', $key, Document::SET_TYPE_APPEND) - ); + $key = $dbForConsole->createDocument("projectKeys", $key); + + $dbForConsole->purgeDocument("projects", $project->getId()); $response->setStatusCode(Response::STATUS_CODE_CREATED); + var_dump(Response::STATUS_CODE_CREATED); $response->dynamic($key, Response::MODEL_KEY); }); @@ -905,7 +983,10 @@ App::get('/v1/projects/:projectId/keys') throw new Exception('Project not found', 404); } - $keys = $project->getAttribute('keys', []); + // TODO: Implement pagination + $keys = $dbForConsole->find("projectKeys", [ + new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]), + ]); $response->dynamic(new Document([ 'keys' => $keys, @@ -928,15 +1009,18 @@ App::get('/v1/projects/:projectId/keys/:keyId') ->inject('response') ->inject('dbForConsole') ->action(function ($projectId, $keyId, $response, $dbForConsole) { + /** @var Appwrite\Utopia\Response $response */ + /** @var Utopia\Database\Database $dbForConsole */ + $project = $dbForConsole->getDocument('projects', $projectId); if ($project->isEmpty()) { throw new Exception('Project not found', 404); } - $key = $project->find('$id', $keyId, 'keys'); + $key = $dbForConsole->getDocument("projectKeys", $keyId); - if (empty($key) || !$key instanceof Document) { + if ($key->isEmpty()) { throw new Exception('Key not found', 404); } @@ -969,18 +1053,18 @@ App::put('/v1/projects/:projectId/keys/:keyId') throw new Exception('Project not found', 404); } - $key = $project->find('$id', $keyId, 'keys'); + $key = $dbForConsole->getDocument("projectKeys", $keyId); - if (empty($key) || !$key instanceof Document) { + if ($key->isEmpty()) { throw new Exception('Key not found', 404); } - $project->findAndReplace('$id', $key->getId(), $key - ->setAttribute('name', $name) - ->setAttribute('scopes', $scopes) - , 'keys'); + $key->setAttribute('name', $name) + ->setAttribute('scopes', $scopes); - $dbForConsole->updateDocument('projects', $project->getId(), $project); + $dbForConsole->updateDocument("projectKeys", $key->getId(), $key); + + $dbForConsole->purgeDocument('projects', $project->getId()); $response->dynamic($key, Response::MODEL_KEY); }); @@ -1008,11 +1092,15 @@ App::delete('/v1/projects/:projectId/keys/:keyId') throw new Exception('Project not found', 404); } - if (!$project->findAndRemove('$id', $keyId, 'keys')) { + $key = $dbForConsole->getDocument("projectKeys", $keyId); + + if($key->isEmpty()) { throw new Exception('Key not found', 404); } - $dbForConsole->updateDocument('projects', $project->getId(), $project); + $dbForConsole->deleteDocument("projectKeys", $key->getId()); + + $dbForConsole->purgeDocument('projects', $project->getId()); $response->noContent(); }); @@ -1050,10 +1138,10 @@ App::post('/v1/projects/:projectId/platforms') $teamId = $project->getAttribute("teamId"); $platform = new Document([ - 'projectId' => $project->getId(), '$id' => $dbForConsole->getId(), '$read' => ['team:' . $teamId], '$write' => ['team:' . $teamId . '/owner', 'team:' . $teamId . '/developer'], + 'projectId' => $project->getId(), 'type' => $type, 'name' => $name, 'key' => $key, @@ -1063,7 +1151,8 @@ App::post('/v1/projects/:projectId/platforms') 'dateUpdated' => \time(), ]); - $dbForConsole->createDocument("projectsPlatforms", $platform); + $platform = $dbForConsole->createDocument("projectsPlatforms", $platform); + $dbForConsole->purgeDocument('projects', $project->getId()); $response->setStatusCode(Response::STATUS_CODE_CREATED); @@ -1087,14 +1176,13 @@ App::get('/v1/projects/:projectId/platforms') /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForConsole */ - // TODO: Implement pagination - $project = $dbForConsole->getDocument('projects', $projectId); if ($project->isEmpty()) { throw new Exception('Project not found', 404); } + // TODO: Implement pagination $platforms = $dbForConsole->find('projectsPlatforms', [ new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) ]); @@ -1131,7 +1219,7 @@ App::get('/v1/projects/:projectId/platforms/:platformId') $platform = $dbForConsole->getDocument("projectsPlatforms", $platformId); - if (empty($platform) || !$platform instanceof Document) { + if ($platform->isEmpty()) { throw new Exception('Platform not found', 404); } @@ -1168,7 +1256,7 @@ App::put('/v1/projects/:projectId/platforms/:platformId') $platform = $dbForConsole->getDocument("projectsPlatforms", $platformId); - if (empty($platform) || !$platform instanceof Document) { + if ($platform->isEmpty()) { throw new Exception('Platform not found', 404); } @@ -1247,9 +1335,12 @@ App::post('/v1/projects/:projectId/domains') throw new Exception('Project not found', 404); } - $document = $project->find('domain', $domain, 'domains'); + $document = $dbForConsole->findOne("projectDomains", [ + new Query('domain', Query::TYPE_EQUAL, [$domain]), + new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]), + ]); - if ($document) { + if ($document && !$document->isEmpty()) { throw new Exception('Domain already exists', 409); } @@ -1259,10 +1350,15 @@ App::post('/v1/projects/:projectId/domains') throw new Exception('Unreachable CNAME target (' . $target->get() . '), please use a domain with a public suffix.', 500); } + $teamId = $project->getAttribute("teamId"); + $domain = new Domain($domain); $domain = new Document([ '$id' => $dbForConsole->getId(), + '$read' => ['team:' . $teamId], + '$write' => ['team:' . $teamId . '/owner', 'team:' . $teamId . '/developer'], + 'projectId' => $project->getId(), 'updated' => \time(), 'domain' => $domain->get(), 'tld' => $domain->getSuffix(), @@ -1271,9 +1367,9 @@ App::post('/v1/projects/:projectId/domains') 'certificateId' => null, ]); - $project = $dbForConsole->updateDocument('projects', $project->getId(), $project - ->setAttribute('domains', $domain, Document::SET_TYPE_APPEND) - ); + $domain = $dbForConsole->createDocument("projectDomains", $domain); + + $dbForConsole->purgeDocument("projects", $project->getId()); $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($domain, Response::MODEL_DOMAIN); @@ -1302,7 +1398,8 @@ App::get('/v1/projects/:projectId/domains') throw new Exception('Project not found', 404); } - $domains = $project->getAttribute('domains', []); + // TODO: Implement pagination + $domains = $dbForConsole->find("projectDomains", []); $response->dynamic(new Document([ 'domains' => $domains, @@ -1334,9 +1431,9 @@ App::get('/v1/projects/:projectId/domains/:domainId') throw new Exception('Project not found', 404); } - $domain = $project->find('$id', $domainId, 'domains'); + $domain = $dbForConsole->getDocument("projectDomains", $domainId); - if (empty($domain) || !$domain instanceof Document) { + if ($domain->isEmpty()) { throw new Exception('Domain not found', 404); } @@ -1367,9 +1464,9 @@ App::patch('/v1/projects/:projectId/domains/:domainId/verification') throw new Exception('Project not found', 404); } - $domain = $project->find('$id', $domainId, 'domains'); + $domain = $dbForConsole->getDocument("projectDomains", $domainId); - if (empty($domain) || !$domain instanceof Document) { + if ($domain->isEmpty()) { throw new Exception('Domain not found', 404); } @@ -1389,11 +1486,10 @@ App::patch('/v1/projects/:projectId/domains/:domainId/verification') throw new Exception('Failed to verify domain', 401); } - $project->findAndReplace('$id', $domain->getId(), $domain - ->setAttribute('verification', true) - , 'domains'); + $domain->setAttribute("verification", true); - $dbForConsole->updateDocument('projects', $project->getId(), $project); + $dbForConsole->updateDocument("projectDomains", $domain->getId(), $domain); + $dbForConsole->purgeDocument("projects", $project->getId()); // Issue a TLS certificate when domain is verified Resque::enqueue('v1-certificates', 'CertificatesV1', [ @@ -1428,13 +1524,15 @@ App::delete('/v1/projects/:projectId/domains/:domainId') throw new Exception('Project not found', 404); } - $domain = $project->find('$id', $domainId, 'domains'); + $domain = $dbForConsole->getDocument("projectDomains", $domainId); - if (!$project->findAndRemove('$id', $domainId, 'domains')) { + if ($domain->isEmpty()) { throw new Exception('Domain not found', 404); } - $dbForConsole->updateDocument('projects', $project->getId(), $project); + $dbForConsole->deleteDocument("projectDomains", $domain->getId()); + + $dbForConsole->purgeDocument("projects", $project->getId()); $deletes ->setParam('type', DELETE_TYPE_CERTIFICATES) diff --git a/app/init.php b/app/init.php index 4da737ebbe..704788b25e 100644 --- a/app/init.php +++ b/app/init.php @@ -30,6 +30,7 @@ use Appwrite\Network\Validator\URL; use Appwrite\OpenSSL\OpenSSL; use Appwrite\Stats\Stats; use Utopia\App; +use Utopia\CLI\Console; use Utopia\View; use Utopia\Config\Config; use Utopia\Locale\Locale; @@ -181,6 +182,7 @@ Database::addFilter('subQueryAttributes', return null; }, function($value, Document $document, Database $database) { + // TODO: Maybe implement pagination? return $database ->find('attributes', [ new Query('collectionId', Query::TYPE_EQUAL, [$document->getId()]) @@ -193,6 +195,7 @@ Database::addFilter('subQueryIndexes', return null; }, function($value, Document $document, Database $database) { + // TODO: Maybe implement pagination? return $database ->find('indexes', [ new Query('collectionId', Query::TYPE_EQUAL, [$document->getId()]) @@ -205,13 +208,94 @@ Database::addFilter('subQueryProjectPlatforms', return null; }, function($value, Document $document, Database $database) { + // TODO: Implement pagination return $database ->find('projectsPlatforms', [ - new Query('collectionId', Query::TYPE_EQUAL, [$document->getId()]) + new Query('projectId', Query::TYPE_EQUAL, [$document->getId()]) ], 100, 0, []); } ); +Database::addFilter('subQueryProjectDomains', + function($value) { + return null; + }, + function($value, Document $document, Database $database) { + // TODO: Implement pagination + return $database + ->find('projectDomains', [ + new Query('projectId', Query::TYPE_EQUAL, [$document->getId()]) + ], 100, 0, []); + } +); + +Database::addFilter('subQueryProjectKeys', + function($value) { + return null; + }, + function($value, Document $document, Database $database) { + // TODO: Implement pagination + return $database + ->find('projectKeys', [ + new Query('projectId', Query::TYPE_EQUAL, [$document->getId()]) + ], 100, 0, []); + } +); + +Database::addFilter('subQueryProjectWebhooks', + function($value) { + return null; + }, + function($value, Document $document, Database $database) { + // TODO: Implement pagination + return $database + ->find('projectWebhooks', [ + new Query('projectId', Query::TYPE_EQUAL, [$document->getId()]) + ], 100, 0, []); + } +); + +Database::addFilter('subQueryProjectServices', + function($value) { + return null; + }, + function($value, Document $document, Database $database) { + // TODO: Implement pagination + $services = $database + ->find('projectsServices', [ + new Query('projectId', Query::TYPE_EQUAL, [$document->getId()]) + ], 100, 0, []); + + $responseJson = []; + foreach($services as $service) { + $responseJson[$service->getAttribute("key")] = $service->getAttribute("status", true); + } + + return $responseJson; + } +); + +Database::addFilter('subQueryProjectProviders', + function($value) { + return null; + }, + function($value, Document $document, Database $database) { + // TODO: Implement pagination + $providers = $database + ->find('projectProviders', [ + new Query('projectId', Query::TYPE_EQUAL, [$document->getId()]) + ], 100, 0, []); + + $responseJson = []; + foreach($providers as $provider) { + $responseJson[$provider->getAttribute("key") . 'Appid'] = $provider->getAttribute("appId"); + $responseJson[$provider->getAttribute("key") . 'Secret'] = $provider->getAttribute("appSecret"); + } + + return $responseJson; + } +); + Database::addFilter('encrypt', function($value) { $key = App::getEnv('_APP_OPENSSL_KEY_V1'); From abfe47787bb2d317884a293963ad641bbda3a9f0 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Mon, 30 Aug 2021 17:16:42 +0200 Subject: [PATCH 104/206] Added TODO --- app/controllers/api/projects.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 15dee15f8d..845dc69b0f 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -23,6 +23,8 @@ use Utopia\Validator\WhiteList; use Utopia\Audit\Audit; use Utopia\Abuse\Adapters\TimeLimit; +// TODO: Meldiron is lazy person from JS who uses "" instead of ''. Yell at him if this comment is still here. We dont want "" in PHP + App::init(function ($project) { /** @var Utopia\Database\Document $project */ From 94e330149f3e9ed1a8865aee99db1bc75a2728e4 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Wed, 4 Aug 2021 17:04:48 -0400 Subject: [PATCH 105/206] Catch limit exception on attributes --- app/controllers/api/database.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 7df3172484..580dc951d4 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -89,6 +89,8 @@ $attributesCallback = function ($collectionId, $attribute, $response, $dbForInte ])); } catch (DuplicateException $th) { throw new Exception('Attribute already exists', 409); + } catch (LimitException $e) { + throw new Exception($e->getMessage(), 400); } $dbForInternal->purgeDocument('collections', $collectionId); From 4f4db30a8a9a34a844f76050aa6603e20dc15b19 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Wed, 4 Aug 2021 17:05:09 -0400 Subject: [PATCH 106/206] Test for attribute limit exception --- .../Database/DatabaseCustomServerTest.php | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tests/e2e/Services/Database/DatabaseCustomServerTest.php b/tests/e2e/Services/Database/DatabaseCustomServerTest.php index e5a75fa685..ca869f79d3 100644 --- a/tests/e2e/Services/Database/DatabaseCustomServerTest.php +++ b/tests/e2e/Services/Database/DatabaseCustomServerTest.php @@ -307,6 +307,54 @@ class DatabaseCustomServerTest extends Scope $this->assertEquals($response['headers']['status-code'], 404); } + /** + * @depends testDeleteCollection + */ + public function testAttributeCountLimit() + { + $collection = $this->client->call(Client::METHOD_POST, '/database/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'name' => 'attributeCountLimit', + 'read' => ['role:all'], + 'write' => ['role:all'], + 'permission' => 'document', + ]); + + $collectionId = $collection['body']['$id']; + + // load the collection up to the limit + for ($i=0; $i < 1012; $i++) { + $attribute = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/attributes/integer', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'attributeId' => "attribute{$i}", + 'required' => false, + ]); + + $this->assertEquals(201, $attribute['headers']['status-code']); + } + + sleep(20); + + $tooMany = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/attributes/integer', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'attributeId' => "tooMany", + 'required' => false, + ]); + + $this->assertEquals(400, $tooMany['headers']['status-code']); + $this->assertEquals('Column limit reached. Cannot create new attribute.', $tooMany['body']['message']); + } + + public function testIndexLimitException() { $collection = $this->client->call(Client::METHOD_POST, '/database/collections', array_merge([ From 42ca474731baade1528d4dd600b2c37bde5f9da6 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Tue, 24 Aug 2021 20:01:11 -0400 Subject: [PATCH 107/206] Search for attributes from internal table --- app/controllers/api/database.php | 11 +++++++++++ .../Services/Database/DatabaseCustomServerTest.php | 7 ++----- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 580dc951d4..609ffc7aab 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -56,6 +56,17 @@ $attributesCallback = function ($collectionId, $attribute, $response, $dbForInte throw new Exception('Collection not found', 404); } + $count = $dbForInternal->count('attributes', [ + new Query('collectionId', Query::TYPE_EQUAL, [$collectionId]) + ], 1012); + + // 1017 limit minus default attributes and one buffer for virtual columns + $limit = 1017 - MariaDB::getNumberOfDefaultAttributes() - 1; + + if ($count >= $limit) { + throw new Exception('Attribute limit exceeded', 400); + } + // TODO@kodumbeats how to depend on $size for Text validator length // Ensure attribute default is within required size if ($size > 0 && !\is_null($default)) { diff --git a/tests/e2e/Services/Database/DatabaseCustomServerTest.php b/tests/e2e/Services/Database/DatabaseCustomServerTest.php index ca869f79d3..3884ca7a3d 100644 --- a/tests/e2e/Services/Database/DatabaseCustomServerTest.php +++ b/tests/e2e/Services/Database/DatabaseCustomServerTest.php @@ -307,9 +307,6 @@ class DatabaseCustomServerTest extends Scope $this->assertEquals($response['headers']['status-code'], 404); } - /** - * @depends testDeleteCollection - */ public function testAttributeCountLimit() { $collection = $this->client->call(Client::METHOD_POST, '/database/collections', array_merge([ @@ -317,6 +314,7 @@ class DatabaseCustomServerTest extends Scope 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] ]), [ + 'collectionId' => 'unique()', 'name' => 'attributeCountLimit', 'read' => ['role:all'], 'write' => ['role:all'], @@ -351,10 +349,9 @@ class DatabaseCustomServerTest extends Scope ]); $this->assertEquals(400, $tooMany['headers']['status-code']); - $this->assertEquals('Column limit reached. Cannot create new attribute.', $tooMany['body']['message']); + $this->assertEquals('Attribute limit exceeded', $tooMany['body']['message']); } - public function testIndexLimitException() { $collection = $this->client->call(Client::METHOD_POST, '/database/collections', array_merge([ From 8f066259801f8c36ff8471e0b49c93e59a5a4490 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Tue, 31 Aug 2021 11:53:20 -0400 Subject: [PATCH 108/206] Reduce sleep time in test --- tests/e2e/Services/Database/DatabaseCustomServerTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/Services/Database/DatabaseCustomServerTest.php b/tests/e2e/Services/Database/DatabaseCustomServerTest.php index 3884ca7a3d..97b12aa279 100644 --- a/tests/e2e/Services/Database/DatabaseCustomServerTest.php +++ b/tests/e2e/Services/Database/DatabaseCustomServerTest.php @@ -337,7 +337,7 @@ class DatabaseCustomServerTest extends Scope $this->assertEquals(201, $attribute['headers']['status-code']); } - sleep(20); + sleep(5); $tooMany = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/attributes/integer', array_merge([ 'content-type' => 'application/json', From efa0b8629926520ba798a2380b8a20ecdf6d5faf Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Tue, 31 Aug 2021 12:35:03 -0400 Subject: [PATCH 109/206] Add assertions for row width limit --- .../Database/DatabaseCustomServerTest.php | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/tests/e2e/Services/Database/DatabaseCustomServerTest.php b/tests/e2e/Services/Database/DatabaseCustomServerTest.php index 97b12aa279..2be62f8fa8 100644 --- a/tests/e2e/Services/Database/DatabaseCustomServerTest.php +++ b/tests/e2e/Services/Database/DatabaseCustomServerTest.php @@ -352,6 +352,56 @@ class DatabaseCustomServerTest extends Scope $this->assertEquals('Attribute limit exceeded', $tooMany['body']['message']); } + public function testAttributeRowWidthLimit() + { + $collection = $this->client->call(Client::METHOD_POST, '/database/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => 'attributeRowWidthLimit', + 'name' => 'attributeRowWidthLimit', + 'read' => ['role:all'], + 'write' => ['role:all'], + 'permission' => 'document', + ]); + + $this->assertEquals($collection['headers']['status-code'], 201); + $this->assertEquals($collection['body']['name'], 'attributeRowWidthLimit'); + + $collectionId = $collection['body']['$id']; + + // Add wide string attributes to approach row width limit + for ($i=0; $i < 15; $i++) { + $attribute = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/attributes/string', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'attributeId' => "attribute{$i}", + 'size' => 1024, + 'required' => true, + ]); + + $this->assertEquals($attribute['headers']['status-code'], 201); + } + + sleep(5); + + $tooWide = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/attributes/string', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'attributeId' => 'tooWide', + 'size' => 1024, + 'required' => true, + ]); + + // $this->assertEquals(400, $tooWide['headers']['status-code']); + // $this->assertEquals('Attribute limit exceeded', $tooWide['body']['message']); + } + public function testIndexLimitException() { $collection = $this->client->call(Client::METHOD_POST, '/database/collections', array_merge([ From ad81e414d99787b4516bdad56f5c3e7f59748506 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 1 Sep 2021 00:34:01 +0530 Subject: [PATCH 110/206] feat(response): add metric and metric list to response models --- src/Appwrite/Utopia/Response.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index 0c4c6bf0aa..7e012bc6c0 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -33,6 +33,7 @@ use Appwrite\Utopia\Response\Model\Team; use Appwrite\Utopia\Response\Model\Locale; use Appwrite\Utopia\Response\Model\Log; use Appwrite\Utopia\Response\Model\Membership; +use Appwrite\Utopia\Response\Model\Metric; use Appwrite\Utopia\Response\Model\Permissions; use Appwrite\Utopia\Response\Model\Phone; use Appwrite\Utopia\Response\Model\Platform; @@ -194,6 +195,7 @@ class Response extends SwooleResponse ->setModel(new BaseList('Languages List', self::MODEL_LANGUAGE_LIST, 'languages', self::MODEL_LANGUAGE)) ->setModel(new BaseList('Currencies List', self::MODEL_CURRENCY_LIST, 'currencies', self::MODEL_CURRENCY)) ->setModel(new BaseList('Phones List', self::MODEL_PHONE_LIST, 'phones', self::MODEL_PHONE)) + ->setModel(new BaseList('Metric List', self::MODEL_METRIC_LIST, 'metric', self::MODEL_METRIC)) // Entities ->setModel(new Collection()) ->setModel(new Attribute()) @@ -229,6 +231,7 @@ class Response extends SwooleResponse ->setModel(new UsageBuckets()) ->setModel(new UsageFunctions()) ->setModel(new UsageProject()) + ->setModel(new Metric()) // Verification // Recovery // Tests (keep last) From ac0ff8ea0fcfc60375b61001a72ed6a4dded9bb0 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 1 Sep 2021 02:25:13 +0530 Subject: [PATCH 111/206] feat(response): use recursive function for swagger/openapi spec generation --- app/controllers/api/database.php | 2 +- app/controllers/api/storage.php | 2 +- app/controllers/api/users.php | 3 +- .../Specification/Format/OpenAPI3.php | 28 ++++++++++++--- .../Specification/Format/Swagger2.php | 34 ++++++++++++++----- src/Appwrite/Utopia/Response.php | 5 ++- 6 files changed, 55 insertions(+), 19 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index e2af571bf7..be0a346047 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -344,7 +344,7 @@ App::get('/v1/database/:collectionId/usage') ->label('scope', 'collections.read') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'database') - ->label('sdk.method', 'getUsage') + ->label('sdk.method', 'getCollectionUsage') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_USAGE_COLLECTION) diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index ee6210d25d..69ed5ba536 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -723,7 +723,7 @@ App::get('/v1/storage/:bucketId/usage') ->label('scope', 'files.read') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) ->label('sdk.namespace', 'storage') - ->label('sdk.method', 'getUsage') + ->label('sdk.method', 'getBucketUsage') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_USAGE_BUCKETS) diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 7fcdffcc8b..48c67daa82 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -18,6 +18,7 @@ use Utopia\Database\Validator\UID; use DeviceDetector\DeviceDetector; use Appwrite\Database\Validator\CustomId; use Appwrite\Utopia\Response\Model; +use Utopia\Config\Config; use Utopia\Database\Database; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; @@ -621,7 +622,7 @@ App::get('/v1/users/usage') ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) ->label('sdk.response.model', Response::MODEL_USAGE_USERS) ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true) - ->param('provider', '', new WhiteList(['email', 'anonymous', 'oauth2-google', 'oauth2-apple'], true), 'Provider Name.', true) + ->param('provider', '', new WhiteList(\array_merge(['email', 'anonymous'], \array_map(function($value) { return "oauth-".$value; }, \array_keys(Config::getParam('providers', [])))), true), 'Provider Name.', true) ->inject('response') ->inject('dbForInternal') ->inject('register') diff --git a/src/Appwrite/Specification/Format/OpenAPI3.php b/src/Appwrite/Specification/Format/OpenAPI3.php index 353369eb65..32e2347c0e 100644 --- a/src/Appwrite/Specification/Format/OpenAPI3.php +++ b/src/Appwrite/Specification/Format/OpenAPI3.php @@ -21,6 +21,28 @@ class OpenAPI3 extends Format return 'Open API 3'; } + /** + * Get Used Models + * + * Recursively get all used models + * + * @param object $model + * @param array $models + * + * @return void + */ + protected function getUsedModels($model, array &$usedModels) + { + if (is_string($model) && !in_array($model, ['string', 'integer', 'boolean', 'json', 'float'])) { + $usedModels[] = $model; + return; + } + if (!is_object($model)) return; + foreach ($model->getRules() as $rule) { + $this->getUsedModels($rule['type'], $usedModels); + } + } + /** * Parse * @@ -352,11 +374,7 @@ class OpenAPI3 extends Format $output['paths'][$url][\strtolower($route->getMethod())] = $temp; } foreach ($this->models as $model) { - foreach ($model->getRules() as $rule) { - if (!in_array($rule['type'], ['string', 'integer', 'boolean', 'json', 'float'])) { - $usedModels[] = $rule['type']; - } - } + $this->getUsedModels($model, $usedModels); } foreach ($this->models as $model) { if (!in_array($model->getType(), $usedModels) && $model->getType() !== 'error') { diff --git a/src/Appwrite/Specification/Format/Swagger2.php b/src/Appwrite/Specification/Format/Swagger2.php index d357283a02..75cac416d7 100644 --- a/src/Appwrite/Specification/Format/Swagger2.php +++ b/src/Appwrite/Specification/Format/Swagger2.php @@ -21,6 +21,28 @@ class Swagger2 extends Format return 'Swagger 2'; } + /** + * Get Used Models + * + * Recursively get all used models + * + * @param object $model + * @param array $models + * + * @return void + */ + protected function getUsedModels($model, array &$usedModels) + { + if (is_string($model) && !in_array($model, ['string', 'integer', 'boolean', 'json', 'float'])) { + $usedModels[] = $model; + return; + } + if (!is_object($model)) return; + foreach ($model->getRules() as $rule) { + $this->getUsedModels($rule['type'], $usedModels); + } + } + /** * Parse * @@ -354,15 +376,11 @@ class Swagger2 extends Format $output['paths'][$url][\strtolower($route->getMethod())] = $temp; } foreach ($this->models as $model) { - foreach ($model->getRules() as $rule) { - if ( - in_array($model->getType(), $usedModels) - && !in_array($rule['type'], ['string', 'integer', 'boolean', 'json', 'float']) - ) { - $usedModels[] = $rule['type']; - } - } + $this->getUsedModels($model, $usedModels); } + + // var_dump($usedModels); + foreach ($this->models as $model) { if (!in_array($model->getType(), $usedModels)) { continue; diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index 7e012bc6c0..329dc86432 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -51,7 +51,6 @@ use Appwrite\Utopia\Response\Model\UsageFunctions; use Appwrite\Utopia\Response\Model\UsageProject; use Appwrite\Utopia\Response\Model\UsageStorage; use Appwrite\Utopia\Response\Model\UsageUsers; -use Appwrite\Utopia\Response\Model\UsersUsage; use stdClass; /** @@ -195,7 +194,7 @@ class Response extends SwooleResponse ->setModel(new BaseList('Languages List', self::MODEL_LANGUAGE_LIST, 'languages', self::MODEL_LANGUAGE)) ->setModel(new BaseList('Currencies List', self::MODEL_CURRENCY_LIST, 'currencies', self::MODEL_CURRENCY)) ->setModel(new BaseList('Phones List', self::MODEL_PHONE_LIST, 'phones', self::MODEL_PHONE)) - ->setModel(new BaseList('Metric List', self::MODEL_METRIC_LIST, 'metric', self::MODEL_METRIC)) + ->setModel(new BaseList('Metric List', self::MODEL_METRIC_LIST, 'metrics', self::MODEL_METRIC, true, false)) // Entities ->setModel(new Collection()) ->setModel(new Attribute()) @@ -224,6 +223,7 @@ class Response extends SwooleResponse ->setModel(new Language()) ->setModel(new Currency()) ->setModel(new Phone()) + ->setModel(new Metric()) ->setModel(new UsageDatabase()) ->setModel(new UsageCollection()) ->setModel(new UsageUsers()) @@ -231,7 +231,6 @@ class Response extends SwooleResponse ->setModel(new UsageBuckets()) ->setModel(new UsageFunctions()) ->setModel(new UsageProject()) - ->setModel(new Metric()) // Verification // Recovery // Tests (keep last) From e2d1b2df58a74b2a3e4c3e51d91ea9ea991ba831 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Tue, 31 Aug 2021 21:13:36 -0400 Subject: [PATCH 112/206] Fetch new checkAttribute database method --- composer.json | 2 +- composer.lock | 46 +++++++++++++++++++++++----------------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/composer.json b/composer.json index d48d2378aa..9bd152fb92 100644 --- a/composer.json +++ b/composer.json @@ -45,7 +45,7 @@ "utopia-php/cache": "0.4.*", "utopia-php/cli": "0.11.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "dev-feat-adjusted-query-validator as 0.10.0", + "utopia-php/database": "dev-feat-check-attribute-method as 0.10.0", "utopia-php/locale": "0.4.*", "utopia-php/registry": "0.5.*", "utopia-php/preloader": "0.2.*", diff --git a/composer.lock b/composer.lock index a5f2a3e062..829e348201 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "175f077512c575216c4c88f1d33c6d00", + "content-hash": "9a7739aabd503572b8010be91192b144", "packages": [ { "name": "adhocore/jwt", @@ -355,16 +355,16 @@ }, { "name": "composer/package-versions-deprecated", - "version": "1.11.99.2", + "version": "1.11.99.3", "source": { "type": "git", "url": "https://github.com/composer/package-versions-deprecated.git", - "reference": "c6522afe5540d5fc46675043d3ed5a45a740b27c" + "reference": "fff576ac850c045158a250e7e27666e146e78d18" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/composer/package-versions-deprecated/zipball/c6522afe5540d5fc46675043d3ed5a45a740b27c", - "reference": "c6522afe5540d5fc46675043d3ed5a45a740b27c", + "url": "https://api.github.com/repos/composer/package-versions-deprecated/zipball/fff576ac850c045158a250e7e27666e146e78d18", + "reference": "fff576ac850c045158a250e7e27666e146e78d18", "shasum": "" }, "require": { @@ -408,7 +408,7 @@ "description": "Composer plugin that provides efficient querying for installed package versions (no runtime IO)", "support": { "issues": "https://github.com/composer/package-versions-deprecated/issues", - "source": "https://github.com/composer/package-versions-deprecated/tree/1.11.99.2" + "source": "https://github.com/composer/package-versions-deprecated/tree/1.11.99.3" }, "funding": [ { @@ -424,7 +424,7 @@ "type": "tidelift" } ], - "time": "2021-05-24T07:46:03+00:00" + "time": "2021-08-17T13:49:14+00:00" }, { "name": "dragonmantank/cron-expression", @@ -1984,11 +1984,11 @@ }, { "name": "utopia-php/database", - "version": "dev-feat-adjusted-query-validator", + "version": "dev-feat-check-attribute-method", "source": { "type": "git", "url": "https://github.com/utopia-php/database", - "reference": "cb73391371f70ddb54bc0000064b15c5f173cb7a" + "reference": "2d1f48345f1284876f6847599c66eee145b7233e" }, "require": { "ext-mongodb": "*", @@ -2037,7 +2037,7 @@ "upf", "utopia" ], - "time": "2021-08-23T14:18:47+00:00" + "time": "2021-09-01T00:50:12+00:00" }, { "name": "utopia-php/domains", @@ -5313,16 +5313,16 @@ }, { "name": "symfony/console", - "version": "v5.3.6", + "version": "v5.3.7", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "51b71afd6d2dc8f5063199357b9880cea8d8bfe2" + "reference": "8b1008344647462ae6ec57559da166c2bfa5e16a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/51b71afd6d2dc8f5063199357b9880cea8d8bfe2", - "reference": "51b71afd6d2dc8f5063199357b9880cea8d8bfe2", + "url": "https://api.github.com/repos/symfony/console/zipball/8b1008344647462ae6ec57559da166c2bfa5e16a", + "reference": "8b1008344647462ae6ec57559da166c2bfa5e16a", "shasum": "" }, "require": { @@ -5392,7 +5392,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v5.3.6" + "source": "https://github.com/symfony/console/tree/v5.3.7" }, "funding": [ { @@ -5408,7 +5408,7 @@ "type": "tidelift" } ], - "time": "2021-07-27T19:10:22+00:00" + "time": "2021-08-25T20:02:16+00:00" }, { "name": "symfony/deprecation-contracts", @@ -5882,16 +5882,16 @@ }, { "name": "symfony/string", - "version": "v5.3.3", + "version": "v5.3.7", "source": { "type": "git", "url": "https://github.com/symfony/string.git", - "reference": "bd53358e3eccec6a670b5f33ab680d8dbe1d4ae1" + "reference": "8d224396e28d30f81969f083a58763b8b9ceb0a5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/string/zipball/bd53358e3eccec6a670b5f33ab680d8dbe1d4ae1", - "reference": "bd53358e3eccec6a670b5f33ab680d8dbe1d4ae1", + "url": "https://api.github.com/repos/symfony/string/zipball/8d224396e28d30f81969f083a58763b8b9ceb0a5", + "reference": "8d224396e28d30f81969f083a58763b8b9ceb0a5", "shasum": "" }, "require": { @@ -5945,7 +5945,7 @@ "utf8" ], "support": { - "source": "https://github.com/symfony/string/tree/v5.3.3" + "source": "https://github.com/symfony/string/tree/v5.3.7" }, "funding": [ { @@ -5961,7 +5961,7 @@ "type": "tidelift" } ], - "time": "2021-06-27T11:44:38+00:00" + "time": "2021-08-26T08:00:08+00:00" }, { "name": "theseer/tokenizer", @@ -6251,7 +6251,7 @@ "aliases": [ { "package": "utopia-php/database", - "version": "dev-feat-adjusted-query-validator", + "version": "dev-feat-check-attribute-method", "alias": "0.10.0", "alias_normalized": "0.10.0.0" } From ae58a600659031a7d3669fa4dce824a83e06ce57 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Tue, 31 Aug 2021 21:13:58 -0400 Subject: [PATCH 113/206] Use checkAttribute to catch index limits --- app/controllers/api/database.php | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 609ffc7aab..7cda576626 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -56,17 +56,6 @@ $attributesCallback = function ($collectionId, $attribute, $response, $dbForInte throw new Exception('Collection not found', 404); } - $count = $dbForInternal->count('attributes', [ - new Query('collectionId', Query::TYPE_EQUAL, [$collectionId]) - ], 1012); - - // 1017 limit minus default attributes and one buffer for virtual columns - $limit = 1017 - MariaDB::getNumberOfDefaultAttributes() - 1; - - if ($count >= $limit) { - throw new Exception('Attribute limit exceeded', 400); - } - // TODO@kodumbeats how to depend on $size for Text validator length // Ensure attribute default is within required size if ($size > 0 && !\is_null($default)) { @@ -82,8 +71,7 @@ $attributesCallback = function ($collectionId, $attribute, $response, $dbForInte } } - try { - $attribute = $dbForInternal->createDocument('attributes', new Document([ + $attribute = new Document([ '$id' => $collectionId.'_'.$attributeId, 'key' => $attributeId, 'collectionId' => $collectionId, @@ -96,8 +84,16 @@ $attributesCallback = function ($collectionId, $attribute, $response, $dbForInte 'array' => $array, 'format' => $format, 'formatOptions' => $formatOptions, - 'filters' => $filters, - ])); + ]); + + try { + $dbForInternal->checkAttribute($collection, $attribute); + } catch (LimitException $exception) { + throw new Exception('Attribute limit exceeded', 400); + } + + try { + $attribute = $dbForInternal->createDocument('attributes', $attribute); } catch (DuplicateException $th) { throw new Exception('Attribute already exists', 409); } catch (LimitException $e) { From 577ab9c3ad27bed118f5ce4b90b48f03472d3119 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Wed, 1 Sep 2021 09:03:22 +0200 Subject: [PATCH 114/206] Fixed permissions, improved database filter docs, added 404 platform update test --- app/controllers/api/projects.php | 37 ++++++------------- app/init.php | 2 + .../Projects/ProjectsConsoleClientTest.php | 12 ++++++ 3 files changed, 26 insertions(+), 25 deletions(-) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 845dc69b0f..8b6a59df46 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -495,12 +495,10 @@ App::patch('/v1/projects/:projectId/service') // TODO: Implement delete method. Do we delete for "true" or for "false"? Same for auth!!!! if($service == false || $service->isEmpty()) { - $teamId = $project->getAttribute("teamId"); - $service = new Document([ '$id' => $dbForConsole->getId(), - '$read' => ['team:' . $teamId], - '$write' => ['team:' . $teamId . '/owner', 'team:' . $teamId . '/developer'], + '$read' => ['role:all'], + '$write' => ['role:all'], 'projectId' => $project->getId(), 'key' => $serviceKey, 'status' => $status, @@ -565,12 +563,10 @@ App::patch('/v1/projects/:projectId/oauth2') } else { // Provider does not exist yet - $teamId = $project->getAttribute("teamId"); - $provider = new Document([ '$id' => $dbForConsole->getId(), - '$read' => ['team:' . $teamId], - '$write' => ['team:' . $teamId . '/owner', 'team:' . $teamId . '/developer'], + '$read' => ['role:all'], + '$write' => ['role:all'], 'projectId' => $project->getId(), 'key' => $providerKey, 'appId' => $appId, @@ -736,12 +732,10 @@ App::post('/v1/projects/:projectId/webhooks') $security = ($security === '1' || $security === 'true' || $security === 1 || $security === true); - $teamId = $project->getAttribute("teamId"); - $webhook = new Document([ '$id' => $dbForConsole->getId(), - '$read' => ['team:' . $teamId], - '$write' => ['team:' . $teamId . '/owner', 'team:' . $teamId . '/developer'], + '$read' => ['role:all'], + '$write' => ['role:all'], 'projectId' => $project->getId(), 'name' => $name, 'events' => $events, @@ -941,12 +935,10 @@ App::post('/v1/projects/:projectId/keys') throw new Exception('Project not found', 404); } - $teamId = $project->getAttribute("teamId"); - $key = new Document([ '$id' => $dbForConsole->getId(), - '$read' => ['team:' . $teamId], - '$write' => ['team:' . $teamId . '/owner', 'team:' . $teamId . '/developer'], + '$read' => ['role:all'], + '$write' => ['role:all'], 'projectId' => $project->getId(), 'name' => $name, 'scopes' => $scopes, @@ -958,7 +950,6 @@ App::post('/v1/projects/:projectId/keys') $dbForConsole->purgeDocument("projects", $project->getId()); $response->setStatusCode(Response::STATUS_CODE_CREATED); - var_dump(Response::STATUS_CODE_CREATED); $response->dynamic($key, Response::MODEL_KEY); }); @@ -1137,12 +1128,10 @@ App::post('/v1/projects/:projectId/platforms') throw new Exception('Project not found', 404); } - $teamId = $project->getAttribute("teamId"); - $platform = new Document([ '$id' => $dbForConsole->getId(), - '$read' => ['team:' . $teamId], - '$write' => ['team:' . $teamId . '/owner', 'team:' . $teamId . '/developer'], + '$read' => ['role:all'], + '$write' => ['role:all'], 'projectId' => $project->getId(), 'type' => $type, 'name' => $name, @@ -1352,14 +1341,12 @@ App::post('/v1/projects/:projectId/domains') throw new Exception('Unreachable CNAME target (' . $target->get() . '), please use a domain with a public suffix.', 500); } - $teamId = $project->getAttribute("teamId"); - $domain = new Domain($domain); $domain = new Document([ '$id' => $dbForConsole->getId(), - '$read' => ['team:' . $teamId], - '$write' => ['team:' . $teamId . '/owner', 'team:' . $teamId . '/developer'], + '$read' => ['role:all'], + '$write' => ['role:all'], 'projectId' => $project->getId(), 'updated' => \time(), 'domain' => $domain->get(), diff --git a/app/init.php b/app/init.php index 704788b25e..7c44ba12e6 100644 --- a/app/init.php +++ b/app/init.php @@ -142,6 +142,8 @@ if(!empty($user) || !empty($pass)) { /** * DB Filters + * + * Make sure the value of an attribute that uses sub-query filters is set to 'null' otherwise the filters might not work properly */ DatabaseOld::addFilter('json', function($value) { diff --git a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php index 4d83efb416..f090158cd0 100644 --- a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php +++ b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php @@ -7,6 +7,7 @@ use Tests\E2E\Scopes\ProjectConsole; use Tests\E2E\Scopes\SideClient; use Tests\E2E\Services\Projects\ProjectsBase; use Tests\E2E\Client; +use function array_merge; class ProjectsConsoleClientTest extends Scope { @@ -1430,6 +1431,17 @@ class ProjectsConsoleClientTest extends Scope /** * Test for FAILURE */ + $response = $this->client->call(Client::METHOD_PUT, '/projects/'.$id.'/platforms/error', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'name' => 'Flutter App (Android) 2', + 'key' => 'com.example.android2', + 'store' => '', + 'hostname' => '', + ]); + + $this->assertEquals(404, $response['headers']['status-code']); return $data; } From 182630ed203e2174e2138bcdc3b397fff7799114 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Wed, 1 Sep 2021 09:52:43 +0200 Subject: [PATCH 115/206] Rollbacked auth attribute, set subquery limits to 5000, Replaced " with ' --- app/controllers/api/projects.php | 100 ++++++++++++++----------------- app/init.php | 30 ++++------ 2 files changed, 57 insertions(+), 73 deletions(-) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 8b6a59df46..cdc43d7b1a 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -23,8 +23,6 @@ use Utopia\Validator\WhiteList; use Utopia\Audit\Audit; use Utopia\Abuse\Adapters\TimeLimit; -// TODO: Meldiron is lazy person from JS who uses "" instead of ''. Yell at him if this comment is still here. We dont want "" in PHP - App::init(function ($project) { /** @var Utopia\Database\Document $project */ @@ -101,11 +99,9 @@ App::post('/v1/projects') 'webhooks' => null, 'keys' => null, 'domains' => null, - 'auths' => null + 'auths' => $auths ])); - // TODO: Implement write of auths from $auths - $collections = Config::getParam('collections2', []); /** @var array $collections */ $dbForInternal->setNamespace('project_' . $project->getId() . '_internal'); @@ -116,7 +112,7 @@ App::post('/v1/projects') $audit = new Audit($dbForInternal); $audit->setup(); - $adapter = new TimeLimit("", 0, 1, $dbForInternal); + $adapter = new TimeLimit('', 0, 1, $dbForInternal); $adapter->setup(); foreach ($collections as $key => $collection) { @@ -488,12 +484,11 @@ App::patch('/v1/projects/:projectId/service') throw new Exception('Project not found', 404); } - $service = $dbForConsole->findOne("projectsServices", [ + $service = $dbForConsole->findOne('projectsServices', [ new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]), new Query('key', Query::TYPE_EQUAL, [$serviceKey]), ]); - // TODO: Implement delete method. Do we delete for "true" or for "false"? Same for auth!!!! if($service == false || $service->isEmpty()) { $service = new Document([ '$id' => $dbForConsole->getId(), @@ -504,13 +499,13 @@ App::patch('/v1/projects/:projectId/service') 'status' => $status, ]); - $dbForConsole->createDocument("projectsServices", $service); + $dbForConsole->createDocument('projectsServices', $service); $dbForConsole->purgeDocument('projects', $project->getId()); } else { - if($service->getAttribute("status") != $status) { - $service->setAttribute("status", $status); - $dbForConsole->updateDocument("projectsServices", $service->getId(), $service); + if($service->getAttribute('status') != $status) { + $service->setAttribute('status', $status); + $dbForConsole->updateDocument('projectsServices', $service->getId(), $service); $dbForConsole->purgeDocument('projects', $project->getId()); } @@ -546,7 +541,7 @@ App::patch('/v1/projects/:projectId/oauth2') throw new Exception('Project not found', 404); } - $provider = $dbForConsole->findOne("projectProviders", [ + $provider = $dbForConsole->findOne('projectProviders', [ new Query('key', Query::TYPE_EQUAL, [$providerKey]), new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]), ]); @@ -554,12 +549,12 @@ App::patch('/v1/projects/:projectId/oauth2') if($provider && !$provider->isEmpty()) { // Provider exists - $provider->setAttribute("appId", $appId) - ->setAttribute("appSecret", $secret); + $provider->setAttribute('appId', $appId) + ->setAttribute('appSecret', $secret); - $dbForConsole->updateDocument("projectProviders", $provider->getId(), $provider); + $dbForConsole->updateDocument('projectProviders', $provider->getId(), $provider); - $dbForConsole->purgeDocument("projects", $project->getId()); + $dbForConsole->purgeDocument('projects', $project->getId()); } else { // Provider does not exist yet @@ -573,9 +568,9 @@ App::patch('/v1/projects/:projectId/oauth2') 'appSecret' => $secret ]); - $dbForConsole->createDocument("projectProviders", $provider); + $dbForConsole->createDocument('projectProviders', $provider); - $dbForConsole->purgeDocument("projects", $project->getId()); + $dbForConsole->purgeDocument('projects', $project->getId()); } $project = $dbForConsole->getDocument('projects', $projectId); @@ -745,9 +740,9 @@ App::post('/v1/projects/:projectId/webhooks') 'httpPass' => $httpPass, ]); - $webhook = $dbForConsole->createDocument("projectWebhooks", $webhook); + $webhook = $dbForConsole->createDocument('projectWebhooks', $webhook); - $dbForConsole->purgeDocument("projects", $project->getId()); + $dbForConsole->purgeDocument('projects', $project->getId()); $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($webhook, Response::MODEL_WEBHOOK); @@ -776,7 +771,7 @@ App::get('/v1/projects/:projectId/webhooks') throw new Exception('Project not found', 404); } - $webhooks = $dbForConsole->find("projectWebhooks", [ + $webhooks = $dbForConsole->find('projectWebhooks', [ new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) ]); @@ -865,7 +860,7 @@ App::put('/v1/projects/:projectId/webhooks/:webhookId') ->setAttribute('httpUser', $httpUser) ->setAttribute('httpPass', $httpPass); - $dbForConsole->updateDocument("projectWebhooks", $webhook->getId(), $webhook); + $dbForConsole->updateDocument('projectWebhooks', $webhook->getId(), $webhook); $dbForConsole->purgeDocument('projects', $project->getId()); @@ -895,13 +890,13 @@ App::delete('/v1/projects/:projectId/webhooks/:webhookId') throw new Exception('Project not found', 404); } - $webhook = $dbForConsole->getDocument("projectWebhooks", $webhookId); + $webhook = $dbForConsole->getDocument('projectWebhooks', $webhookId); if($webhook->isEmpty()) { throw new Exception('Webhook not found', 404); } - $dbForConsole->deleteDocument("projectWebhooks", $webhook->getId()); + $dbForConsole->deleteDocument('projectWebhooks', $webhook->getId()); $dbForConsole->purgeDocument('projects', $project->getId()); @@ -945,9 +940,9 @@ App::post('/v1/projects/:projectId/keys') 'secret' => \bin2hex(\random_bytes(128)), ]); - $key = $dbForConsole->createDocument("projectKeys", $key); + $key = $dbForConsole->createDocument('projectKeys', $key); - $dbForConsole->purgeDocument("projects", $project->getId()); + $dbForConsole->purgeDocument('projects', $project->getId()); $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($key, Response::MODEL_KEY); @@ -976,10 +971,9 @@ App::get('/v1/projects/:projectId/keys') throw new Exception('Project not found', 404); } - // TODO: Implement pagination - $keys = $dbForConsole->find("projectKeys", [ + $keys = $dbForConsole->find('projectKeys', [ new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]), - ]); + ], 5000); $response->dynamic(new Document([ 'keys' => $keys, @@ -1011,7 +1005,7 @@ App::get('/v1/projects/:projectId/keys/:keyId') throw new Exception('Project not found', 404); } - $key = $dbForConsole->getDocument("projectKeys", $keyId); + $key = $dbForConsole->getDocument('projectKeys', $keyId); if ($key->isEmpty()) { throw new Exception('Key not found', 404); @@ -1046,7 +1040,7 @@ App::put('/v1/projects/:projectId/keys/:keyId') throw new Exception('Project not found', 404); } - $key = $dbForConsole->getDocument("projectKeys", $keyId); + $key = $dbForConsole->getDocument('projectKeys', $keyId); if ($key->isEmpty()) { throw new Exception('Key not found', 404); @@ -1055,7 +1049,7 @@ App::put('/v1/projects/:projectId/keys/:keyId') $key->setAttribute('name', $name) ->setAttribute('scopes', $scopes); - $dbForConsole->updateDocument("projectKeys", $key->getId(), $key); + $dbForConsole->updateDocument('projectKeys', $key->getId(), $key); $dbForConsole->purgeDocument('projects', $project->getId()); @@ -1085,13 +1079,13 @@ App::delete('/v1/projects/:projectId/keys/:keyId') throw new Exception('Project not found', 404); } - $key = $dbForConsole->getDocument("projectKeys", $keyId); + $key = $dbForConsole->getDocument('projectKeys', $keyId); if($key->isEmpty()) { throw new Exception('Key not found', 404); } - $dbForConsole->deleteDocument("projectKeys", $key->getId()); + $dbForConsole->deleteDocument('projectKeys', $key->getId()); $dbForConsole->purgeDocument('projects', $project->getId()); @@ -1142,7 +1136,7 @@ App::post('/v1/projects/:projectId/platforms') 'dateUpdated' => \time(), ]); - $platform = $dbForConsole->createDocument("projectsPlatforms", $platform); + $platform = $dbForConsole->createDocument('projectsPlatforms', $platform); $dbForConsole->purgeDocument('projects', $project->getId()); @@ -1173,10 +1167,9 @@ App::get('/v1/projects/:projectId/platforms') throw new Exception('Project not found', 404); } - // TODO: Implement pagination $platforms = $dbForConsole->find('projectsPlatforms', [ new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) - ]); + ], 5000); $response->dynamic(new Document([ 'platforms' => $platforms, @@ -1208,7 +1201,7 @@ App::get('/v1/projects/:projectId/platforms/:platformId') throw new Exception('Project not found', 404); } - $platform = $dbForConsole->getDocument("projectsPlatforms", $platformId); + $platform = $dbForConsole->getDocument('projectsPlatforms', $platformId); if ($platform->isEmpty()) { throw new Exception('Platform not found', 404); @@ -1245,7 +1238,7 @@ App::put('/v1/projects/:projectId/platforms/:platformId') throw new Exception('Project not found', 404); } - $platform = $dbForConsole->getDocument("projectsPlatforms", $platformId); + $platform = $dbForConsole->getDocument('projectsPlatforms', $platformId); if ($platform->isEmpty()) { throw new Exception('Platform not found', 404); @@ -1289,7 +1282,7 @@ App::delete('/v1/projects/:projectId/platforms/:platformId') throw new Exception('Project not found', 404); } - $deleteRecord = $dbForConsole->deleteDocument("projectsPlatforms", $platformId); + $deleteRecord = $dbForConsole->deleteDocument('projectsPlatforms', $platformId); if($deleteRecord == false) { throw new Exception('Platform not found', 404); @@ -1326,7 +1319,7 @@ App::post('/v1/projects/:projectId/domains') throw new Exception('Project not found', 404); } - $document = $dbForConsole->findOne("projectDomains", [ + $document = $dbForConsole->findOne('projectDomains', [ new Query('domain', Query::TYPE_EQUAL, [$domain]), new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]), ]); @@ -1356,9 +1349,9 @@ App::post('/v1/projects/:projectId/domains') 'certificateId' => null, ]); - $domain = $dbForConsole->createDocument("projectDomains", $domain); + $domain = $dbForConsole->createDocument('projectDomains', $domain); - $dbForConsole->purgeDocument("projects", $project->getId()); + $dbForConsole->purgeDocument('projects', $project->getId()); $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($domain, Response::MODEL_DOMAIN); @@ -1387,8 +1380,7 @@ App::get('/v1/projects/:projectId/domains') throw new Exception('Project not found', 404); } - // TODO: Implement pagination - $domains = $dbForConsole->find("projectDomains", []); + $domains = $dbForConsole->find('projectDomains', [], 5000); $response->dynamic(new Document([ 'domains' => $domains, @@ -1420,7 +1412,7 @@ App::get('/v1/projects/:projectId/domains/:domainId') throw new Exception('Project not found', 404); } - $domain = $dbForConsole->getDocument("projectDomains", $domainId); + $domain = $dbForConsole->getDocument('projectDomains', $domainId); if ($domain->isEmpty()) { throw new Exception('Domain not found', 404); @@ -1453,7 +1445,7 @@ App::patch('/v1/projects/:projectId/domains/:domainId/verification') throw new Exception('Project not found', 404); } - $domain = $dbForConsole->getDocument("projectDomains", $domainId); + $domain = $dbForConsole->getDocument('projectDomains', $domainId); if ($domain->isEmpty()) { throw new Exception('Domain not found', 404); @@ -1475,10 +1467,10 @@ App::patch('/v1/projects/:projectId/domains/:domainId/verification') throw new Exception('Failed to verify domain', 401); } - $domain->setAttribute("verification", true); + $domain->setAttribute('verification', true); - $dbForConsole->updateDocument("projectDomains", $domain->getId(), $domain); - $dbForConsole->purgeDocument("projects", $project->getId()); + $dbForConsole->updateDocument('projectDomains', $domain->getId(), $domain); + $dbForConsole->purgeDocument('projects', $project->getId()); // Issue a TLS certificate when domain is verified Resque::enqueue('v1-certificates', 'CertificatesV1', [ @@ -1513,15 +1505,15 @@ App::delete('/v1/projects/:projectId/domains/:domainId') throw new Exception('Project not found', 404); } - $domain = $dbForConsole->getDocument("projectDomains", $domainId); + $domain = $dbForConsole->getDocument('projectDomains', $domainId); if ($domain->isEmpty()) { throw new Exception('Domain not found', 404); } - $dbForConsole->deleteDocument("projectDomains", $domain->getId()); + $dbForConsole->deleteDocument('projectDomains', $domain->getId()); - $dbForConsole->purgeDocument("projects", $project->getId()); + $dbForConsole->purgeDocument('projects', $project->getId()); $deletes ->setParam('type', DELETE_TYPE_CERTIFICATES) diff --git a/app/init.php b/app/init.php index 7c44ba12e6..adb42fbdd6 100644 --- a/app/init.php +++ b/app/init.php @@ -184,11 +184,10 @@ Database::addFilter('subQueryAttributes', return null; }, function($value, Document $document, Database $database) { - // TODO: Maybe implement pagination? return $database ->find('attributes', [ new Query('collectionId', Query::TYPE_EQUAL, [$document->getId()]) - ], 100, 0, []); + ], 5000, 0, []); } ); @@ -197,11 +196,10 @@ Database::addFilter('subQueryIndexes', return null; }, function($value, Document $document, Database $database) { - // TODO: Maybe implement pagination? return $database ->find('indexes', [ new Query('collectionId', Query::TYPE_EQUAL, [$document->getId()]) - ], 100, 0, []); + ], 5000, 0, []); } ); @@ -210,11 +208,10 @@ Database::addFilter('subQueryProjectPlatforms', return null; }, function($value, Document $document, Database $database) { - // TODO: Implement pagination return $database ->find('projectsPlatforms', [ new Query('projectId', Query::TYPE_EQUAL, [$document->getId()]) - ], 100, 0, []); + ], 5000, 0, []); } ); @@ -223,11 +220,10 @@ Database::addFilter('subQueryProjectDomains', return null; }, function($value, Document $document, Database $database) { - // TODO: Implement pagination return $database ->find('projectDomains', [ new Query('projectId', Query::TYPE_EQUAL, [$document->getId()]) - ], 100, 0, []); + ], 5000, 0, []); } ); @@ -236,11 +232,10 @@ Database::addFilter('subQueryProjectKeys', return null; }, function($value, Document $document, Database $database) { - // TODO: Implement pagination return $database ->find('projectKeys', [ new Query('projectId', Query::TYPE_EQUAL, [$document->getId()]) - ], 100, 0, []); + ], 5000, 0, []); } ); @@ -249,11 +244,10 @@ Database::addFilter('subQueryProjectWebhooks', return null; }, function($value, Document $document, Database $database) { - // TODO: Implement pagination return $database ->find('projectWebhooks', [ new Query('projectId', Query::TYPE_EQUAL, [$document->getId()]) - ], 100, 0, []); + ], 5000, 0, []); } ); @@ -262,15 +256,14 @@ Database::addFilter('subQueryProjectServices', return null; }, function($value, Document $document, Database $database) { - // TODO: Implement pagination $services = $database ->find('projectsServices', [ new Query('projectId', Query::TYPE_EQUAL, [$document->getId()]) - ], 100, 0, []); + ], 5000, 0, []); $responseJson = []; foreach($services as $service) { - $responseJson[$service->getAttribute("key")] = $service->getAttribute("status", true); + $responseJson[$service->getAttribute('key')] = $service->getAttribute('status', true); } return $responseJson; @@ -282,16 +275,15 @@ Database::addFilter('subQueryProjectProviders', return null; }, function($value, Document $document, Database $database) { - // TODO: Implement pagination $providers = $database ->find('projectProviders', [ new Query('projectId', Query::TYPE_EQUAL, [$document->getId()]) - ], 100, 0, []); + ], 5000, 0, []); $responseJson = []; foreach($providers as $provider) { - $responseJson[$provider->getAttribute("key") . 'Appid'] = $provider->getAttribute("appId"); - $responseJson[$provider->getAttribute("key") . 'Secret'] = $provider->getAttribute("appSecret"); + $responseJson[$provider->getAttribute('key') . 'Appid'] = $provider->getAttribute('appId'); + $responseJson[$provider->getAttribute('key') . 'Secret'] = $provider->getAttribute('appSecret'); } return $responseJson; From 51b1a7e72c8aa3152d790c1165b01e1b0b9917e4 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Wed, 1 Sep 2021 10:25:54 +0200 Subject: [PATCH 116/206] Removed re-selecting queries, improved selecting from sub-tables --- app/controllers/api/projects.php | 122 +++++++++++++++++++++---------- 1 file changed, 85 insertions(+), 37 deletions(-) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index cdc43d7b1a..97a914bafc 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -473,7 +473,7 @@ App::patch('/v1/projects/:projectId/service') ->param('status', true, new Boolean(), 'Service status.') ->inject('response') ->inject('dbForConsole') - ->action(function ($projectId, $serviceKey, $status, $response, $dbForConsole) { + ->action(function ($projectId, $service, $status, $response, $dbForConsole) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForConsole */ /** @var Boolean $status */ @@ -484,34 +484,38 @@ App::patch('/v1/projects/:projectId/service') throw new Exception('Project not found', 404); } - $service = $dbForConsole->findOne('projectsServices', [ + $document = $dbForConsole->findOne('projectsServices', [ new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]), - new Query('key', Query::TYPE_EQUAL, [$serviceKey]), + new Query('key', Query::TYPE_EQUAL, [$service]), ]); - if($service == false || $service->isEmpty()) { - $service = new Document([ + if($document == false || $document->isEmpty()) { + $document = new Document([ '$id' => $dbForConsole->getId(), '$read' => ['role:all'], '$write' => ['role:all'], 'projectId' => $project->getId(), - 'key' => $serviceKey, + 'key' => $service, 'status' => $status, ]); - $dbForConsole->createDocument('projectsServices', $service); + $dbForConsole->createDocument('projectsServices', $document); + + $project + ->setAttribute('services', $document, Document::SET_TYPE_APPEND); $dbForConsole->purgeDocument('projects', $project->getId()); } else { - if($service->getAttribute('status') != $status) { - $service->setAttribute('status', $status); - $dbForConsole->updateDocument('projectsServices', $service->getId(), $service); + if($document->getAttribute('status') != $status) { + $document->setAttribute('status', $status); + $dbForConsole->updateDocument('projectsServices', $document->getId(), $document); + + $project->findAndReplace('$id', $document->getId(), $document, 'services'); $dbForConsole->purgeDocument('projects', $project->getId()); } } - $project = $dbForConsole->getDocument('projects', $projectId); $response->dynamic($project, Response::MODEL_PROJECT); }); @@ -554,6 +558,8 @@ App::patch('/v1/projects/:projectId/oauth2') $dbForConsole->updateDocument('projectProviders', $provider->getId(), $provider); + $project->findAndReplace('$id', $provider->getId(), $provider, 'providers'); + $dbForConsole->purgeDocument('projects', $project->getId()); } else { // Provider does not exist yet @@ -570,10 +576,12 @@ App::patch('/v1/projects/:projectId/oauth2') $dbForConsole->createDocument('projectProviders', $provider); + $project + ->setAttribute('providers', $provider, Document::SET_TYPE_APPEND); + $dbForConsole->purgeDocument('projects', $project->getId()); } - $project = $dbForConsole->getDocument('projects', $projectId); $response->dynamic($project, Response::MODEL_PROJECT); }); @@ -805,9 +813,12 @@ App::get('/v1/projects/:projectId/webhooks/:webhookId') throw new Exception('Project not found', 404); } - $webhook = $dbForConsole->getDocument('projectWebhooks', $webhookId); + $webhook = $dbForConsole->findOne('projectWebhooks', [ + new Query('_uid', Query::TYPE_EQUAL, [$webhookId]), + new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) + ]); - if ($webhook->isEmpty()) { + if ($webhook == false || $webhook->isEmpty()) { throw new Exception('Webhook not found', 404); } @@ -846,9 +857,12 @@ App::put('/v1/projects/:projectId/webhooks/:webhookId') $security = ($security === '1' || $security === 'true' || $security === 1 || $security === true); - $webhook = $dbForConsole->getDocument('projectWebhooks', $webhookId); + $webhook = $dbForConsole->findOne('projectWebhooks', [ + new Query('_uid', Query::TYPE_EQUAL, [$webhookId]), + new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) + ]); - if ($webhook->isEmpty()) { + if ($webhook == false || $webhook->isEmpty()) { throw new Exception('Webhook not found', 404); } @@ -890,9 +904,12 @@ App::delete('/v1/projects/:projectId/webhooks/:webhookId') throw new Exception('Project not found', 404); } - $webhook = $dbForConsole->getDocument('projectWebhooks', $webhookId); + $webhook = $dbForConsole->findOne('projectWebhooks', [ + new Query('_uid', Query::TYPE_EQUAL, [$webhookId]), + new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) + ]); - if($webhook->isEmpty()) { + if($webhook == false || $webhook->isEmpty()) { throw new Exception('Webhook not found', 404); } @@ -1005,9 +1022,12 @@ App::get('/v1/projects/:projectId/keys/:keyId') throw new Exception('Project not found', 404); } - $key = $dbForConsole->getDocument('projectKeys', $keyId); + $key = $dbForConsole->findOne('projectKeys', [ + new Query('_uid', Query::TYPE_EQUAL, [$keyId]), + new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) + ]); - if ($key->isEmpty()) { + if ($key !== true || $key->isEmpty()) { throw new Exception('Key not found', 404); } @@ -1040,9 +1060,12 @@ App::put('/v1/projects/:projectId/keys/:keyId') throw new Exception('Project not found', 404); } - $key = $dbForConsole->getDocument('projectKeys', $keyId); + $key = $dbForConsole->findOne('projectKeys', [ + new Query('_uid', Query::TYPE_EQUAL, [$keyId]), + new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) + ]); - if ($key->isEmpty()) { + if ($key == false || $key->isEmpty()) { throw new Exception('Key not found', 404); } @@ -1079,9 +1102,12 @@ App::delete('/v1/projects/:projectId/keys/:keyId') throw new Exception('Project not found', 404); } - $key = $dbForConsole->getDocument('projectKeys', $keyId); + $key = $dbForConsole->findOne('projectKeys', [ + new Query('_uid', Query::TYPE_EQUAL, [$keyId]), + new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) + ]); - if($key->isEmpty()) { + if($key == false || $key->isEmpty()) { throw new Exception('Key not found', 404); } @@ -1201,9 +1227,12 @@ App::get('/v1/projects/:projectId/platforms/:platformId') throw new Exception('Project not found', 404); } - $platform = $dbForConsole->getDocument('projectsPlatforms', $platformId); + $platform = $dbForConsole->findOne('projectsPlatforms', [ + new Query('_uid', Query::TYPE_EQUAL, [$platformId]), + new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) + ]); - if ($platform->isEmpty()) { + if ($platform == false || $platform->isEmpty()) { throw new Exception('Platform not found', 404); } @@ -1238,9 +1267,12 @@ App::put('/v1/projects/:projectId/platforms/:platformId') throw new Exception('Project not found', 404); } - $platform = $dbForConsole->getDocument('projectsPlatforms', $platformId); + $platform = $dbForConsole->findOne('projectsPlatforms', [ + new Query('_uid', Query::TYPE_EQUAL, [$platformId]), + new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) + ]); - if ($platform->isEmpty()) { + if ($platform == false || $platform->isEmpty()) { throw new Exception('Platform not found', 404); } @@ -1282,12 +1314,17 @@ App::delete('/v1/projects/:projectId/platforms/:platformId') throw new Exception('Project not found', 404); } - $deleteRecord = $dbForConsole->deleteDocument('projectsPlatforms', $platformId); + $platform = $dbForConsole->findOne('projectsPlatforms', [ + new Query('_uid', Query::TYPE_EQUAL, [$platformId]), + new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) + ]); - if($deleteRecord == false) { + if ($platform == false || $platform->isEmpty()) { throw new Exception('Platform not found', 404); } + $dbForConsole->deleteDocument('projectsPlatforms', $platformId); + $dbForConsole->purgeDocument('projects', $project->getId()); $response->noContent(); @@ -1380,7 +1417,9 @@ App::get('/v1/projects/:projectId/domains') throw new Exception('Project not found', 404); } - $domains = $dbForConsole->find('projectDomains', [], 5000); + $domains = $dbForConsole->find('projectDomains', [ + new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) + ], 5000); $response->dynamic(new Document([ 'domains' => $domains, @@ -1412,9 +1451,12 @@ App::get('/v1/projects/:projectId/domains/:domainId') throw new Exception('Project not found', 404); } - $domain = $dbForConsole->getDocument('projectDomains', $domainId); + $domain = $dbForConsole->findOne('projectDomains', [ + new Query('_uid', Query::TYPE_EQUAL, [$domainId]), + new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) + ]); - if ($domain->isEmpty()) { + if ($domain == false || $domain->isEmpty()) { throw new Exception('Domain not found', 404); } @@ -1445,9 +1487,12 @@ App::patch('/v1/projects/:projectId/domains/:domainId/verification') throw new Exception('Project not found', 404); } - $domain = $dbForConsole->getDocument('projectDomains', $domainId); + $domain = $dbForConsole->findOne('projectDomains', [ + new Query('_uid', Query::TYPE_EQUAL, [$domainId]), + new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) + ]); - if ($domain->isEmpty()) { + if ($domain == false || $domain->isEmpty()) { throw new Exception('Domain not found', 404); } @@ -1505,9 +1550,12 @@ App::delete('/v1/projects/:projectId/domains/:domainId') throw new Exception('Project not found', 404); } - $domain = $dbForConsole->getDocument('projectDomains', $domainId); + $domain = $dbForConsole->findOne('projectDomains', [ + new Query('_uid', Query::TYPE_EQUAL, [$domainId]), + new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) + ]); - if ($domain->isEmpty()) { + if ($domain == false || $domain->isEmpty()) { throw new Exception('Domain not found', 404); } From 0ab76a1c6f0ae30a07c881a05245f8eeae3767c1 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Wed, 1 Sep 2021 10:43:35 +0200 Subject: [PATCH 117/206] Update sub-collection names and filter names, improved error log, fixed docs typo --- app/config/collections2.php | 48 +++++++++---------- app/controllers/api/projects.php | 74 +++++++++++++++--------------- app/init.php | 24 +++++----- src/Appwrite/Database/Database.php | 4 +- 4 files changed, 75 insertions(+), 75 deletions(-) diff --git a/app/config/collections2.php b/app/config/collections2.php index a29a9728c8..df2f2dbab7 100644 --- a/app/config/collections2.php +++ b/app/config/collections2.php @@ -478,7 +478,7 @@ $collections = [ 'required' => false, 'default' => null, 'array' => false, - 'filters' => ['subQueryProjectServices'], + 'filters' => ['subQueryServices'], ], [ '$id' => 'auths', @@ -500,7 +500,7 @@ $collections = [ 'required' => false, 'default' => null, 'array' => false, - 'filters' => ['subQueryProjectProviders'], + 'filters' => ['subQueryProviders'], ], [ '$id' => 'platforms', @@ -511,7 +511,7 @@ $collections = [ 'required' => false, 'default' => null, 'array' => false, - 'filters' => ['subQueryProjectPlatforms'], + 'filters' => ['subQueryPlatforms'], ], [ '$id' => 'webhooks', @@ -522,7 +522,7 @@ $collections = [ 'required' => false, 'default' => null, 'array' => false, - 'filters' => ['subQueryProjectWebhooks'], + 'filters' => ['subQueryWebhooks'], ], [ '$id' => 'keys', @@ -533,7 +533,7 @@ $collections = [ 'required' => false, 'default' => null, 'array' => false, - 'filters' => ['subQueryProjectKeys'], + 'filters' => ['subQueryKeys'], ], [ '$id' => 'domains', @@ -544,7 +544,7 @@ $collections = [ 'required' => false, 'default' => null, 'array' => false, - 'filters' => ['subQueryProjectDomains'], + 'filters' => ['subQueryDomains'], ], ], 'indexes' => [ @@ -558,10 +558,10 @@ $collections = [ ], ], - 'projectsPlatforms' => [ + 'platforms' => [ '$collection' => Database::METADATA, - '$id' => 'projectsPlatforms', - 'name' => 'projectsPlatforms', + '$id' => 'platforms', + 'name' => 'platforms', 'attributes' => [ [ '$id' => 'projectId', @@ -663,10 +663,10 @@ $collections = [ ], ], - 'projectsServices' => [ + 'services' => [ '$collection' => Database::METADATA, - '$id' => 'projectsServices', - 'name' => 'projectsServices', + '$id' => 'services', + 'name' => 'services', 'attributes' => [ [ '$id' => 'projectId', @@ -713,10 +713,10 @@ $collections = [ ], ], - 'projectProviders' => [ + 'providers' => [ '$collection' => Database::METADATA, - '$id' => 'projectProviders', - 'name' => 'projectProviders', + '$id' => 'providers', + 'name' => 'providers', 'attributes' => [ [ '$id' => 'projectId', @@ -774,10 +774,10 @@ $collections = [ ], ], - 'projectDomains' => [ + 'domains' => [ '$collection' => Database::METADATA, - '$id' => 'projectDomains', - 'name' => 'projectDomains', + '$id' => 'domains', + 'name' => 'domains', 'attributes' => [ [ '$id' => 'projectId', @@ -868,10 +868,10 @@ $collections = [ ], ], - 'projectKeys' => [ + 'keys' => [ '$collection' => Database::METADATA, - '$id' => 'projectKeys', - 'name' => 'projectKeys', + '$id' => 'keys', + 'name' => 'keys', 'attributes' => [ [ '$id' => 'projectId', @@ -929,10 +929,10 @@ $collections = [ ], ], - 'projectWebhooks' => [ + 'webhooks' => [ '$collection' => Database::METADATA, - '$id' => 'projectWebhooks', - 'name' => 'projectWebhooks', + '$id' => 'webhooks', + 'name' => 'webhooks', 'attributes' => [ [ '$id' => 'projectId', diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 97a914bafc..42cb3ebc1b 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -217,7 +217,7 @@ App::get('/v1/projects/:projectId') }); App::get('/v1/projects/:projectId/usage') - ->desc('Get Project') + ->desc('Get Project Usage') ->groups(['api', 'projects']) ->label('scope', 'projects.read') ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN]) @@ -484,7 +484,7 @@ App::patch('/v1/projects/:projectId/service') throw new Exception('Project not found', 404); } - $document = $dbForConsole->findOne('projectsServices', [ + $document = $dbForConsole->findOne('services', [ new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]), new Query('key', Query::TYPE_EQUAL, [$service]), ]); @@ -499,7 +499,7 @@ App::patch('/v1/projects/:projectId/service') 'status' => $status, ]); - $dbForConsole->createDocument('projectsServices', $document); + $dbForConsole->createDocument('services', $document); $project ->setAttribute('services', $document, Document::SET_TYPE_APPEND); @@ -508,7 +508,7 @@ App::patch('/v1/projects/:projectId/service') } else { if($document->getAttribute('status') != $status) { $document->setAttribute('status', $status); - $dbForConsole->updateDocument('projectsServices', $document->getId(), $document); + $dbForConsole->updateDocument('services', $document->getId(), $document); $project->findAndReplace('$id', $document->getId(), $document, 'services'); @@ -545,7 +545,7 @@ App::patch('/v1/projects/:projectId/oauth2') throw new Exception('Project not found', 404); } - $provider = $dbForConsole->findOne('projectProviders', [ + $provider = $dbForConsole->findOne('providers', [ new Query('key', Query::TYPE_EQUAL, [$providerKey]), new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]), ]); @@ -556,7 +556,7 @@ App::patch('/v1/projects/:projectId/oauth2') $provider->setAttribute('appId', $appId) ->setAttribute('appSecret', $secret); - $dbForConsole->updateDocument('projectProviders', $provider->getId(), $provider); + $dbForConsole->updateDocument('providers', $provider->getId(), $provider); $project->findAndReplace('$id', $provider->getId(), $provider, 'providers'); @@ -574,7 +574,7 @@ App::patch('/v1/projects/:projectId/oauth2') 'appSecret' => $secret ]); - $dbForConsole->createDocument('projectProviders', $provider); + $dbForConsole->createDocument('providers', $provider); $project ->setAttribute('providers', $provider, Document::SET_TYPE_APPEND); @@ -748,7 +748,7 @@ App::post('/v1/projects/:projectId/webhooks') 'httpPass' => $httpPass, ]); - $webhook = $dbForConsole->createDocument('projectWebhooks', $webhook); + $webhook = $dbForConsole->createDocument('webhooks', $webhook); $dbForConsole->purgeDocument('projects', $project->getId()); @@ -779,7 +779,7 @@ App::get('/v1/projects/:projectId/webhooks') throw new Exception('Project not found', 404); } - $webhooks = $dbForConsole->find('projectWebhooks', [ + $webhooks = $dbForConsole->find('webhooks', [ new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) ]); @@ -813,7 +813,7 @@ App::get('/v1/projects/:projectId/webhooks/:webhookId') throw new Exception('Project not found', 404); } - $webhook = $dbForConsole->findOne('projectWebhooks', [ + $webhook = $dbForConsole->findOne('webhooks', [ new Query('_uid', Query::TYPE_EQUAL, [$webhookId]), new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) ]); @@ -857,7 +857,7 @@ App::put('/v1/projects/:projectId/webhooks/:webhookId') $security = ($security === '1' || $security === 'true' || $security === 1 || $security === true); - $webhook = $dbForConsole->findOne('projectWebhooks', [ + $webhook = $dbForConsole->findOne('webhooks', [ new Query('_uid', Query::TYPE_EQUAL, [$webhookId]), new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) ]); @@ -874,7 +874,7 @@ App::put('/v1/projects/:projectId/webhooks/:webhookId') ->setAttribute('httpUser', $httpUser) ->setAttribute('httpPass', $httpPass); - $dbForConsole->updateDocument('projectWebhooks', $webhook->getId(), $webhook); + $dbForConsole->updateDocument('webhooks', $webhook->getId(), $webhook); $dbForConsole->purgeDocument('projects', $project->getId()); @@ -904,7 +904,7 @@ App::delete('/v1/projects/:projectId/webhooks/:webhookId') throw new Exception('Project not found', 404); } - $webhook = $dbForConsole->findOne('projectWebhooks', [ + $webhook = $dbForConsole->findOne('webhooks', [ new Query('_uid', Query::TYPE_EQUAL, [$webhookId]), new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) ]); @@ -913,7 +913,7 @@ App::delete('/v1/projects/:projectId/webhooks/:webhookId') throw new Exception('Webhook not found', 404); } - $dbForConsole->deleteDocument('projectWebhooks', $webhook->getId()); + $dbForConsole->deleteDocument('webhooks', $webhook->getId()); $dbForConsole->purgeDocument('projects', $project->getId()); @@ -957,7 +957,7 @@ App::post('/v1/projects/:projectId/keys') 'secret' => \bin2hex(\random_bytes(128)), ]); - $key = $dbForConsole->createDocument('projectKeys', $key); + $key = $dbForConsole->createDocument('keys', $key); $dbForConsole->purgeDocument('projects', $project->getId()); @@ -988,7 +988,7 @@ App::get('/v1/projects/:projectId/keys') throw new Exception('Project not found', 404); } - $keys = $dbForConsole->find('projectKeys', [ + $keys = $dbForConsole->find('keys', [ new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]), ], 5000); @@ -1022,12 +1022,12 @@ App::get('/v1/projects/:projectId/keys/:keyId') throw new Exception('Project not found', 404); } - $key = $dbForConsole->findOne('projectKeys', [ + $key = $dbForConsole->findOne('keys', [ new Query('_uid', Query::TYPE_EQUAL, [$keyId]), new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) ]); - if ($key !== true || $key->isEmpty()) { + if ($key == false || $key->isEmpty()) { throw new Exception('Key not found', 404); } @@ -1060,7 +1060,7 @@ App::put('/v1/projects/:projectId/keys/:keyId') throw new Exception('Project not found', 404); } - $key = $dbForConsole->findOne('projectKeys', [ + $key = $dbForConsole->findOne('keys', [ new Query('_uid', Query::TYPE_EQUAL, [$keyId]), new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) ]); @@ -1072,7 +1072,7 @@ App::put('/v1/projects/:projectId/keys/:keyId') $key->setAttribute('name', $name) ->setAttribute('scopes', $scopes); - $dbForConsole->updateDocument('projectKeys', $key->getId(), $key); + $dbForConsole->updateDocument('keys', $key->getId(), $key); $dbForConsole->purgeDocument('projects', $project->getId()); @@ -1102,7 +1102,7 @@ App::delete('/v1/projects/:projectId/keys/:keyId') throw new Exception('Project not found', 404); } - $key = $dbForConsole->findOne('projectKeys', [ + $key = $dbForConsole->findOne('keys', [ new Query('_uid', Query::TYPE_EQUAL, [$keyId]), new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) ]); @@ -1111,7 +1111,7 @@ App::delete('/v1/projects/:projectId/keys/:keyId') throw new Exception('Key not found', 404); } - $dbForConsole->deleteDocument('projectKeys', $key->getId()); + $dbForConsole->deleteDocument('keys', $key->getId()); $dbForConsole->purgeDocument('projects', $project->getId()); @@ -1162,7 +1162,7 @@ App::post('/v1/projects/:projectId/platforms') 'dateUpdated' => \time(), ]); - $platform = $dbForConsole->createDocument('projectsPlatforms', $platform); + $platform = $dbForConsole->createDocument('platforms', $platform); $dbForConsole->purgeDocument('projects', $project->getId()); @@ -1193,7 +1193,7 @@ App::get('/v1/projects/:projectId/platforms') throw new Exception('Project not found', 404); } - $platforms = $dbForConsole->find('projectsPlatforms', [ + $platforms = $dbForConsole->find('platforms', [ new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) ], 5000); @@ -1227,7 +1227,7 @@ App::get('/v1/projects/:projectId/platforms/:platformId') throw new Exception('Project not found', 404); } - $platform = $dbForConsole->findOne('projectsPlatforms', [ + $platform = $dbForConsole->findOne('platforms', [ new Query('_uid', Query::TYPE_EQUAL, [$platformId]), new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) ]); @@ -1267,7 +1267,7 @@ App::put('/v1/projects/:projectId/platforms/:platformId') throw new Exception('Project not found', 404); } - $platform = $dbForConsole->findOne('projectsPlatforms', [ + $platform = $dbForConsole->findOne('platforms', [ new Query('_uid', Query::TYPE_EQUAL, [$platformId]), new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) ]); @@ -1284,7 +1284,7 @@ App::put('/v1/projects/:projectId/platforms/:platformId') ->setAttribute('hostname', $hostname) ; - $dbForConsole->updateDocument('projectsPlatforms', $platform->getId(), $platform); + $dbForConsole->updateDocument('platforms', $platform->getId(), $platform); $dbForConsole->purgeDocument('projects', $project->getId()); @@ -1314,7 +1314,7 @@ App::delete('/v1/projects/:projectId/platforms/:platformId') throw new Exception('Project not found', 404); } - $platform = $dbForConsole->findOne('projectsPlatforms', [ + $platform = $dbForConsole->findOne('platforms', [ new Query('_uid', Query::TYPE_EQUAL, [$platformId]), new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) ]); @@ -1323,7 +1323,7 @@ App::delete('/v1/projects/:projectId/platforms/:platformId') throw new Exception('Platform not found', 404); } - $dbForConsole->deleteDocument('projectsPlatforms', $platformId); + $dbForConsole->deleteDocument('platforms', $platformId); $dbForConsole->purgeDocument('projects', $project->getId()); @@ -1356,7 +1356,7 @@ App::post('/v1/projects/:projectId/domains') throw new Exception('Project not found', 404); } - $document = $dbForConsole->findOne('projectDomains', [ + $document = $dbForConsole->findOne('domains', [ new Query('domain', Query::TYPE_EQUAL, [$domain]), new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]), ]); @@ -1386,7 +1386,7 @@ App::post('/v1/projects/:projectId/domains') 'certificateId' => null, ]); - $domain = $dbForConsole->createDocument('projectDomains', $domain); + $domain = $dbForConsole->createDocument('domains', $domain); $dbForConsole->purgeDocument('projects', $project->getId()); @@ -1417,7 +1417,7 @@ App::get('/v1/projects/:projectId/domains') throw new Exception('Project not found', 404); } - $domains = $dbForConsole->find('projectDomains', [ + $domains = $dbForConsole->find('domains', [ new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) ], 5000); @@ -1451,7 +1451,7 @@ App::get('/v1/projects/:projectId/domains/:domainId') throw new Exception('Project not found', 404); } - $domain = $dbForConsole->findOne('projectDomains', [ + $domain = $dbForConsole->findOne('domains', [ new Query('_uid', Query::TYPE_EQUAL, [$domainId]), new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) ]); @@ -1487,7 +1487,7 @@ App::patch('/v1/projects/:projectId/domains/:domainId/verification') throw new Exception('Project not found', 404); } - $domain = $dbForConsole->findOne('projectDomains', [ + $domain = $dbForConsole->findOne('domains', [ new Query('_uid', Query::TYPE_EQUAL, [$domainId]), new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) ]); @@ -1514,7 +1514,7 @@ App::patch('/v1/projects/:projectId/domains/:domainId/verification') $domain->setAttribute('verification', true); - $dbForConsole->updateDocument('projectDomains', $domain->getId(), $domain); + $dbForConsole->updateDocument('domains', $domain->getId(), $domain); $dbForConsole->purgeDocument('projects', $project->getId()); // Issue a TLS certificate when domain is verified @@ -1550,7 +1550,7 @@ App::delete('/v1/projects/:projectId/domains/:domainId') throw new Exception('Project not found', 404); } - $domain = $dbForConsole->findOne('projectDomains', [ + $domain = $dbForConsole->findOne('domains', [ new Query('_uid', Query::TYPE_EQUAL, [$domainId]), new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) ]); @@ -1559,7 +1559,7 @@ App::delete('/v1/projects/:projectId/domains/:domainId') throw new Exception('Domain not found', 404); } - $dbForConsole->deleteDocument('projectDomains', $domain->getId()); + $dbForConsole->deleteDocument('domains', $domain->getId()); $dbForConsole->purgeDocument('projects', $project->getId()); diff --git a/app/init.php b/app/init.php index adb42fbdd6..e30b7f696d 100644 --- a/app/init.php +++ b/app/init.php @@ -203,61 +203,61 @@ Database::addFilter('subQueryIndexes', } ); -Database::addFilter('subQueryProjectPlatforms', +Database::addFilter('subQueryPlatforms', function($value) { return null; }, function($value, Document $document, Database $database) { return $database - ->find('projectsPlatforms', [ + ->find('platforms', [ new Query('projectId', Query::TYPE_EQUAL, [$document->getId()]) ], 5000, 0, []); } ); -Database::addFilter('subQueryProjectDomains', +Database::addFilter('subQueryDomains', function($value) { return null; }, function($value, Document $document, Database $database) { return $database - ->find('projectDomains', [ + ->find('domains', [ new Query('projectId', Query::TYPE_EQUAL, [$document->getId()]) ], 5000, 0, []); } ); -Database::addFilter('subQueryProjectKeys', +Database::addFilter('subQueryKeys', function($value) { return null; }, function($value, Document $document, Database $database) { return $database - ->find('projectKeys', [ + ->find('keys', [ new Query('projectId', Query::TYPE_EQUAL, [$document->getId()]) ], 5000, 0, []); } ); -Database::addFilter('subQueryProjectWebhooks', +Database::addFilter('subQueryWebhooks', function($value) { return null; }, function($value, Document $document, Database $database) { return $database - ->find('projectWebhooks', [ + ->find('webhooks', [ new Query('projectId', Query::TYPE_EQUAL, [$document->getId()]) ], 5000, 0, []); } ); -Database::addFilter('subQueryProjectServices', +Database::addFilter('subQueryServices', function($value) { return null; }, function($value, Document $document, Database $database) { $services = $database - ->find('projectsServices', [ + ->find('services', [ new Query('projectId', Query::TYPE_EQUAL, [$document->getId()]) ], 5000, 0, []); @@ -270,13 +270,13 @@ Database::addFilter('subQueryProjectServices', } ); -Database::addFilter('subQueryProjectProviders', +Database::addFilter('subQueryProviders', function($value) { return null; }, function($value, Document $document, Database $database) { $providers = $database - ->find('projectProviders', [ + ->find('providers', [ new Query('projectId', Query::TYPE_EQUAL, [$document->getId()]) ], 5000, 0, []); diff --git a/src/Appwrite/Database/Database.php b/src/Appwrite/Database/Database.php index e6cfa001d1..37978ab65f 100644 --- a/src/Appwrite/Database/Database.php +++ b/src/Appwrite/Database/Database.php @@ -577,7 +577,7 @@ class Database { if (!isset(self::$filters[$name])) { return $value; - throw new Exception('Filter not found'); + throw new Exception("Filter '{$name}' not found"); } try { @@ -599,7 +599,7 @@ class Database { if (!isset(self::$filters[$name])) { return $value; - throw new Exception('Filter not found'); + throw new Exception("Filter '{$name}' not found"); } try { From dd9cddc525b8bfe8178d68daffce673f0184b0f4 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Wed, 1 Sep 2021 16:54:36 +0530 Subject: [PATCH 118/206] feat(review): fix review comments --- src/Appwrite/Specification/Format/OpenAPI3.php | 2 +- src/Appwrite/Specification/Format/Swagger2.php | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Appwrite/Specification/Format/OpenAPI3.php b/src/Appwrite/Specification/Format/OpenAPI3.php index 32e2347c0e..5095c5395f 100644 --- a/src/Appwrite/Specification/Format/OpenAPI3.php +++ b/src/Appwrite/Specification/Format/OpenAPI3.php @@ -27,7 +27,7 @@ class OpenAPI3 extends Format * Recursively get all used models * * @param object $model - * @param array $models + * @param array $models * * @return void */ diff --git a/src/Appwrite/Specification/Format/Swagger2.php b/src/Appwrite/Specification/Format/Swagger2.php index 75cac416d7..2a3be45d17 100644 --- a/src/Appwrite/Specification/Format/Swagger2.php +++ b/src/Appwrite/Specification/Format/Swagger2.php @@ -27,7 +27,7 @@ class Swagger2 extends Format * Recursively get all used models * * @param object $model - * @param array $models + * @param array $models * * @return void */ @@ -379,8 +379,6 @@ class Swagger2 extends Format $this->getUsedModels($model, $usedModels); } - // var_dump($usedModels); - foreach ($this->models as $model) { if (!in_array($model->getType(), $usedModels)) { continue; From be47d162907dffb4e486e9d56bb23bbfd0d30bd4 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Wed, 1 Sep 2021 10:08:59 -0400 Subject: [PATCH 119/206] Use MariaDB max limit for index/attribute subqueries --- app/init.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/init.php b/app/init.php index 57f2dc7130..d68f9f4851 100644 --- a/app/init.php +++ b/app/init.php @@ -184,7 +184,7 @@ Database::addFilter('subQueryAttributes', return $database ->find('attributes', [ new Query('collectionId', Query::TYPE_EQUAL, [$document->getId()]) - ], 100, 0, []); + ], 1017, 0, []); } ); @@ -196,7 +196,7 @@ Database::addFilter('subQueryIndexes', return $database ->find('indexes', [ new Query('collectionId', Query::TYPE_EQUAL, [$document->getId()]) - ], 100, 0, []); + ], 64, 0, []); } ); From 624d099bf9b71aabb598b658f60a807268e8dc78 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Wed, 1 Sep 2021 10:36:01 -0400 Subject: [PATCH 120/206] Assert row width exception is thrown --- tests/e2e/Services/Database/DatabaseCustomServerTest.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/e2e/Services/Database/DatabaseCustomServerTest.php b/tests/e2e/Services/Database/DatabaseCustomServerTest.php index 2be62f8fa8..6c0a500756 100644 --- a/tests/e2e/Services/Database/DatabaseCustomServerTest.php +++ b/tests/e2e/Services/Database/DatabaseCustomServerTest.php @@ -398,8 +398,8 @@ class DatabaseCustomServerTest extends Scope 'required' => true, ]); - // $this->assertEquals(400, $tooWide['headers']['status-code']); - // $this->assertEquals('Attribute limit exceeded', $tooWide['body']['message']); + $this->assertEquals(400, $tooWide['headers']['status-code']); + $this->assertEquals('Attribute limit exceeded', $tooWide['body']['message']); } public function testIndexLimitException() From 6b7059c034a8096e6067e71ec5fc10f038bf094e Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Wed, 1 Sep 2021 10:57:20 -0400 Subject: [PATCH 121/206] Fix missing lines from conflict resolution --- tests/e2e/Services/Database/DatabaseBase.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/e2e/Services/Database/DatabaseBase.php b/tests/e2e/Services/Database/DatabaseBase.php index ba0da3d2e4..96339b1fd2 100644 --- a/tests/e2e/Services/Database/DatabaseBase.php +++ b/tests/e2e/Services/Database/DatabaseBase.php @@ -1159,6 +1159,9 @@ trait DatabaseBase sleep(2); $document1 = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ 'documentId' => 'unique()', 'data' => [ 'attribute' => 'one', From 3a29d6a565487d535a96860312a420ceb83c85c4 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 2 Sep 2021 12:16:03 +0545 Subject: [PATCH 122/206] fix deletes, remove 15m period after refactor --- app/workers/deletes.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 4d5bd0cacc..3142065fa5 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -105,7 +105,7 @@ class DeletesV1 extends Worker // Delete Usage stats $this->deleteByGroup('stats', [ new Query('time', Query::TYPE_LESSER, [$timestamp1d]), - new Query('period', Query::TYPE_EQUAL, ['1d', '15m']), + new Query('period', Query::TYPE_EQUAL, ['1d']), ], $dbForInternal); $this->deleteByGroup('stats', [ From 334a59e3c1e226a648c4f15f7e5aa28d6e2773ba Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Thu, 2 Sep 2021 12:17:32 +0545 Subject: [PATCH 123/206] schedule usage stats deletion --- app/tasks/maintenance.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index a5cf63f821..f30c2a9a42 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -56,11 +56,12 @@ $cli $usageStatsRetention30m = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_30M', '129600');//36 hours $usageStatsRetention1d = (int) App::getEnv('_APP_MAINTENANCE_RETENTION_USAGE_1D', '8640000'); // 100 days - Console::loop(function() use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention){ + Console::loop(function() use ($interval, $executionLogsRetention, $abuseLogsRetention, $auditLogRetention, $usageStatsRetention30m, $usageStatsRetention1d) { $time = date('d-m-Y H:i:s', time()); Console::info("[{$time}] Notifying deletes workers every {$interval} seconds"); notifyDeleteExecutionLogs($executionLogsRetention); notifyDeleteAbuseLogs($abuseLogsRetention); notifyDeleteAuditLogs($auditLogRetention); + notifyDeleteUsageStats($usageStatsRetention30m, $usageStatsRetention1d); }, $interval); }); \ No newline at end of file From 524abc80d70fbec15820b92763247c6cf0097499 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Thu, 2 Sep 2021 11:38:24 +0200 Subject: [PATCH 124/206] Rollback of services and providers sub-collections --- app/config/collections2.php | 4 +- app/controllers/api/projects.php | 71 ++++---------------------------- app/init.php | 39 ------------------ 3 files changed, 11 insertions(+), 103 deletions(-) diff --git a/app/config/collections2.php b/app/config/collections2.php index df2f2dbab7..29375bec5c 100644 --- a/app/config/collections2.php +++ b/app/config/collections2.php @@ -478,7 +478,7 @@ $collections = [ 'required' => false, 'default' => null, 'array' => false, - 'filters' => ['subQueryServices'], + 'filters' => ['json'], ], [ '$id' => 'auths', @@ -500,7 +500,7 @@ $collections = [ 'required' => false, 'default' => null, 'array' => false, - 'filters' => ['subQueryProviders'], + 'filters' => ['json'], ], [ '$id' => 'platforms', diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 42cb3ebc1b..973df927d4 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -93,9 +93,9 @@ App::post('/v1/projects') 'legalCity' => $legalCity, 'legalAddress' => $legalAddress, 'legalTaxId' => $legalTaxId, - 'services' => null, + 'services' => new stdClass(), 'platforms' => null, - 'providers' => null, + 'providers' => [], 'webhooks' => null, 'keys' => null, 'domains' => null, @@ -484,37 +484,10 @@ App::patch('/v1/projects/:projectId/service') throw new Exception('Project not found', 404); } - $document = $dbForConsole->findOne('services', [ - new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]), - new Query('key', Query::TYPE_EQUAL, [$service]), - ]); + $services = $project->getAttribute('services', []); + $services[$service] = $status; - if($document == false || $document->isEmpty()) { - $document = new Document([ - '$id' => $dbForConsole->getId(), - '$read' => ['role:all'], - '$write' => ['role:all'], - 'projectId' => $project->getId(), - 'key' => $service, - 'status' => $status, - ]); - - $dbForConsole->createDocument('services', $document); - - $project - ->setAttribute('services', $document, Document::SET_TYPE_APPEND); - - $dbForConsole->purgeDocument('projects', $project->getId()); - } else { - if($document->getAttribute('status') != $status) { - $document->setAttribute('status', $status); - $dbForConsole->updateDocument('services', $document->getId(), $document); - - $project->findAndReplace('$id', $document->getId(), $document, 'services'); - - $dbForConsole->purgeDocument('projects', $project->getId()); - } - } + $project = $dbForConsole->updateDocument('projects', $project->getId(), $project->setAttribute('services', $services)); $response->dynamic($project, Response::MODEL_PROJECT); }); @@ -550,37 +523,11 @@ App::patch('/v1/projects/:projectId/oauth2') new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]), ]); - if($provider && !$provider->isEmpty()) { - // Provider exists + $providers = $project->getAttribute('providers', []); + $providers[$provider . 'Appid'] = $appId; + $providers[$provider . 'Secret'] = $secret; - $provider->setAttribute('appId', $appId) - ->setAttribute('appSecret', $secret); - - $dbForConsole->updateDocument('providers', $provider->getId(), $provider); - - $project->findAndReplace('$id', $provider->getId(), $provider, 'providers'); - - $dbForConsole->purgeDocument('projects', $project->getId()); - } else { - // Provider does not exist yet - - $provider = new Document([ - '$id' => $dbForConsole->getId(), - '$read' => ['role:all'], - '$write' => ['role:all'], - 'projectId' => $project->getId(), - 'key' => $providerKey, - 'appId' => $appId, - 'appSecret' => $secret - ]); - - $dbForConsole->createDocument('providers', $provider); - - $project - ->setAttribute('providers', $provider, Document::SET_TYPE_APPEND); - - $dbForConsole->purgeDocument('projects', $project->getId()); - } + $project = $dbForConsole->updateDocument('projects', $project->getId(), $project->setAttribute('providers', $providers)); $response->dynamic($project, Response::MODEL_PROJECT); }); diff --git a/app/init.php b/app/init.php index e30b7f696d..86c69ae599 100644 --- a/app/init.php +++ b/app/init.php @@ -251,45 +251,6 @@ Database::addFilter('subQueryWebhooks', } ); -Database::addFilter('subQueryServices', - function($value) { - return null; - }, - function($value, Document $document, Database $database) { - $services = $database - ->find('services', [ - new Query('projectId', Query::TYPE_EQUAL, [$document->getId()]) - ], 5000, 0, []); - - $responseJson = []; - foreach($services as $service) { - $responseJson[$service->getAttribute('key')] = $service->getAttribute('status', true); - } - - return $responseJson; - } -); - -Database::addFilter('subQueryProviders', - function($value) { - return null; - }, - function($value, Document $document, Database $database) { - $providers = $database - ->find('providers', [ - new Query('projectId', Query::TYPE_EQUAL, [$document->getId()]) - ], 5000, 0, []); - - $responseJson = []; - foreach($providers as $provider) { - $responseJson[$provider->getAttribute('key') . 'Appid'] = $provider->getAttribute('appId'); - $responseJson[$provider->getAttribute('key') . 'Secret'] = $provider->getAttribute('appSecret'); - } - - return $responseJson; - } -); - Database::addFilter('encrypt', function($value) { $key = App::getEnv('_APP_OPENSSL_KEY_V1'); From 0fa6c764132cb4efc34aaa128818436a62448958 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Thu, 2 Sep 2021 13:12:10 +0200 Subject: [PATCH 125/206] Bug fix, tests are now green --- app/controllers/api/projects.php | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 973df927d4..0c8d254c22 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -508,7 +508,7 @@ App::patch('/v1/projects/:projectId/oauth2') ->param('secret', '', new text(512), 'Provider secret key. Max length: 512 chars.', true) ->inject('response') ->inject('dbForConsole') - ->action(function ($projectId, $providerKey, $appId, $secret, $response, $dbForConsole) { + ->action(function ($projectId, $provider, $appId, $secret, $response, $dbForConsole) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForConsole */ @@ -518,11 +518,6 @@ App::patch('/v1/projects/:projectId/oauth2') throw new Exception('Project not found', 404); } - $provider = $dbForConsole->findOne('providers', [ - new Query('key', Query::TYPE_EQUAL, [$providerKey]), - new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]), - ]); - $providers = $project->getAttribute('providers', []); $providers[$provider . 'Appid'] = $appId; $providers[$provider . 'Secret'] = $secret; From 84e9881a6a4638f9253ca409d685442178ce3a39 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Thu, 2 Sep 2021 12:45:03 -0400 Subject: [PATCH 126/206] Delete from attribute/index tables on deleteCollection --- app/controllers/api/database.php | 5 +++++ app/workers/deletes.php | 22 ++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 7df3172484..191ed38e5f 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -426,6 +426,11 @@ App::delete('/v1/database/collections/:collectionId') throw new Exception('Failed to remove collection from DB', 500); } + $deletes + ->setParam('type', DELETE_TYPE_DOCUMENT) + ->setParam('document', $collection) + ; + $events ->setParam('eventData', $response->output($collection, Response::MODEL_COLLECTION)) ; diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 1894aaf76b..759409d286 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -44,6 +44,9 @@ class DeletesV1 extends Worker switch ($document->getCollection()) { // TODO@kodumbeats define these as constants somewhere + case 'collections': + $this->deleteCollection($document, $projectId); + break; case 'projects': $this->deleteProject($document); break; @@ -88,6 +91,25 @@ class DeletesV1 extends Worker public function shutdown(): void { } + + /** + * @param Document $document teams document + * @param string $projectId + */ + protected function deleteCollection(Document $document, string $projectId): void + { + $collectionId = $document->getId(); + + $dbForInternal = $this->getInternalDB($projectId); + + $this->deleteByGroup('attributes', [ + new Query('collectionId', Query::TYPE_EQUAL, [$collectionId]) + ], $dbForInternal); + + $this->deleteByGroup('indexes', [ + new Query('collectionId', Query::TYPE_EQUAL, [$collectionId]) + ], $dbForInternal); + } /** * @param Document $document teams document From 8c3ca6808c8a1d9c48b095b66f669d53fe91144d Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Thu, 2 Sep 2021 12:46:05 -0400 Subject: [PATCH 127/206] Test deleteCollection removal of attributes/indexes --- tests/e2e/Services/Database/DatabaseCustomServerTest.php | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/e2e/Services/Database/DatabaseCustomServerTest.php b/tests/e2e/Services/Database/DatabaseCustomServerTest.php index e5a75fa685..79750318b2 100644 --- a/tests/e2e/Services/Database/DatabaseCustomServerTest.php +++ b/tests/e2e/Services/Database/DatabaseCustomServerTest.php @@ -403,5 +403,13 @@ class DatabaseCustomServerTest extends Scope $this->assertEquals(400, $tooMany['headers']['status-code']); $this->assertEquals('Index limit exceeded', $tooMany['body']['message']); + + $collection = $this->client->call(Client::METHOD_DELETE, '/database/collections/' . $collectionId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ])); + + $this->assertEquals(204, $collection['headers']['status-code']); } } \ No newline at end of file From 31977e462c27d1788de0714e20751701c418646b Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Thu, 2 Sep 2021 17:38:33 -0400 Subject: [PATCH 128/206] Clean up indexes when attribute is deleted --- app/workers/database.php | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/app/workers/database.php b/app/workers/database.php index fd38246150..ecd617cc52 100644 --- a/app/workers/database.php +++ b/app/workers/database.php @@ -120,6 +120,28 @@ class DatabaseV1 extends Worker $dbForInternal->updateDocument('attributes', $attribute->getId(), $attribute->setAttribute('status', 'failed')); } + // the underlying database removes/rebuilds indexes when attribute is removed + // update indexes table with changes + /** @var Document[] $indexes */ + $indexes = $collection->getAttribute('indexes', []); + + foreach ($indexes as $index) { + /** @var string[] $attributes */ + $attributes = $index->getAttribute('attributes'); + $found = array_search($key, $attributes); + + if ($found !== false) { + $remove = [$attributes[$found]]; + $attributes = array_diff($attributes, $remove); // remove attribute from array + + if (empty($attributes)) { + $dbForInternal->deleteDocument('indexes', $index->getId()); + } else { + $dbForInternal->updateDocument('indexes', $index->getId(), $index->setAttribute('attributes', $attributes, Document::SET_TYPE_ASSIGN)); + } + } + } + $dbForInternal->purgeDocument('collections', $collectionId); } From 1410101db80cd4219ccf5f2c6d42edbf46202570 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Thu, 2 Sep 2021 17:38:46 -0400 Subject: [PATCH 129/206] Test index cleanup when attribute is deleted --- .../Database/DatabaseCustomServerTest.php | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/tests/e2e/Services/Database/DatabaseCustomServerTest.php b/tests/e2e/Services/Database/DatabaseCustomServerTest.php index e5a75fa685..b642a57673 100644 --- a/tests/e2e/Services/Database/DatabaseCustomServerTest.php +++ b/tests/e2e/Services/Database/DatabaseCustomServerTest.php @@ -241,6 +241,103 @@ class DatabaseCustomServerTest extends Scope /** * @depends testDeleteIndex */ + public function testDeleteIndexOnDeleteAttribute($data) + { + $attribute1 = $this->client->call(Client::METHOD_POST, '/database/collections/' . $data['collectionId'] . '/attributes/string', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'attributeId' => 'attribute1', + 'size' => 16, + 'required' => true, + ]); + + $attribute2 = $this->client->call(Client::METHOD_POST, '/database/collections/' . $data['collectionId'] . '/attributes/string', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'attributeId' => 'attribute2', + 'size' => 16, + 'required' => true, + ]); + + $this->assertEquals(201, $attribute1['headers']['status-code']); + $this->assertEquals(201, $attribute2['headers']['status-code']); + $this->assertEquals('attribute1', $attribute1['body']['key']); + $this->assertEquals('attribute2', $attribute2['body']['key']); + + sleep(2); + + $index1 = $this->client->call(Client::METHOD_POST, '/database/collections/' . $data['collectionId'] . '/indexes', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'indexId' => 'index1', + 'type' => 'key', + 'attributes' => ['attribute1', 'attribute2'], + 'orders' => ['ASC', 'ASC'], + ]); + + $index2 = $this->client->call(Client::METHOD_POST, '/database/collections/' . $data['collectionId'] . '/indexes', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'indexId' => 'index2', + 'type' => 'key', + 'attributes' => ['attribute2'], + ]); + + $this->assertEquals(201, $index1['headers']['status-code']); + $this->assertEquals(201, $index2['headers']['status-code']); + $this->assertEquals('index1', $index1['body']['key']); + $this->assertEquals('index2', $index2['body']['key']); + + sleep(2); + + $deleted = $this->client->call(Client::METHOD_DELETE, '/database/collections/' . $data['collectionId'] . '/attributes/'. $attribute2['body']['key'], array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ])); + + $this->assertEquals($deleted['headers']['status-code'], 204); + + // wait for database worker to complete + sleep(2); + + $collection = $this->client->call(Client::METHOD_GET, '/database/collections/' . $data['collectionId'], array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ])); + + $this->assertEquals(200, $collection['headers']['status-code']); + $this->assertIsArray($collection['body']['indexes']); + $this->assertCount(1, $collection['body']['indexes']); + $this->assertEquals($index1['body']['key'], $collection['body']['indexes'][0]['key']); + $this->assertIsArray($collection['body']['indexes'][0]['attributes']); + $this->assertCount(1, $collection['body']['indexes'][0]['attributes']); + $this->assertEquals($attribute1['body']['key'], $collection['body']['indexes'][0]['attributes'][0]); + + // Delete attribute + $deleted = $this->client->call(Client::METHOD_DELETE, '/database/collections/' . $data['collectionId'] . '/attributes/' . $attribute1['body']['key'], array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ])); + + $this->assertEquals($deleted['headers']['status-code'], 204); + + return $data; + } + + /** + * @depends testDeleteIndexOnDeleteAttribute + */ public function testDeleteCollection($data) { $collectionId = $data['collectionId']; From 1a21c8fdbbf53fd0ff94622244895a470016325e Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 6 Sep 2021 11:48:44 +0545 Subject: [PATCH 130/206] fix constant value --- app/init.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/init.php b/app/init.php index 475bb3e40d..78c44076e5 100644 --- a/app/init.php +++ b/app/init.php @@ -89,7 +89,7 @@ const DELETE_TYPE_EXECUTIONS = 'executions'; const DELETE_TYPE_AUDIT = 'audit'; const DELETE_TYPE_ABUSE = 'abuse'; const DELETE_TYPE_CERTIFICATES = 'certificates'; -const DELETE_TYPE_USAGE_STATS = 'certificates'; +const DELETE_TYPE_USAGE_STATS = 'usageStats'; // Mail Worker Types const MAIL_TYPE_VERIFICATION = 'verification'; const MAIL_TYPE_RECOVERY = 'recovery'; From eccc19e5dc46fa6c087106a77b10da899d19339a Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Mon, 6 Sep 2021 12:13:20 +0530 Subject: [PATCH 131/206] Apply suggestions from code review Co-authored-by: Damodar Lohani --- app/controllers/api/database.php | 4 ++-- app/controllers/api/functions.php | 2 +- app/controllers/api/projects.php | 2 +- app/controllers/api/storage.php | 2 +- app/controllers/api/users.php | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index be0a346047..5a8a599900 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -270,7 +270,7 @@ App::get('/v1/database/usage') if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $period = [ '24h' => [ - 'period' => '15m', + 'period' => '30m', 'limit' => 48, ], '7d' => [ @@ -369,7 +369,7 @@ App::get('/v1/database/:collectionId/usage') if(App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $period = [ '24h' => [ - 'period' => '15m', + 'period' => '30m', 'limit' => 48, ], '7d' => [ diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index fadc323832..2a67c6d4c6 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -171,7 +171,7 @@ App::get('/v1/functions/:functionId/usage') if(App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $period = [ '24h' => [ - 'period' => '15m', + 'period' => '30m', 'limit' => 48, ], '7d' => [ diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 0d1c3b713a..3c7dbb8e5e 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -248,7 +248,7 @@ App::get('/v1/projects/:projectId/usage') if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $period = [ '24h' => [ - 'period' => '15m', + 'period' => '30m', 'limit' => 48, ], '7d' => [ diff --git a/app/controllers/api/storage.php b/app/controllers/api/storage.php index 69ed5ba536..b9b2b8e8a1 100644 --- a/app/controllers/api/storage.php +++ b/app/controllers/api/storage.php @@ -665,7 +665,7 @@ App::get('/v1/storage/usage') if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $period = [ '24h' => [ - 'period' => '15m', + 'period' => '30m', 'limit' => 48, ], '7d' => [ diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 48c67daa82..c55f5382f5 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -634,7 +634,7 @@ App::get('/v1/users/usage') if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') { $period = [ '24h' => [ - 'period' => '15m', + 'period' => '30m', 'limit' => 48, ], '7d' => [ From 9f93ceec13cdb962d4c604bdc6ced7d046eefc34 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 6 Sep 2021 15:24:36 +0545 Subject: [PATCH 132/206] missing break --- app/workers/deletes.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 3142065fa5..f9571949f2 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -81,7 +81,7 @@ class DeletesV1 extends Worker case DELETE_TYPE_USAGE_STATS: $this->deleteUsageStats($this->args['timestamp1d'], $this->args['timestamp30m']); - + break; default: Console::error('No delete operation for type: '.$type); break; From 1ecd9eecffab2f804140e89c21a8bdc548063950 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Tue, 7 Sep 2021 11:03:30 +0545 Subject: [PATCH 133/206] fix deletes interval --- app/tasks/maintenance.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index f30c2a9a42..fdfd7c942b 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -43,8 +43,8 @@ $cli { Resque::enqueue(Event::DELETE_QUEUE_NAME, Event::DELETE_CLASS_NAME, [ 'type' => DELETE_TYPE_USAGE_STATS, - 'timestamp1d' => $interval1d, - 'timestamp30m' => $interval30m, + 'timestamp1d' => time() - $interval1d, + 'timestamp30m' => time() - $interval30m, ]); } From b38c77255e0d03e4e5206a5da09ede133d7ec508 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Wed, 8 Sep 2021 12:40:48 +0545 Subject: [PATCH 134/206] Update app/init.php Co-authored-by: Eldad A. Fux --- app/init.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/init.php b/app/init.php index 78c44076e5..4c1e6cf219 100644 --- a/app/init.php +++ b/app/init.php @@ -89,7 +89,7 @@ const DELETE_TYPE_EXECUTIONS = 'executions'; const DELETE_TYPE_AUDIT = 'audit'; const DELETE_TYPE_ABUSE = 'abuse'; const DELETE_TYPE_CERTIFICATES = 'certificates'; -const DELETE_TYPE_USAGE_STATS = 'usageStats'; +const DELETE_TYPE_USAGE = 'usage'; // Mail Worker Types const MAIL_TYPE_VERIFICATION = 'verification'; const MAIL_TYPE_RECOVERY = 'recovery'; From e1c7e4908b60a93daf8d6f9e8046d92ef640968b Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Thu, 9 Sep 2021 12:52:56 -0400 Subject: [PATCH 135/206] Remove unneeded var --- app/controllers/api/database.php | 1 - 1 file changed, 1 deletion(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 1d2b7c9977..633e6f2325 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -1230,7 +1230,6 @@ App::get('/v1/database/collections/:collectionId/documents') throw new Exception($validator->getDescription(), 400); } - $afterDocument = null; if (!empty($after)) { $afterDocument = $dbForExternal->getDocument($collectionId, $after); From f5d69b4b0aaaba03480323d798097412179f2811 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Thu, 9 Sep 2021 13:15:30 -0400 Subject: [PATCH 136/206] Catch exceptions in one block --- app/controllers/api/database.php | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 7cda576626..5560d931d9 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -88,16 +88,13 @@ $attributesCallback = function ($collectionId, $attribute, $response, $dbForInte try { $dbForInternal->checkAttribute($collection, $attribute); - } catch (LimitException $exception) { - throw new Exception('Attribute limit exceeded', 400); - } - - try { $attribute = $dbForInternal->createDocument('attributes', $attribute); - } catch (DuplicateException $th) { + } + catch (DuplicateException $exception) { throw new Exception('Attribute already exists', 409); - } catch (LimitException $e) { - throw new Exception($e->getMessage(), 400); + } + catch (LimitException $exception) { + throw new Exception('Attribute limit exceeded', 400); } $dbForInternal->purgeDocument('collections', $collectionId); From 5802212dd8d56d61ea2cf518fdc64d32ed5b9813 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Fri, 10 Sep 2021 16:08:49 +0545 Subject: [PATCH 137/206] fix usage daemon first time startup --- app/tasks/usage.php | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index e66bec113f..4ff5e8cad9 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -238,10 +238,27 @@ $cli /** * Aggregate InfluxDB every 30 seconds + * @var InfluxDB\Client $client */ $client = $register->get('influxdb'); if ($client) { + $attempts = 0; + $max = 10; + $sleep = 1; + $database = $client->selectDB('telegraf'); + do { // check if telegraf database is ready + $attempts++; + if(!in_array('telegraf', $client->listDatabases())) { + Console::warning("InfluxDB not ready. Retrying connection ({$attempts})..."); + if($attempts >= $max) { + throw new \Exception('InfluxDB database not ready yet'); + } + sleep($sleep); + } else { + break; // leave the do-while if successful + } + } while ($attempts < $max); // sync data foreach ($globalMetrics as $metric => $options) { //for each metrics @@ -318,7 +335,23 @@ $cli $latestProject = null; do { // Loop over all the projects - $projects = $dbForConsole->find('projects', [], 100, orderAfter:$latestProject); + $attempts = 0; + $max = 10; + $sleep = 1; + + do { // list projects + try { + $attempts++; + $projects = $dbForConsole->find('projects', [], 100, orderAfter:$latestProject); + break; // leave the do-while if successful + } catch (\Exception $e) { + Console::warning("Console DB not ready yet. Retrying ({$attempts})..."); + if ($attempts >= $max) { + throw new \Exception('Failed access console db: ' . $e->getMessage()); + } + sleep($sleep); + } + } while ($attempts < $max); if (empty($projects)) { continue; From e9124f28c722603102d1dd73d0987966e73a1a3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Sat, 11 Sep 2021 19:29:17 +0200 Subject: [PATCH 138/206] Removed unnecessary array_merge import Co-authored-by: Eldad A. Fux --- tests/e2e/Services/Projects/ProjectsConsoleClientTest.php | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php index f090158cd0..2f91663c98 100644 --- a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php +++ b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php @@ -7,7 +7,6 @@ use Tests\E2E\Scopes\ProjectConsole; use Tests\E2E\Scopes\SideClient; use Tests\E2E\Services\Projects\ProjectsBase; use Tests\E2E\Client; -use function array_merge; class ProjectsConsoleClientTest extends Scope { From d3e02db31f670aef41ac8e16ad66a023cdf69b62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 14 Sep 2021 08:57:55 +0200 Subject: [PATCH 139/206] Apply suggestions from code review Co-authored-by: kodumbeats --- app/controllers/api/projects.php | 36 +++++++++++++++++--------------- app/init.php | 4 ++-- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 0c8d254c22..f726371922 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -760,7 +760,7 @@ App::get('/v1/projects/:projectId/webhooks/:webhookId') new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) ]); - if ($webhook == false || $webhook->isEmpty()) { + if ($webhook === false || $webhook->isEmpty()) { throw new Exception('Webhook not found', 404); } @@ -804,7 +804,7 @@ App::put('/v1/projects/:projectId/webhooks/:webhookId') new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) ]); - if ($webhook == false || $webhook->isEmpty()) { + if ($webhook === false || $webhook->isEmpty()) { throw new Exception('Webhook not found', 404); } @@ -814,7 +814,8 @@ App::put('/v1/projects/:projectId/webhooks/:webhookId') ->setAttribute('url', $url) ->setAttribute('security', $security) ->setAttribute('httpUser', $httpUser) - ->setAttribute('httpPass', $httpPass); + ->setAttribute('httpPass', $httpPass) + ; $dbForConsole->updateDocument('webhooks', $webhook->getId(), $webhook); @@ -851,7 +852,7 @@ App::delete('/v1/projects/:projectId/webhooks/:webhookId') new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) ]); - if($webhook == false || $webhook->isEmpty()) { + if($webhook === false || $webhook->isEmpty()) { throw new Exception('Webhook not found', 404); } @@ -969,7 +970,7 @@ App::get('/v1/projects/:projectId/keys/:keyId') new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) ]); - if ($key == false || $key->isEmpty()) { + if ($key === false || $key->isEmpty()) { throw new Exception('Key not found', 404); } @@ -1007,12 +1008,14 @@ App::put('/v1/projects/:projectId/keys/:keyId') new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) ]); - if ($key == false || $key->isEmpty()) { + if ($key === false || $key->isEmpty()) { throw new Exception('Key not found', 404); } - $key->setAttribute('name', $name) - ->setAttribute('scopes', $scopes); + $key + ->setAttribute('name', $name) + ->setAttribute('scopes', $scopes) + ; $dbForConsole->updateDocument('keys', $key->getId(), $key); @@ -1049,7 +1052,7 @@ App::delete('/v1/projects/:projectId/keys/:keyId') new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) ]); - if($key == false || $key->isEmpty()) { + if($key === false || $key->isEmpty()) { throw new Exception('Key not found', 404); } @@ -1174,7 +1177,7 @@ App::get('/v1/projects/:projectId/platforms/:platformId') new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) ]); - if ($platform == false || $platform->isEmpty()) { + if ($platform === false || $platform->isEmpty()) { throw new Exception('Platform not found', 404); } @@ -1214,7 +1217,7 @@ App::put('/v1/projects/:projectId/platforms/:platformId') new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) ]); - if ($platform == false || $platform->isEmpty()) { + if ($platform === false || $platform->isEmpty()) { throw new Exception('Platform not found', 404); } @@ -1261,7 +1264,7 @@ App::delete('/v1/projects/:projectId/platforms/:platformId') new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) ]); - if ($platform == false || $platform->isEmpty()) { + if ($platform === false || $platform->isEmpty()) { throw new Exception('Platform not found', 404); } @@ -1398,7 +1401,7 @@ App::get('/v1/projects/:projectId/domains/:domainId') new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) ]); - if ($domain == false || $domain->isEmpty()) { + if ($domain === false || $domain->isEmpty()) { throw new Exception('Domain not found', 404); } @@ -1434,7 +1437,7 @@ App::patch('/v1/projects/:projectId/domains/:domainId/verification') new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) ]); - if ($domain == false || $domain->isEmpty()) { + if ($domain === false || $domain->isEmpty()) { throw new Exception('Domain not found', 404); } @@ -1454,9 +1457,8 @@ App::patch('/v1/projects/:projectId/domains/:domainId/verification') throw new Exception('Failed to verify domain', 401); } - $domain->setAttribute('verification', true); - $dbForConsole->updateDocument('domains', $domain->getId(), $domain); + $dbForConsole->updateDocument('domains', $domain->getId(), $domain->setAttribute('verification', true)); $dbForConsole->purgeDocument('projects', $project->getId()); // Issue a TLS certificate when domain is verified @@ -1497,7 +1499,7 @@ App::delete('/v1/projects/:projectId/domains/:domainId') new Query('projectId', Query::TYPE_EQUAL, [$project->getId()]) ]); - if ($domain == false || $domain->isEmpty()) { + if ($domain === false || $domain->isEmpty()) { throw new Exception('Domain not found', 404); } diff --git a/app/init.php b/app/init.php index 86c69ae599..7892ac8a10 100644 --- a/app/init.php +++ b/app/init.php @@ -187,7 +187,7 @@ Database::addFilter('subQueryAttributes', return $database ->find('attributes', [ new Query('collectionId', Query::TYPE_EQUAL, [$document->getId()]) - ], 5000, 0, []); + ], 1017, 0, []); } ); @@ -199,7 +199,7 @@ Database::addFilter('subQueryIndexes', return $database ->find('indexes', [ new Query('collectionId', Query::TYPE_EQUAL, [$document->getId()]) - ], 5000, 0, []); + ], 64, 0, []); } ); From ba6c52861711e98ad84ac9a2ee70a11fbdfb5785 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Tue, 14 Sep 2021 08:58:19 +0200 Subject: [PATCH 140/206] Apply suggestions from code review 2 --- app/config/collections2.php | 111 ------------------------------- app/controllers/api/projects.php | 2 +- 2 files changed, 1 insertion(+), 112 deletions(-) diff --git a/app/config/collections2.php b/app/config/collections2.php index 29375bec5c..97d6db253c 100644 --- a/app/config/collections2.php +++ b/app/config/collections2.php @@ -663,117 +663,6 @@ $collections = [ ], ], - 'services' => [ - '$collection' => Database::METADATA, - '$id' => 'services', - 'name' => 'services', - 'attributes' => [ - [ - '$id' => 'projectId', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => Database::LENGTH_KEY, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'key', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => Database::LENGTH_KEY, - 'signed' => true, - 'required' => true, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'status', - 'type' => Database::VAR_BOOLEAN, - 'format' => '', - 'size' => 0, - 'signed' => true, - 'required' => true, - 'default' => null, - 'array' => false, - 'filters' => [], - ] - ], - 'indexes' => [ - [ - '$id' => '_key_project', - 'type' => Database::INDEX_KEY, - 'attributes' => ['projectId'], - 'lengths' => [Database::LENGTH_KEY], - 'orders' => [Database::ORDER_ASC], - ], - ], - ], - - 'providers' => [ - '$collection' => Database::METADATA, - '$id' => 'providers', - 'name' => 'providers', - 'attributes' => [ - [ - '$id' => 'projectId', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => Database::LENGTH_KEY, - 'signed' => true, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'key', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => Database::LENGTH_KEY, - 'signed' => true, - 'required' => true, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'appId', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => Database::LENGTH_KEY, - 'signed' => true, - 'required' => true, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - [ - '$id' => 'appSecret', - 'type' => Database::VAR_STRING, - 'format' => '', - 'size' => 16384, - 'signed' => true, - 'required' => true, - 'default' => null, - 'array' => false, - 'filters' => [], - ], - ], - 'indexes' => [ - [ - '$id' => '_key_project', - 'type' => Database::INDEX_KEY, - 'attributes' => ['projectId'], - 'lengths' => [Database::LENGTH_KEY], - 'orders' => [Database::ORDER_ASC], - ], - ], - ], - 'domains' => [ '$collection' => Database::METADATA, '$id' => 'domains', diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index f726371922..8e99f200a7 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -470,7 +470,7 @@ App::patch('/v1/projects/:projectId/service') ->label('sdk.response.model', Response::MODEL_PROJECT) ->param('projectId', '', new UID(), 'Project unique ID.') ->param('service', '', new WhiteList(array_keys(array_filter(Config::getParam('services'), function($element) {return $element['optional'];})), true), 'Service name.') - ->param('status', true, new Boolean(), 'Service status.') + ->param('status', null, new Boolean(), 'Service status.') ->inject('response') ->inject('dbForConsole') ->action(function ($projectId, $service, $status, $response, $dbForConsole) { From 0992576e525394c6dbbf5f8b22eaa381cb0bb756 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Tue, 14 Sep 2021 10:26:16 +0200 Subject: [PATCH 141/206] introduce response type array --- src/Appwrite/Utopia/Response.php | 21 +++++++++++++-- src/Appwrite/Utopia/Response/Model.php | 7 ++--- .../Utopia/Response/Model/Attribute.php | 2 ++ .../Response/Model/AttributeBoolean.php | 4 +++ .../Utopia/Response/Model/AttributeEmail.php | 5 ++++ .../Utopia/Response/Model/AttributeFloat.php | 4 +++ .../Utopia/Response/Model/AttributeIP.php | 5 ++++ .../Response/Model/AttributeInteger.php | 4 +++ .../Utopia/Response/Model/AttributeList.php | 26 +++++++------------ .../Utopia/Response/Model/AttributeString.php | 4 +++ .../Utopia/Response/Model/AttributeURL.php | 5 ++++ .../Utopia/Response/Model/Collection.php | 24 +++++++---------- 12 files changed, 73 insertions(+), 38 deletions(-) diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index a653ddbb0f..00e4194c8b 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -341,7 +341,24 @@ class Response extends SwooleResponse foreach ($data[$key] as &$item) { if ($item instanceof Document) { - $ruleType = (!\is_null($rule['getNestedType'])) ? $rule['getNestedType']($item) : $rule['type']; + if (\is_array($rule['type'])) { + foreach ($rule['type'] as $type) { + $condition = false; + foreach ($this->getModel($type)->conditions as $attribute => $val) { + $condition = $item->getAttribute($attribute) === $val; + if(!$condition) { + break; + } + } + if ($condition) { + $ruleType = $type; + break; + } + } + } else { + $ruleType = $rule['type']; + } + if (!array_key_exists($ruleType, $this->models)) { throw new Exception('Missing model for rule: '. $ruleType); } @@ -350,7 +367,7 @@ class Response extends SwooleResponse } } } - + $output[$key] = $data[$key]; } diff --git a/src/Appwrite/Utopia/Response/Model.php b/src/Appwrite/Utopia/Response/Model.php index a763692c71..4c3143ca90 100644 --- a/src/Appwrite/Utopia/Response/Model.php +++ b/src/Appwrite/Utopia/Response/Model.php @@ -69,13 +69,11 @@ abstract class Model /** * Add a New Rule * If rule is an array of documents with varying models - * Pass callable $getNestedType that accepts Document and returns the nested response type * * @param string $key * @param array $options - * @param callable $getNestedType function(Document $value): string */ - protected function addRule(string $key, array $options, callable $getNestedType = null): self + protected function addRule(string $key, array $options): self { $this->rules[$key] = array_merge([ 'require' => true, @@ -83,8 +81,7 @@ abstract class Model 'description' => '', 'default' => null, 'example' => '', - 'array' => false, - 'getNestedType' => $getNestedType + 'array' => false ], $options); return $this; diff --git a/src/Appwrite/Utopia/Response/Model/Attribute.php b/src/Appwrite/Utopia/Response/Model/Attribute.php index 2c9dbf7c22..281a37a733 100644 --- a/src/Appwrite/Utopia/Response/Model/Attribute.php +++ b/src/Appwrite/Utopia/Response/Model/Attribute.php @@ -44,6 +44,8 @@ class Attribute extends Model ; } + public array $conditions = []; + /** * Get Name * diff --git a/src/Appwrite/Utopia/Response/Model/AttributeBoolean.php b/src/Appwrite/Utopia/Response/Model/AttributeBoolean.php index 4076bc9eb4..66aab770dd 100644 --- a/src/Appwrite/Utopia/Response/Model/AttributeBoolean.php +++ b/src/Appwrite/Utopia/Response/Model/AttributeBoolean.php @@ -23,6 +23,10 @@ class AttributeBoolean extends Attribute ; } + public array $conditions = [ + 'type' => self::TYPE_BOOLEAN + ]; + /** * Get Name * diff --git a/src/Appwrite/Utopia/Response/Model/AttributeEmail.php b/src/Appwrite/Utopia/Response/Model/AttributeEmail.php index 15bc4674ed..a048e2c41e 100644 --- a/src/Appwrite/Utopia/Response/Model/AttributeEmail.php +++ b/src/Appwrite/Utopia/Response/Model/AttributeEmail.php @@ -31,6 +31,11 @@ class AttributeEmail extends Attribute ; } + public array $conditions = [ + 'type' => self::TYPE_STRING, + 'format' => \APP_DATABASE_ATTRIBUTE_EMAIL + ]; + /** * Get Name * diff --git a/src/Appwrite/Utopia/Response/Model/AttributeFloat.php b/src/Appwrite/Utopia/Response/Model/AttributeFloat.php index 7c4ead9e77..56ed9c68d2 100644 --- a/src/Appwrite/Utopia/Response/Model/AttributeFloat.php +++ b/src/Appwrite/Utopia/Response/Model/AttributeFloat.php @@ -39,6 +39,10 @@ class AttributeFloat extends Attribute ; } + public array $conditions = [ + 'type' => self::TYPE_FLOAT, + ]; + /** * Get Name * diff --git a/src/Appwrite/Utopia/Response/Model/AttributeIP.php b/src/Appwrite/Utopia/Response/Model/AttributeIP.php index ec3a9cfed6..6e3a8d7baa 100644 --- a/src/Appwrite/Utopia/Response/Model/AttributeIP.php +++ b/src/Appwrite/Utopia/Response/Model/AttributeIP.php @@ -31,6 +31,11 @@ class AttributeIP extends Attribute ; } + public array $conditions = [ + 'type' => self::TYPE_STRING, + 'format' => \APP_DATABASE_ATTRIBUTE_IP + ]; + /** * Get Name * diff --git a/src/Appwrite/Utopia/Response/Model/AttributeInteger.php b/src/Appwrite/Utopia/Response/Model/AttributeInteger.php index f11344d79e..b2e86c79b8 100644 --- a/src/Appwrite/Utopia/Response/Model/AttributeInteger.php +++ b/src/Appwrite/Utopia/Response/Model/AttributeInteger.php @@ -39,6 +39,10 @@ class AttributeInteger extends Attribute ; } + public array $conditions = [ + 'type' => self::TYPE_INTEGER, + ]; + /** * Get Name * * @return string diff --git a/src/Appwrite/Utopia/Response/Model/AttributeList.php b/src/Appwrite/Utopia/Response/Model/AttributeList.php index 26ea146ec8..cad333bdfc 100644 --- a/src/Appwrite/Utopia/Response/Model/AttributeList.php +++ b/src/Appwrite/Utopia/Response/Model/AttributeList.php @@ -18,24 +18,18 @@ class AttributeList extends Model 'example' => 5, ]) ->addRule('attributes', [ - 'type' => Response::MODEL_ATTRIBUTE, + 'type' => [ + Response::MODEL_ATTRIBUTE_BOOLEAN, + Response::MODEL_ATTRIBUTE_INTEGER, + Response::MODEL_ATTRIBUTE_FLOAT, + Response::MODEL_ATTRIBUTE_EMAIL, + Response::MODEL_ATTRIBUTE_URL, + Response::MODEL_ATTRIBUTE_IP, + Response::MODEL_ATTRIBUTE_STRING // needs to be last, since its condition would dominate any other string attribute + ], 'description' => 'List of attributes.', 'default' => [], - 'array' => true, - 'getNestedType' => function(Document $attribute) { - return match($attribute->getAttribute('type')) { - self::TYPE_BOOLEAN => Response::MODEL_ATTRIBUTE_BOOLEAN, - self::TYPE_INTEGER => Response::MODEL_ATTRIBUTE_INTEGER, - self::TYPE_FLOAT => Response::MODEL_ATTRIBUTE_FLOAT, - self::TYPE_STRING => match($attribute->getAttribute('format')) { - APP_DATABASE_ATTRIBUTE_EMAIL => Response::MODEL_ATTRIBUTE_EMAIL, - APP_DATABASE_ATTRIBUTE_IP => Response::MODEL_ATTRIBUTE_IP, - APP_DATABASE_ATTRIBUTE_URL => Response::MODEL_ATTRIBUTE_URL, - default => Response::MODEL_ATTRIBUTE_STRING, - }, - default => Response::MODEL_ATTRIBUTE, - }; - }, + 'array' => true ]) ; } diff --git a/src/Appwrite/Utopia/Response/Model/AttributeString.php b/src/Appwrite/Utopia/Response/Model/AttributeString.php index 22f7e52a3f..9feaf6b7be 100644 --- a/src/Appwrite/Utopia/Response/Model/AttributeString.php +++ b/src/Appwrite/Utopia/Response/Model/AttributeString.php @@ -29,6 +29,10 @@ class AttributeString extends Attribute ; } + public array $conditions = [ + 'type' => self::TYPE_STRING, + ]; + /** * Get Name * diff --git a/src/Appwrite/Utopia/Response/Model/AttributeURL.php b/src/Appwrite/Utopia/Response/Model/AttributeURL.php index 3caddbc106..476d9bba01 100644 --- a/src/Appwrite/Utopia/Response/Model/AttributeURL.php +++ b/src/Appwrite/Utopia/Response/Model/AttributeURL.php @@ -31,6 +31,11 @@ class AttributeURL extends Attribute ; } + public array $conditions = [ + 'type' => self::TYPE_STRING, + 'format' => \APP_DATABASE_ATTRIBUTE_URL + ]; + /** * Get Name * diff --git a/src/Appwrite/Utopia/Response/Model/Collection.php b/src/Appwrite/Utopia/Response/Model/Collection.php index f023be261f..e52539691d 100644 --- a/src/Appwrite/Utopia/Response/Model/Collection.php +++ b/src/Appwrite/Utopia/Response/Model/Collection.php @@ -45,25 +45,19 @@ class Collection extends Model 'example' => 'document', ]) ->addRule('attributes', [ - 'type' => Response::MODEL_ATTRIBUTE, + 'type' => [ + Response::MODEL_ATTRIBUTE_BOOLEAN, + Response::MODEL_ATTRIBUTE_INTEGER, + Response::MODEL_ATTRIBUTE_FLOAT, + Response::MODEL_ATTRIBUTE_EMAIL, + Response::MODEL_ATTRIBUTE_URL, + Response::MODEL_ATTRIBUTE_IP, + Response::MODEL_ATTRIBUTE_STRING, // needs to be last, since its condition would dominate any other string attribute + ], 'description' => 'Collection attributes.', 'default' => [], 'example' => new stdClass, 'array' => true, - 'getNestedType' => function(Document $attribute) { - return match($attribute->getAttribute('type')) { - self::TYPE_BOOLEAN => Response::MODEL_ATTRIBUTE_BOOLEAN, - self::TYPE_INTEGER => Response::MODEL_ATTRIBUTE_INTEGER, - self::TYPE_FLOAT => Response::MODEL_ATTRIBUTE_FLOAT, - self::TYPE_STRING => match($attribute->getAttribute('format')) { - APP_DATABASE_ATTRIBUTE_EMAIL => Response::MODEL_ATTRIBUTE_EMAIL, - APP_DATABASE_ATTRIBUTE_IP => Response::MODEL_ATTRIBUTE_IP, - APP_DATABASE_ATTRIBUTE_URL => Response::MODEL_ATTRIBUTE_URL, - default => Response::MODEL_ATTRIBUTE_STRING, - }, - default => Response::MODEL_ATTRIBUTE, - }; - }, ]) ->addRule('indexes', [ 'type' => Response::MODEL_INDEX, From 4cf16c88edf04ae97724762bdafdfb56f2075499 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Wed, 15 Sep 2021 11:17:09 -0400 Subject: [PATCH 142/206] Update docblocks for getType --- src/Appwrite/Utopia/Response/Model/AttributeBoolean.php | 2 +- src/Appwrite/Utopia/Response/Model/AttributeEmail.php | 2 +- src/Appwrite/Utopia/Response/Model/AttributeFloat.php | 2 +- src/Appwrite/Utopia/Response/Model/AttributeIP.php | 2 +- src/Appwrite/Utopia/Response/Model/AttributeInteger.php | 2 +- src/Appwrite/Utopia/Response/Model/AttributeList.php | 2 +- src/Appwrite/Utopia/Response/Model/AttributeString.php | 2 +- src/Appwrite/Utopia/Response/Model/AttributeURL.php | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Appwrite/Utopia/Response/Model/AttributeBoolean.php b/src/Appwrite/Utopia/Response/Model/AttributeBoolean.php index 66aab770dd..de7ae58130 100644 --- a/src/Appwrite/Utopia/Response/Model/AttributeBoolean.php +++ b/src/Appwrite/Utopia/Response/Model/AttributeBoolean.php @@ -38,7 +38,7 @@ class AttributeBoolean extends Attribute } /** - * Get Collection + * Get Type * * @return string */ diff --git a/src/Appwrite/Utopia/Response/Model/AttributeEmail.php b/src/Appwrite/Utopia/Response/Model/AttributeEmail.php index a048e2c41e..268412a557 100644 --- a/src/Appwrite/Utopia/Response/Model/AttributeEmail.php +++ b/src/Appwrite/Utopia/Response/Model/AttributeEmail.php @@ -47,7 +47,7 @@ class AttributeEmail extends Attribute } /** - * Get Collection + * Get Type * * @return string */ diff --git a/src/Appwrite/Utopia/Response/Model/AttributeFloat.php b/src/Appwrite/Utopia/Response/Model/AttributeFloat.php index 56ed9c68d2..93033f5b0a 100644 --- a/src/Appwrite/Utopia/Response/Model/AttributeFloat.php +++ b/src/Appwrite/Utopia/Response/Model/AttributeFloat.php @@ -54,7 +54,7 @@ class AttributeFloat extends Attribute } /** - * Get Collection + * Get Type * * @return string */ diff --git a/src/Appwrite/Utopia/Response/Model/AttributeIP.php b/src/Appwrite/Utopia/Response/Model/AttributeIP.php index 6e3a8d7baa..2b8f64dede 100644 --- a/src/Appwrite/Utopia/Response/Model/AttributeIP.php +++ b/src/Appwrite/Utopia/Response/Model/AttributeIP.php @@ -47,7 +47,7 @@ class AttributeIP extends Attribute } /** - * Get Collection + * Get Type * * @return string */ diff --git a/src/Appwrite/Utopia/Response/Model/AttributeInteger.php b/src/Appwrite/Utopia/Response/Model/AttributeInteger.php index b2e86c79b8..9ff94c8f56 100644 --- a/src/Appwrite/Utopia/Response/Model/AttributeInteger.php +++ b/src/Appwrite/Utopia/Response/Model/AttributeInteger.php @@ -53,7 +53,7 @@ class AttributeInteger extends Attribute } /** - * Get Collection + * Get Type * * @return string */ diff --git a/src/Appwrite/Utopia/Response/Model/AttributeList.php b/src/Appwrite/Utopia/Response/Model/AttributeList.php index cad333bdfc..0cf67b3bd2 100644 --- a/src/Appwrite/Utopia/Response/Model/AttributeList.php +++ b/src/Appwrite/Utopia/Response/Model/AttributeList.php @@ -45,7 +45,7 @@ class AttributeList extends Model } /** - * Get Collection + * Get Type * * @return string */ diff --git a/src/Appwrite/Utopia/Response/Model/AttributeString.php b/src/Appwrite/Utopia/Response/Model/AttributeString.php index 9feaf6b7be..e7ea853260 100644 --- a/src/Appwrite/Utopia/Response/Model/AttributeString.php +++ b/src/Appwrite/Utopia/Response/Model/AttributeString.php @@ -44,7 +44,7 @@ class AttributeString extends Attribute } /** - * Get Collection + * Get Type * * @return string */ diff --git a/src/Appwrite/Utopia/Response/Model/AttributeURL.php b/src/Appwrite/Utopia/Response/Model/AttributeURL.php index 476d9bba01..489e038851 100644 --- a/src/Appwrite/Utopia/Response/Model/AttributeURL.php +++ b/src/Appwrite/Utopia/Response/Model/AttributeURL.php @@ -47,7 +47,7 @@ class AttributeURL extends Attribute } /** - * Get Collection + * Get Type * * @return string */ From 8cde715d04027c4f7b9559c15b65ff19f4d7d7e1 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Wed, 15 Sep 2021 11:28:40 -0400 Subject: [PATCH 143/206] Use correct constant when deleting usage stats --- app/tasks/maintenance.php | 2 +- app/workers/deletes.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/tasks/maintenance.php b/app/tasks/maintenance.php index fdfd7c942b..e0ef1adc01 100644 --- a/app/tasks/maintenance.php +++ b/app/tasks/maintenance.php @@ -42,7 +42,7 @@ $cli function notifyDeleteUsageStats(int $interval30m, int $interval1d) { Resque::enqueue(Event::DELETE_QUEUE_NAME, Event::DELETE_CLASS_NAME, [ - 'type' => DELETE_TYPE_USAGE_STATS, + 'type' => DELETE_TYPE_USAGE, 'timestamp1d' => time() - $interval1d, 'timestamp30m' => time() - $interval30m, ]); diff --git a/app/workers/deletes.php b/app/workers/deletes.php index f9571949f2..7016244f92 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -79,7 +79,7 @@ class DeletesV1 extends Worker $this->deleteCertificates($document); break; - case DELETE_TYPE_USAGE_STATS: + case DELETE_TYPE_USAGE: $this->deleteUsageStats($this->args['timestamp1d'], $this->args['timestamp30m']); break; default: From e90036f67451b3a52bdedc473b240f80c154deb7 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Wed, 15 Sep 2021 11:46:08 -0400 Subject: [PATCH 144/206] Exit tests if a container has status exited --- .travis.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.travis.yml b/.travis.yml index a90890423c..325865fcd2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,6 +35,12 @@ install: script: - docker ps -a +# Tests should fail if any container is in exited status +- ALL_UP=`docker ps -aq --filter "status=exited"` +- > + if [[ "$ALL_UP" != "" ]]; then + exit 1 + fi - docker-compose logs appwrite - docker-compose logs mariadb - docker-compose logs appwrite-worker-functions From bfdf6de15f101ec1cfd62373e646cb0862ab3eb9 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Wed, 15 Sep 2021 14:02:18 -0400 Subject: [PATCH 145/206] Delete collection document from internal collections table --- app/workers/deletes.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/workers/deletes.php b/app/workers/deletes.php index 78ceeb45a6..f557a3a549 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -112,6 +112,8 @@ class DeletesV1 extends Worker $this->deleteByGroup('indexes', [ new Query('collectionId', Query::TYPE_EQUAL, [$collectionId]) ], $dbForInternal); + + $dbForInternal->deleteDocument('collections', $collectionId); } /** From 43d5d67f4321bd5e33af515228e152ca1e058302 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 16 Sep 2021 10:46:59 +0530 Subject: [PATCH 146/206] feat(tests): check for failing test --- phpunit.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/phpunit.xml b/phpunit.xml index f0a2813f5a..56885de032 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -19,7 +19,7 @@ ./tests/e2e/Client.php ./tests/e2e/General ./tests/e2e/Scopes - ./tests/e2e/Services/Account + ./tests/e2e/Services/Functions/FunctionsBase.php ./tests/e2e/Services/Functions/FunctionsCustomServerTest.php ./tests/e2e/Services/Functions/FunctionsCustomClientTest.php From f8067b9cfbb72ba2d5368a2daa3c5e9bdee38f79 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 16 Sep 2021 11:08:21 +0530 Subject: [PATCH 147/206] feat(tests): check for failing test --- phpunit.xml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/phpunit.xml b/phpunit.xml index 56885de032..57e374c80c 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -13,14 +13,14 @@ - ./tests/unit/ + - ./tests/e2e/Client.php + + ./tests/e2e/Services/Account + - ./tests/e2e/Services/Functions/FunctionsBase.php + \ No newline at end of file From 3f4a0756ad60c104f6e459285ca6f26dfc3604b7 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 16 Sep 2021 12:36:24 +0530 Subject: [PATCH 148/206] feat(tests): check for failing test --- phpunit.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/phpunit.xml b/phpunit.xml index 57e374c80c..572dceaeea 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -19,10 +19,10 @@ - ./tests/e2e/Services/Account - + ./tests/e2e/Services/Database - ./tests/e2e/Services/Health + - ./tests/e2e/Services/Database - + ./tests/e2e/Services/Health ./tests/e2e/Services/Locale - ./tests/e2e/Services/Projects - ./tests/e2e/Services/Storage - ./tests/e2e/Services/Teams - ./tests/e2e/Services/Users - ./tests/e2e/Services/Workers - ./tests/e2e/Services/Webhooks --> + + + + + + From b580810c7ddef61cd0573cb0b259257a39e6c041 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 16 Sep 2021 13:29:20 +0530 Subject: [PATCH 150/206] feat(tests): check for failing test --- .travis.yml | 3 +++ phpunit.xml | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index a90890423c..f5e7ee4681 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,9 @@ arch: os: linux +vm: + size: x-large + language: shell notifications: diff --git a/phpunit.xml b/phpunit.xml index 44cc631d0e..bd38ab510a 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -19,11 +19,11 @@ - + ./tests/e2e/Services/Account - ./tests/e2e/Services/Health - ./tests/e2e/Services/Locale + From cad09af1bdb37c8d31a072ae641b83702f306884 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 16 Sep 2021 13:49:18 +0530 Subject: [PATCH 151/206] feat(tests): check for failing test --- .travis.yml | 2 +- phpunit.xml | 30 +++++++++++++++--------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index f5e7ee4681..2fa9ff9357 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ arch: os: linux vm: - size: x-large + size: large language: shell diff --git a/phpunit.xml b/phpunit.xml index bd38ab510a..f0a2813f5a 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -13,26 +13,26 @@ - + ./tests/unit/ - + ./tests/e2e/Scopes ./tests/e2e/Services/Account - - - - - - - - - - + ./tests/e2e/Services/Functions/FunctionsCustomClientTest.php \ No newline at end of file From 086f6312116cb1eea975e36286738b1c4f9ae955 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 16 Sep 2021 17:04:23 +0530 Subject: [PATCH 152/206] feat(tests): check for failing test --- .travis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2fa9ff9357..fe7bc3098e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,9 +6,10 @@ arch: os: linux +# Small change to check build failure vm: - size: large - + size: large + language: shell notifications: From 351b252c2cf2a1aa6e64242c0bc9b8a046f6cad0 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 16 Sep 2021 17:35:53 +0530 Subject: [PATCH 153/206] feat(tests): check for failing test --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index fe7bc3098e..077c8cc787 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ arch: os: linux -# Small change to check build failure +# Small change to check build vm: size: large From 2853b7f52a5958a293b6877491cc9ad122529d33 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 16 Sep 2021 18:10:18 +0530 Subject: [PATCH 154/206] feat(tests): check for failing test --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 077c8cc787..d2f93874b5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ arch: os: linux -# Small change to check build +# Small change to check vm: size: large From c03dc64adb8d1c524119abaac5026c75acd50b33 Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 16 Sep 2021 18:48:23 +0530 Subject: [PATCH 155/206] feat(tests): check for failing test --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d2f93874b5..89dad4bbab 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ arch: os: linux -# Small change to check +# Small change to vm: size: large From 13024e97f2ebe8dc244b9f1a0fd20fb4be73ae3f Mon Sep 17 00:00:00 2001 From: Christy Jacob Date: Thu, 16 Sep 2021 19:27:12 +0530 Subject: [PATCH 156/206] feat(tests): check for failing test --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 89dad4bbab..a1a2f6bd65 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ arch: os: linux -# Small change to +# Small change vm: size: large From b390c663a456bd855ef93847dacff78615a21e84 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Thu, 16 Sep 2021 20:57:25 -0400 Subject: [PATCH 157/206] Upgrade travis instance from medium to large --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index a90890423c..2fa9ff9357 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,9 @@ arch: os: linux +vm: + size: large + language: shell notifications: From 38a6e147c2cc05438074cff84aa92b6b7ab461f3 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Thu, 16 Sep 2021 21:52:35 -0400 Subject: [PATCH 158/206] Use limits defined by database library --- app/init.php | 4 ++-- composer.json | 2 +- composer.lock | 50 +++++++++++++++++++++++++------------------------- 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/app/init.php b/app/init.php index d68f9f4851..c5a15cc734 100644 --- a/app/init.php +++ b/app/init.php @@ -184,7 +184,7 @@ Database::addFilter('subQueryAttributes', return $database ->find('attributes', [ new Query('collectionId', Query::TYPE_EQUAL, [$document->getId()]) - ], 1017, 0, []); + ], $database->getAttributeLimit(), 0, []); } ); @@ -196,7 +196,7 @@ Database::addFilter('subQueryIndexes', return $database ->find('indexes', [ new Query('collectionId', Query::TYPE_EQUAL, [$document->getId()]) - ], 64, 0, []); + ], $database->getIndexLimit(), 0, []); } ); diff --git a/composer.json b/composer.json index 9bd152fb92..c51501f712 100644 --- a/composer.json +++ b/composer.json @@ -45,7 +45,7 @@ "utopia-php/cache": "0.4.*", "utopia-php/cli": "0.11.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "dev-feat-check-attribute-method as 0.10.0", + "utopia-php/database": "dev-feat-get-limit-methods-on-database as 0.10.0", "utopia-php/locale": "0.4.*", "utopia-php/registry": "0.5.*", "utopia-php/preloader": "0.2.*", diff --git a/composer.lock b/composer.lock index 829e348201..b6418d1482 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "9a7739aabd503572b8010be91192b144", + "content-hash": "ffd14884e3b377752bdb699c5693cb0d", "packages": [ { "name": "adhocore/jwt", @@ -248,16 +248,16 @@ }, { "name": "chillerlan/php-settings-container", - "version": "2.1.1", + "version": "2.1.2", "source": { "type": "git", "url": "https://github.com/chillerlan/php-settings-container.git", - "reference": "98ccc1b31b31a53bcb563465c4961879b2b93096" + "reference": "ec834493a88682dd69652a1eeaf462789ed0c5f5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/chillerlan/php-settings-container/zipball/98ccc1b31b31a53bcb563465c4961879b2b93096", - "reference": "98ccc1b31b31a53bcb563465c4961879b2b93096", + "url": "https://api.github.com/repos/chillerlan/php-settings-container/zipball/ec834493a88682dd69652a1eeaf462789ed0c5f5", + "reference": "ec834493a88682dd69652a1eeaf462789ed0c5f5", "shasum": "" }, "require": { @@ -307,7 +307,7 @@ "type": "ko_fi" } ], - "time": "2021-01-06T15:57:03+00:00" + "time": "2021-09-06T15:17:01+00:00" }, { "name": "colinmollenhour/credis", @@ -355,16 +355,16 @@ }, { "name": "composer/package-versions-deprecated", - "version": "1.11.99.3", + "version": "1.11.99.4", "source": { "type": "git", "url": "https://github.com/composer/package-versions-deprecated.git", - "reference": "fff576ac850c045158a250e7e27666e146e78d18" + "reference": "b174585d1fe49ceed21928a945138948cb394600" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/composer/package-versions-deprecated/zipball/fff576ac850c045158a250e7e27666e146e78d18", - "reference": "fff576ac850c045158a250e7e27666e146e78d18", + "url": "https://api.github.com/repos/composer/package-versions-deprecated/zipball/b174585d1fe49ceed21928a945138948cb394600", + "reference": "b174585d1fe49ceed21928a945138948cb394600", "shasum": "" }, "require": { @@ -408,7 +408,7 @@ "description": "Composer plugin that provides efficient querying for installed package versions (no runtime IO)", "support": { "issues": "https://github.com/composer/package-versions-deprecated/issues", - "source": "https://github.com/composer/package-versions-deprecated/tree/1.11.99.3" + "source": "https://github.com/composer/package-versions-deprecated/tree/1.11.99.4" }, "funding": [ { @@ -424,7 +424,7 @@ "type": "tidelift" } ], - "time": "2021-08-17T13:49:14+00:00" + "time": "2021-09-13T08:41:34+00:00" }, { "name": "dragonmantank/cron-expression", @@ -1984,11 +1984,11 @@ }, { "name": "utopia-php/database", - "version": "dev-feat-check-attribute-method", + "version": "dev-feat-get-limit-methods-on-database", "source": { "type": "git", "url": "https://github.com/utopia-php/database", - "reference": "2d1f48345f1284876f6847599c66eee145b7233e" + "reference": "4cc9d5eb9c6be1a9116a4d9dc2e6bd9db4e8e9c0" }, "require": { "ext-mongodb": "*", @@ -2037,7 +2037,7 @@ "upf", "utopia" ], - "time": "2021-09-01T00:50:12+00:00" + "time": "2021-09-17T01:44:11+00:00" }, { "name": "utopia-php/domains", @@ -3761,33 +3761,33 @@ }, { "name": "phpspec/prophecy", - "version": "1.13.0", + "version": "1.14.0", "source": { "type": "git", "url": "https://github.com/phpspec/prophecy.git", - "reference": "be1996ed8adc35c3fd795488a653f4b518be70ea" + "reference": "d86dfc2e2a3cd366cee475e52c6bb3bbc371aa0e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpspec/prophecy/zipball/be1996ed8adc35c3fd795488a653f4b518be70ea", - "reference": "be1996ed8adc35c3fd795488a653f4b518be70ea", + "url": "https://api.github.com/repos/phpspec/prophecy/zipball/d86dfc2e2a3cd366cee475e52c6bb3bbc371aa0e", + "reference": "d86dfc2e2a3cd366cee475e52c6bb3bbc371aa0e", "shasum": "" }, "require": { "doctrine/instantiator": "^1.2", - "php": "^7.2 || ~8.0, <8.1", + "php": "^7.2 || ~8.0, <8.2", "phpdocumentor/reflection-docblock": "^5.2", "sebastian/comparator": "^3.0 || ^4.0", "sebastian/recursion-context": "^3.0 || ^4.0" }, "require-dev": { - "phpspec/phpspec": "^6.0", + "phpspec/phpspec": "^6.0 || ^7.0", "phpunit/phpunit": "^8.0 || ^9.0" }, "type": "library", "extra": { "branch-alias": { - "dev-master": "1.11.x-dev" + "dev-master": "1.x-dev" } }, "autoload": { @@ -3822,9 +3822,9 @@ ], "support": { "issues": "https://github.com/phpspec/prophecy/issues", - "source": "https://github.com/phpspec/prophecy/tree/1.13.0" + "source": "https://github.com/phpspec/prophecy/tree/1.14.0" }, - "time": "2021-03-17T13:42:18+00:00" + "time": "2021-09-10T09:02:12+00:00" }, { "name": "phpunit/php-code-coverage", @@ -6251,7 +6251,7 @@ "aliases": [ { "package": "utopia-php/database", - "version": "dev-feat-check-attribute-method", + "version": "dev-feat-get-limit-methods-on-database", "alias": "0.10.0", "alias_normalized": "0.10.0.0" } From cb7c810207c79eddf8daac344eadc9b8ed4092e4 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Sun, 19 Sep 2021 11:49:24 -0400 Subject: [PATCH 159/206] Drop collection table in deletes worker --- app/controllers/api/database.php | 2 -- app/workers/deletes.php | 3 ++- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index dcef7105ea..26d93ae5be 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -620,8 +620,6 @@ App::delete('/v1/database/collections/:collectionId') throw new Exception('Collection not found', 404); } - $dbForExternal->deleteCollection($collectionId); // TDOD move to DB worker - if (!$dbForInternal->deleteDocument('collections', $collectionId)) { throw new Exception('Failed to remove collection from DB', 500); } diff --git a/app/workers/deletes.php b/app/workers/deletes.php index f557a3a549..8382e14b14 100644 --- a/app/workers/deletes.php +++ b/app/workers/deletes.php @@ -104,6 +104,7 @@ class DeletesV1 extends Worker $collectionId = $document->getId(); $dbForInternal = $this->getInternalDB($projectId); + $dbForExternal = $this->getExternalDB($projectId); $this->deleteByGroup('attributes', [ new Query('collectionId', Query::TYPE_EQUAL, [$collectionId]) @@ -113,7 +114,7 @@ class DeletesV1 extends Worker new Query('collectionId', Query::TYPE_EQUAL, [$collectionId]) ], $dbForInternal); - $dbForInternal->deleteDocument('collections', $collectionId); + $dbForExternal->deleteCollection($collectionId); } /** From c9601160a6089aaa25ac7438fb142d1e1d48cda3 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Mon, 20 Sep 2021 21:22:08 -0400 Subject: [PATCH 160/206] Handle case of duplicate indexes on deleteAttribute --- app/workers/database.php | 45 +++++++++++++++++++++++++++++++++------- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/app/workers/database.php b/app/workers/database.php index 6cdcead7be..0ca7aa45f3 100644 --- a/app/workers/database.php +++ b/app/workers/database.php @@ -120,24 +120,53 @@ class DatabaseV1 extends Worker $dbForInternal->updateDocument('attributes', $attribute->getId(), $attribute->setAttribute('status', 'failed')); } - // the underlying database removes/rebuilds indexes when attribute is removed - // update indexes table with changes + // The underlying database removes/rebuilds indexes when attribute is removed + // Update indexes table with changes /** @var Document[] $indexes */ $indexes = $collection->getAttribute('indexes', []); foreach ($indexes as $index) { /** @var string[] $attributes */ - $attributes = $index->getAttribute('attributes'); - $found = array_search($key, $attributes); + $attributes = $index->getAttribute('attributes'); + $lengths = $index->getAttribute('lengths'); + $orders = $index->getAttribute('orders'); + + $found = \array_search($key, $attributes); if ($found !== false) { - $remove = [$attributes[$found]]; - $attributes = array_diff($attributes, $remove); // remove attribute from array + // If found, remove entry from attributes, lengths, and orders + // array_values wraps array_diff to reindex array keys + // when found attribute is removed from array + $attributes = \array_values(\array_diff($attributes, [$attributes[$found]])); + $lengths = \array_values(\array_diff($lengths, [$lengths[$found]])); + $orders = \array_values(\array_diff($orders, [$orders[$found]])); if (empty($attributes)) { $dbForInternal->deleteDocument('indexes', $index->getId()); } else { - $dbForInternal->updateDocument('indexes', $index->getId(), $index->setAttribute('attributes', $attributes, Document::SET_TYPE_ASSIGN)); + $index + ->setAttribute('attributes', $attributes, Document::SET_TYPE_ASSIGN) + ->setAttribute('lengths', $lengths, Document::SET_TYPE_ASSIGN) + ->setAttribute('orders', $orders, Document::SET_TYPE_ASSIGN) + ; + + // Check if an index exists with the same attributes and orders + $exists = false; + foreach ($indexes as $existing) { + if ($existing->getAttribute('key') !== $index->getAttribute('key') // Ignore itself + && $existing->getAttribute('attributes') === $index->getAttribute('attributes') + && $existing->getAttribute('orders') === $index->getAttribute('orders') + ) { + $exists = true; + break; + } + } + + if ($exists) { // Delete the duplicate if created, else update in db + $this->deleteIndex($collection, $index, $projectId); + } else { + $dbForInternal->updateDocument('indexes', $index->getId(), $index); + } } } } @@ -190,7 +219,7 @@ class DatabaseV1 extends Worker try { if(!$dbForExternal->deleteIndex($collectionId, $key)) { - throw new Exception('Failed to delete Attribute'); + throw new Exception('Failed to delete index'); } $dbForInternal->deleteDocument('indexes', $index->getId()); From fdd43837c16bfe50a5f9e28ec75a23b001960262 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Mon, 20 Sep 2021 21:23:41 -0400 Subject: [PATCH 161/206] Test duplicate index is removed in edge case of deleteAttribute --- .../Database/DatabaseCustomServerTest.php | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/tests/e2e/Services/Database/DatabaseCustomServerTest.php b/tests/e2e/Services/Database/DatabaseCustomServerTest.php index 2fe0f97315..fd4a12b5a1 100644 --- a/tests/e2e/Services/Database/DatabaseCustomServerTest.php +++ b/tests/e2e/Services/Database/DatabaseCustomServerTest.php @@ -298,6 +298,7 @@ class DatabaseCustomServerTest extends Scope sleep(2); + // Expected behavior: deleting attribute2 will cause index2 to be dropped, and index1 rebuilt with a single key $deleted = $this->client->call(Client::METHOD_DELETE, '/database/collections/' . $data['collectionId'] . '/attributes/'. $attribute2['body']['key'], array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], @@ -335,6 +336,104 @@ class DatabaseCustomServerTest extends Scope return $data; } + /** + * @depends testDeleteIndex + */ + public function testCleanupDuplicateIndexOnDeleteAttribute($data) + { + $attribute1 = $this->client->call(Client::METHOD_POST, '/database/collections/' . $data['collectionId'] . '/attributes/string', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'attributeId' => 'attribute1', + 'size' => 16, + 'required' => true, + ]); + + $attribute2 = $this->client->call(Client::METHOD_POST, '/database/collections/' . $data['collectionId'] . '/attributes/string', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'attributeId' => 'attribute2', + 'size' => 16, + 'required' => true, + ]); + + $this->assertEquals(201, $attribute1['headers']['status-code']); + $this->assertEquals(201, $attribute2['headers']['status-code']); + $this->assertEquals('attribute1', $attribute1['body']['key']); + $this->assertEquals('attribute2', $attribute2['body']['key']); + + sleep(2); + + $index1 = $this->client->call(Client::METHOD_POST, '/database/collections/' . $data['collectionId'] . '/indexes', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'indexId' => 'index1', + 'type' => 'key', + 'attributes' => ['attribute1', 'attribute2'], + 'orders' => ['ASC', 'ASC'], + ]); + + $index2 = $this->client->call(Client::METHOD_POST, '/database/collections/' . $data['collectionId'] . '/indexes', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'indexId' => 'index2', + 'type' => 'key', + 'attributes' => ['attribute2'], + ]); + + $this->assertEquals(201, $index1['headers']['status-code']); + $this->assertEquals(201, $index2['headers']['status-code']); + $this->assertEquals('index1', $index1['body']['key']); + $this->assertEquals('index2', $index2['body']['key']); + + sleep(2); + + // Expected behavior: deleting attribute1 would cause index1 to be a duplicate of index2 and automatically removed + $deleted = $this->client->call(Client::METHOD_DELETE, '/database/collections/' . $data['collectionId'] . '/attributes/'. $attribute1['body']['key'], array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ])); + + $this->assertEquals($deleted['headers']['status-code'], 204); + + // wait for database worker to complete + sleep(2); + + $collection = $this->client->call(Client::METHOD_GET, '/database/collections/' . $data['collectionId'], array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ])); + + $this->assertEquals(200, $collection['headers']['status-code']); + $this->assertIsArray($collection['body']['indexes']); + $this->assertCount(1, $collection['body']['indexes']); + $this->assertEquals($index2['body']['key'], $collection['body']['indexes'][0]['key']); + $this->assertIsArray($collection['body']['indexes'][0]['attributes']); + $this->assertCount(1, $collection['body']['indexes'][0]['attributes']); + $this->assertEquals($attribute2['body']['key'], $collection['body']['indexes'][0]['attributes'][0]); + + // Delete attribute + $deleted = $this->client->call(Client::METHOD_DELETE, '/database/collections/' . $data['collectionId'] . '/attributes/' . $attribute2['body']['key'], array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ])); + + $this->assertEquals($deleted['headers']['status-code'], 204); + + return $data; + } + /** * @depends testDeleteIndexOnDeleteAttribute */ From 732c59a03de82ccebe5f4cb1b3a785cd8345abc7 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Tue, 21 Sep 2021 10:22:13 +0200 Subject: [PATCH 162/206] Implemented search tests --- app/controllers/api/users.php | 4 +- composer.lock | 147 +++++++++--------- .../Functions/FunctionsCustomClientTest.php | 1 + .../Functions/FunctionsCustomServerTest.php | 74 +++++++++ .../Projects/ProjectsConsoleClientTest.php | 23 +++ tests/e2e/Services/Storage/StorageBase.php | 38 ++++- tests/e2e/Services/Teams/TeamsBase.php | 14 ++ tests/e2e/Services/Users/UsersBase.php | 80 +++++++++- 8 files changed, 294 insertions(+), 87 deletions(-) diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 4ca682b92a..e4008e545e 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -97,7 +97,7 @@ App::get('/v1/users') $afterUser = $dbForInternal->getDocument('users', $after); if ($afterUser->isEmpty()) { - throw new Exception('User for after not found', 400); + throw new Exception("User '{$after}' for the 'after' value not found.", 400); } } @@ -107,7 +107,7 @@ App::get('/v1/users') $queries[] = new Query('search', Query::TYPE_SEARCH, [$search]); } - $results = $dbForInternal->find('users', $queries, $limit, $offset, ['_id'], [$orderType]); + $results = $dbForInternal->find('users', $queries, $limit, $offset, [], [$orderType], $afterUser ?? null); $sum = $dbForInternal->count('users', $queries, APP_LIMIT_COUNT); $response->dynamic(new Document([ diff --git a/composer.lock b/composer.lock index db07f5df2b..f1709a381a 100644 --- a/composer.lock +++ b/composer.lock @@ -248,16 +248,16 @@ }, { "name": "chillerlan/php-settings-container", - "version": "2.1.1", + "version": "2.1.2", "source": { "type": "git", "url": "https://github.com/chillerlan/php-settings-container.git", - "reference": "98ccc1b31b31a53bcb563465c4961879b2b93096" + "reference": "ec834493a88682dd69652a1eeaf462789ed0c5f5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/chillerlan/php-settings-container/zipball/98ccc1b31b31a53bcb563465c4961879b2b93096", - "reference": "98ccc1b31b31a53bcb563465c4961879b2b93096", + "url": "https://api.github.com/repos/chillerlan/php-settings-container/zipball/ec834493a88682dd69652a1eeaf462789ed0c5f5", + "reference": "ec834493a88682dd69652a1eeaf462789ed0c5f5", "shasum": "" }, "require": { @@ -307,7 +307,7 @@ "type": "ko_fi" } ], - "time": "2021-01-06T15:57:03+00:00" + "time": "2021-09-06T15:17:01+00:00" }, { "name": "colinmollenhour/credis", @@ -355,16 +355,16 @@ }, { "name": "composer/package-versions-deprecated", - "version": "1.11.99.2", + "version": "1.11.99.4", "source": { "type": "git", "url": "https://github.com/composer/package-versions-deprecated.git", - "reference": "c6522afe5540d5fc46675043d3ed5a45a740b27c" + "reference": "b174585d1fe49ceed21928a945138948cb394600" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/composer/package-versions-deprecated/zipball/c6522afe5540d5fc46675043d3ed5a45a740b27c", - "reference": "c6522afe5540d5fc46675043d3ed5a45a740b27c", + "url": "https://api.github.com/repos/composer/package-versions-deprecated/zipball/b174585d1fe49ceed21928a945138948cb394600", + "reference": "b174585d1fe49ceed21928a945138948cb394600", "shasum": "" }, "require": { @@ -408,7 +408,7 @@ "description": "Composer plugin that provides efficient querying for installed package versions (no runtime IO)", "support": { "issues": "https://github.com/composer/package-versions-deprecated/issues", - "source": "https://github.com/composer/package-versions-deprecated/tree/1.11.99.2" + "source": "https://github.com/composer/package-versions-deprecated/tree/1.11.99.4" }, "funding": [ { @@ -424,7 +424,7 @@ "type": "tidelift" } ], - "time": "2021-05-24T07:46:03+00:00" + "time": "2021-09-13T08:41:34+00:00" }, { "name": "dragonmantank/cron-expression", @@ -1666,22 +1666,22 @@ }, { "name": "utopia-php/abuse", - "version": "0.6.2", + "version": "0.6.3", "source": { "type": "git", "url": "https://github.com/utopia-php/abuse.git", - "reference": "4cd9c16610f7398d2e1737663ef682fa721ae736" + "reference": "d63e928c2c50b367495a499a85ba9806ee274c5e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/abuse/zipball/4cd9c16610f7398d2e1737663ef682fa721ae736", - "reference": "4cd9c16610f7398d2e1737663ef682fa721ae736", + "url": "https://api.github.com/repos/utopia-php/abuse/zipball/d63e928c2c50b367495a499a85ba9806ee274c5e", + "reference": "d63e928c2c50b367495a499a85ba9806ee274c5e", "shasum": "" }, "require": { "ext-pdo": "*", "php": ">=7.4", - "utopia-php/database": "0.7.*" + "utopia-php/database": ">=0.6 <1.0" }, "require-dev": { "phpunit/phpunit": "^9.4", @@ -1713,9 +1713,9 @@ ], "support": { "issues": "https://github.com/utopia-php/abuse/issues", - "source": "https://github.com/utopia-php/abuse/tree/0.6.2" + "source": "https://github.com/utopia-php/abuse/tree/0.6.3" }, - "time": "2021-08-13T07:52:34+00:00" + "time": "2021-08-16T18:38:31+00:00" }, { "name": "utopia-php/analytics", @@ -1774,22 +1774,22 @@ }, { "name": "utopia-php/audit", - "version": "0.6.2", + "version": "0.6.3", "source": { "type": "git", "url": "https://github.com/utopia-php/audit.git", - "reference": "2ec39a53eb98a5f9d230550ad56c7c04de5d77df" + "reference": "d79b467fbc7d03e5e02f12cdeb08761507a60ca0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/audit/zipball/2ec39a53eb98a5f9d230550ad56c7c04de5d77df", - "reference": "2ec39a53eb98a5f9d230550ad56c7c04de5d77df", + "url": "https://api.github.com/repos/utopia-php/audit/zipball/d79b467fbc7d03e5e02f12cdeb08761507a60ca0", + "reference": "d79b467fbc7d03e5e02f12cdeb08761507a60ca0", "shasum": "" }, "require": { "ext-pdo": "*", "php": ">=7.4", - "utopia-php/database": "0.7.*" + "utopia-php/database": ">=0.6 <1.0" }, "require-dev": { "phpunit/phpunit": "^9.3", @@ -1821,9 +1821,9 @@ ], "support": { "issues": "https://github.com/utopia-php/audit/issues", - "source": "https://github.com/utopia-php/audit/tree/0.6.2" + "source": "https://github.com/utopia-php/audit/tree/0.6.3" }, - "time": "2021-08-13T08:05:20+00:00" + "time": "2021-08-16T18:49:55+00:00" }, { "name": "utopia-php/cache", @@ -3389,16 +3389,16 @@ }, { "name": "nikic/php-parser", - "version": "v4.12.0", + "version": "v4.13.0", "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "6608f01670c3cc5079e18c1dab1104e002579143" + "reference": "50953a2691a922aa1769461637869a0a2faa3f53" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/6608f01670c3cc5079e18c1dab1104e002579143", - "reference": "6608f01670c3cc5079e18c1dab1104e002579143", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/50953a2691a922aa1769461637869a0a2faa3f53", + "reference": "50953a2691a922aa1769461637869a0a2faa3f53", "shasum": "" }, "require": { @@ -3439,9 +3439,9 @@ ], "support": { "issues": "https://github.com/nikic/PHP-Parser/issues", - "source": "https://github.com/nikic/PHP-Parser/tree/v4.12.0" + "source": "https://github.com/nikic/PHP-Parser/tree/v4.13.0" }, - "time": "2021-07-21T10:44:31+00:00" + "time": "2021-09-20T12:20:58+00:00" }, { "name": "openlss/lib-array2xml", @@ -3718,16 +3718,16 @@ }, { "name": "phpdocumentor/type-resolver", - "version": "1.4.0", + "version": "1.5.0", "source": { "type": "git", "url": "https://github.com/phpDocumentor/TypeResolver.git", - "reference": "6a467b8989322d92aa1c8bf2bebcc6e5c2ba55c0" + "reference": "30f38bffc6f24293dadd1823936372dfa9e86e2f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/6a467b8989322d92aa1c8bf2bebcc6e5c2ba55c0", - "reference": "6a467b8989322d92aa1c8bf2bebcc6e5c2ba55c0", + "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/30f38bffc6f24293dadd1823936372dfa9e86e2f", + "reference": "30f38bffc6f24293dadd1823936372dfa9e86e2f", "shasum": "" }, "require": { @@ -3735,7 +3735,8 @@ "phpdocumentor/reflection-common": "^2.0" }, "require-dev": { - "ext-tokenizer": "*" + "ext-tokenizer": "*", + "psalm/phar": "^4.8" }, "type": "library", "extra": { @@ -3761,39 +3762,39 @@ "description": "A PSR-5 based resolver of Class names, Types and Structural Element Names", "support": { "issues": "https://github.com/phpDocumentor/TypeResolver/issues", - "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.4.0" + "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.5.0" }, - "time": "2020-09-17T18:55:26+00:00" + "time": "2021-09-17T15:28:14+00:00" }, { "name": "phpspec/prophecy", - "version": "1.13.0", + "version": "1.14.0", "source": { "type": "git", "url": "https://github.com/phpspec/prophecy.git", - "reference": "be1996ed8adc35c3fd795488a653f4b518be70ea" + "reference": "d86dfc2e2a3cd366cee475e52c6bb3bbc371aa0e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpspec/prophecy/zipball/be1996ed8adc35c3fd795488a653f4b518be70ea", - "reference": "be1996ed8adc35c3fd795488a653f4b518be70ea", + "url": "https://api.github.com/repos/phpspec/prophecy/zipball/d86dfc2e2a3cd366cee475e52c6bb3bbc371aa0e", + "reference": "d86dfc2e2a3cd366cee475e52c6bb3bbc371aa0e", "shasum": "" }, "require": { "doctrine/instantiator": "^1.2", - "php": "^7.2 || ~8.0, <8.1", + "php": "^7.2 || ~8.0, <8.2", "phpdocumentor/reflection-docblock": "^5.2", "sebastian/comparator": "^3.0 || ^4.0", "sebastian/recursion-context": "^3.0 || ^4.0" }, "require-dev": { - "phpspec/phpspec": "^6.0", + "phpspec/phpspec": "^6.0 || ^7.0", "phpunit/phpunit": "^8.0 || ^9.0" }, "type": "library", "extra": { "branch-alias": { - "dev-master": "1.11.x-dev" + "dev-master": "1.x-dev" } }, "autoload": { @@ -3828,29 +3829,29 @@ ], "support": { "issues": "https://github.com/phpspec/prophecy/issues", - "source": "https://github.com/phpspec/prophecy/tree/1.13.0" + "source": "https://github.com/phpspec/prophecy/tree/1.14.0" }, - "time": "2021-03-17T13:42:18+00:00" + "time": "2021-09-10T09:02:12+00:00" }, { "name": "phpunit/php-code-coverage", - "version": "9.2.6", + "version": "9.2.7", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "f6293e1b30a2354e8428e004689671b83871edde" + "reference": "d4c798ed8d51506800b441f7a13ecb0f76f12218" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/f6293e1b30a2354e8428e004689671b83871edde", - "reference": "f6293e1b30a2354e8428e004689671b83871edde", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/d4c798ed8d51506800b441f7a13ecb0f76f12218", + "reference": "d4c798ed8d51506800b441f7a13ecb0f76f12218", "shasum": "" }, "require": { "ext-dom": "*", "ext-libxml": "*", "ext-xmlwriter": "*", - "nikic/php-parser": "^4.10.2", + "nikic/php-parser": "^4.12.0", "php": ">=7.3", "phpunit/php-file-iterator": "^3.0.3", "phpunit/php-text-template": "^2.0.2", @@ -3899,7 +3900,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/php-code-coverage/issues", - "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.6" + "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.7" }, "funding": [ { @@ -3907,7 +3908,7 @@ "type": "github" } ], - "time": "2021-03-28T07:26:59+00:00" + "time": "2021-09-17T05:39:03+00:00" }, { "name": "phpunit/php-file-iterator", @@ -5319,16 +5320,16 @@ }, { "name": "symfony/console", - "version": "v5.3.6", + "version": "v5.3.7", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "51b71afd6d2dc8f5063199357b9880cea8d8bfe2" + "reference": "8b1008344647462ae6ec57559da166c2bfa5e16a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/51b71afd6d2dc8f5063199357b9880cea8d8bfe2", - "reference": "51b71afd6d2dc8f5063199357b9880cea8d8bfe2", + "url": "https://api.github.com/repos/symfony/console/zipball/8b1008344647462ae6ec57559da166c2bfa5e16a", + "reference": "8b1008344647462ae6ec57559da166c2bfa5e16a", "shasum": "" }, "require": { @@ -5398,7 +5399,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v5.3.6" + "source": "https://github.com/symfony/console/tree/v5.3.7" }, "funding": [ { @@ -5414,7 +5415,7 @@ "type": "tidelift" } ], - "time": "2021-07-27T19:10:22+00:00" + "time": "2021-08-25T20:02:16+00:00" }, { "name": "symfony/deprecation-contracts", @@ -5888,16 +5889,16 @@ }, { "name": "symfony/string", - "version": "v5.3.3", + "version": "v5.3.7", "source": { "type": "git", "url": "https://github.com/symfony/string.git", - "reference": "bd53358e3eccec6a670b5f33ab680d8dbe1d4ae1" + "reference": "8d224396e28d30f81969f083a58763b8b9ceb0a5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/string/zipball/bd53358e3eccec6a670b5f33ab680d8dbe1d4ae1", - "reference": "bd53358e3eccec6a670b5f33ab680d8dbe1d4ae1", + "url": "https://api.github.com/repos/symfony/string/zipball/8d224396e28d30f81969f083a58763b8b9ceb0a5", + "reference": "8d224396e28d30f81969f083a58763b8b9ceb0a5", "shasum": "" }, "require": { @@ -5951,7 +5952,7 @@ "utf8" ], "support": { - "source": "https://github.com/symfony/string/tree/v5.3.3" + "source": "https://github.com/symfony/string/tree/v5.3.7" }, "funding": [ { @@ -5967,7 +5968,7 @@ "type": "tidelift" } ], - "time": "2021-06-27T11:44:38+00:00" + "time": "2021-08-26T08:00:08+00:00" }, { "name": "theseer/tokenizer", @@ -6021,16 +6022,16 @@ }, { "name": "twig/twig", - "version": "v2.14.6", + "version": "v2.14.7", "source": { "type": "git", "url": "https://github.com/twigphp/Twig.git", - "reference": "27e5cf2b05e3744accf39d4c68a3235d9966d260" + "reference": "8e202327ee1ed863629de9b18a5ec70ac614d88f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/twigphp/Twig/zipball/27e5cf2b05e3744accf39d4c68a3235d9966d260", - "reference": "27e5cf2b05e3744accf39d4c68a3235d9966d260", + "url": "https://api.github.com/repos/twigphp/Twig/zipball/8e202327ee1ed863629de9b18a5ec70ac614d88f", + "reference": "8e202327ee1ed863629de9b18a5ec70ac614d88f", "shasum": "" }, "require": { @@ -6040,7 +6041,7 @@ }, "require-dev": { "psr/container": "^1.0", - "symfony/phpunit-bridge": "^4.4.9|^5.0.9" + "symfony/phpunit-bridge": "^4.4.9|^5.0.9|^6.0" }, "type": "library", "extra": { @@ -6084,7 +6085,7 @@ ], "support": { "issues": "https://github.com/twigphp/Twig/issues", - "source": "https://github.com/twigphp/Twig/tree/v2.14.6" + "source": "https://github.com/twigphp/Twig/tree/v2.14.7" }, "funding": [ { @@ -6096,7 +6097,7 @@ "type": "tidelift" } ], - "time": "2021-05-16T12:12:47+00:00" + "time": "2021-09-17T08:39:54+00:00" }, { "name": "vimeo/psalm", diff --git a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php index 86f6fde482..0d229aafce 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php @@ -7,6 +7,7 @@ use Tests\E2E\Client; use Tests\E2E\Scopes\ProjectCustom; use Tests\E2E\Scopes\Scope; use Tests\E2E\Scopes\SideClient; +use function var_dump; class FunctionsCustomClientTest extends Scope { diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index cb820b45c3..e118df8a5c 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -7,6 +7,8 @@ use Tests\E2E\Client; use Tests\E2E\Scopes\ProjectCustom; use Tests\E2E\Scopes\Scope; use Tests\E2E\Scopes\SideServer; +use function array_merge; +use function var_dump; class FunctionsCustomServerTest extends Scope { @@ -77,6 +79,39 @@ class FunctionsCustomServerTest extends Scope /** * Test for SUCCESS */ + $response = $this->client->call(Client::METHOD_GET, '/functions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'search' => $data['functionId'] + ]); + + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertCount(1, $response['body']['functions']); + $this->assertEquals($response['body']['functions'][0]['name'], 'Test'); + + $response = $this->client->call(Client::METHOD_GET, '/functions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'search' => 'Test' + ]); + + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertCount(1, $response['body']['functions']); + $this->assertEquals($response['body']['functions'][0]['$id'], $data['functionId']); + + $response = $this->client->call(Client::METHOD_GET, '/functions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'search' => 'php-8.0' + ]); + + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertCount(1, $response['body']['functions']); + $this->assertEquals($response['body']['functions'][0]['$id'], $data['functionId']); + $response = $this->client->call(Client::METHOD_POST, '/functions', array_merge([ 'content-type' => 'application/json', @@ -283,6 +318,45 @@ class FunctionsCustomServerTest extends Scope $this->assertIsArray($function['body']['tags']); $this->assertCount(1, $function['body']['tags']); + $function = $this->client->call(Client::METHOD_GET, '/functions/'.$data['functionId'].'/tags', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders(), [ + 'search' => $data['functionId'] + ])); + + $this->assertEquals($function['headers']['status-code'], 200); + $this->assertEquals($function['body']['sum'], 1); + $this->assertIsArray($function['body']['tags']); + $this->assertCount(1, $function['body']['tags']); + $this->assertEquals($function['body']['tags'][0]['$id'], $data['tagId']); + + $function = $this->client->call(Client::METHOD_GET, '/functions/'.$data['functionId'].'/tags', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders(), [ + 'search' => 'Test' + ])); + + $this->assertEquals($function['headers']['status-code'], 200); + $this->assertEquals($function['body']['sum'], 1); + $this->assertIsArray($function['body']['tags']); + $this->assertCount(1, $function['body']['tags']); + $this->assertEquals($function['body']['tags'][0]['$id'], $data['tagId']); + + $function = $this->client->call(Client::METHOD_GET, '/functions/'.$data['functionId'].'/tags', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders(), [ + 'search' => 'php-8.0' + ])); + + $this->assertEquals($function['headers']['status-code'], 200); + $this->assertEquals($function['body']['sum'], 1); + $this->assertIsArray($function['body']['tags']); + $this->assertCount(1, $function['body']['tags']); + $this->assertEquals($function['body']['tags'][0]['$id'], $data['tagId']); + return $data; } diff --git a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php index 4d83efb416..6c0813cc21 100644 --- a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php +++ b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php @@ -7,6 +7,7 @@ use Tests\E2E\Scopes\ProjectConsole; use Tests\E2E\Scopes\SideClient; use Tests\E2E\Services\Projects\ProjectsBase; use Tests\E2E\Client; +use function array_merge; class ProjectsConsoleClientTest extends Scope { @@ -97,6 +98,28 @@ class ProjectsConsoleClientTest extends Scope $this->assertEquals($id, $response['body']['projects'][0]['$id']); $this->assertEquals('Project Test', $response['body']['projects'][0]['name']); + $response = $this->client->call(Client::METHOD_GET, '/projects', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders(), [ + 'search' => $id + ])); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertNotEmpty($response['body']); + $this->assertEquals('Project Test', $response['body']['projects'][0]['name']); + + $response = $this->client->call(Client::METHOD_GET, '/projects', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders(), [ + 'search' => 'Project Test' + ])); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertNotEmpty($response['body']); + $this->assertEquals($id, $response['body']['projects'][0]['$id']); + /** * Test after pagination */ diff --git a/tests/e2e/Services/Storage/StorageBase.php b/tests/e2e/Services/Storage/StorageBase.php index 57a5ee6972..fde1fc9fb0 100644 --- a/tests/e2e/Services/Storage/StorageBase.php +++ b/tests/e2e/Services/Storage/StorageBase.php @@ -5,6 +5,9 @@ namespace Tests\E2E\Services\Storage; use CURLFile; use Tests\E2E\Client; use Utopia\Image\Image; +use function array_merge; +use function realpath; +use function var_dump; trait StorageBase { @@ -163,7 +166,7 @@ trait StorageBase 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ 'fileId' => 'unique()', - 'file' => new CURLFile(realpath(__DIR__ . '/../../../resources/logo.png'), 'image/png', 'logo.png'), + 'file' => new CURLFile(realpath(__DIR__ . '/../../../resources/file.png'), 'image/png', 'file.png'), 'read' => ['role:all'], 'write' => ['role:all'], ]); @@ -171,9 +174,9 @@ trait StorageBase $this->assertEquals($file['headers']['status-code'], 201); $this->assertNotEmpty($file['body']['$id']); $this->assertIsInt($file['body']['dateCreated']); - $this->assertEquals('logo.png', $file['body']['name']); - $this->assertEquals('image/png', $file['body']['mimeType']); - $this->assertEquals(47218, $file['body']['sizeOriginal']); + $this->assertEquals('file.png', $file['body']['name']); + $this->assertEquals('image/jpeg', $file['body']['mimeType']); + $this->assertEquals(16804, $file['body']['sizeOriginal']); $files = $this->client->call(Client::METHOD_GET, '/storage/files', array_merge([ 'content-type' => 'application/json', @@ -197,6 +200,33 @@ trait StorageBase $this->assertEquals($files['body']['files'][1]['$id'], $response['body']['files'][0]['$id']); $this->assertCount(1, $response['body']['files']); + $response = $this->client->call(Client::METHOD_GET, '/storage/files', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'search' => $data['fileId'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertGreaterThan(0, $response['body']['sum']); + $this->assertIsInt($response['body']['sum']); + $this->assertCount(1, $response['body']['files']); + $this->assertEquals('logo.png', $response['body']['files'][0]['name']); + + + $response = $this->client->call(Client::METHOD_GET, '/storage/files', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'search' => 'logo', + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertGreaterThan(0, $response['body']['sum']); + $this->assertIsInt($response['body']['sum']); + $this->assertGreaterThan(0, $response['body']['files']); + $this->assertEquals($data['fileId'], $response['body']['files'][0]['$id']); + /** * Test for FAILURE */ diff --git a/tests/e2e/Services/Teams/TeamsBase.php b/tests/e2e/Services/Teams/TeamsBase.php index 6c1ebbe36a..fd1b064a72 100644 --- a/tests/e2e/Services/Teams/TeamsBase.php +++ b/tests/e2e/Services/Teams/TeamsBase.php @@ -3,6 +3,7 @@ namespace Tests\E2E\Services\Teams; use Tests\E2E\Client; +use function array_merge; trait TeamsBase { @@ -172,6 +173,19 @@ trait TeamsBase $this->assertCount(1, $response['body']['teams']); $this->assertEquals('Manchester United', $response['body']['teams'][0]['name']); + $response = $this->client->call(Client::METHOD_GET, '/teams', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'search' => $data['teamUid'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertGreaterThan(0, $response['body']['sum']); + $this->assertIsInt($response['body']['sum']); + $this->assertCount(1, $response['body']['teams']); + $this->assertEquals('Arsenal', $response['body']['teams'][0]['name']); + $teams = $this->client->call(Client::METHOD_GET, '/teams', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], diff --git a/tests/e2e/Services/Users/UsersBase.php b/tests/e2e/Services/Users/UsersBase.php index b35f355c90..e85e2a44a6 100644 --- a/tests/e2e/Services/Users/UsersBase.php +++ b/tests/e2e/Services/Users/UsersBase.php @@ -3,6 +3,8 @@ namespace Tests\E2E\Services\Users; use Tests\E2E\Client; +use function array_merge; +use function var_dump; trait UsersBase { @@ -16,14 +18,14 @@ trait UsersBase 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ 'userId' => 'unique()', - 'email' => 'users.service@example.com', + 'email' => 'first.user@example.com', 'password' => 'password', - 'name' => 'Project User', + 'name' => 'First Name', ]); $this->assertEquals($user['headers']['status-code'], 201); - $this->assertEquals($user['body']['name'], 'Project User'); - $this->assertEquals($user['body']['email'], 'users.service@example.com'); + $this->assertEquals($user['body']['name'], 'First Name'); + $this->assertEquals($user['body']['email'], 'first.user@example.com'); $this->assertEquals($user['body']['status'], true); $this->assertGreaterThan(0, $user['body']['registration']); @@ -56,7 +58,7 @@ trait UsersBase public function testListUsers(array $data): void { /** - * Test for SUCCESS + * Test for SUCCESS listUsers */ $response = $this->client->call(Client::METHOD_GET, '/users', array_merge([ 'content-type' => 'application/json', @@ -82,8 +84,70 @@ trait UsersBase $this->assertNotEmpty($response['body']); $this->assertNotEmpty($response['body']['users']); $this->assertCount(1, $response['body']['users']); - $this->assertEquals($response['body']['users'][0]['$id'], 'user1'); + + /** + * Test for SUCCESS searchUsers + */ + $response = $this->client->call(Client::METHOD_GET, '/users', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'search' => 'First Name' + ]); + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertNotEmpty($response['body']); + $this->assertNotEmpty($response['body']['users']); + $this->assertGreaterThan(0, $response['body']['users']); + $this->assertEquals($response['body']['users'][0]['$id'], $data['userId']); + + $response = $this->client->call(Client::METHOD_GET, '/users', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'search' => 'first.user' + ]); + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertNotEmpty($response['body']); + $this->assertNotEmpty($response['body']['users']); + $this->assertGreaterThan(0, $response['body']['users']); + $this->assertEquals($response['body']['users'][0]['$id'], $data['userId']); + + $response = $this->client->call(Client::METHOD_GET, '/users', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'search' => 'first user name' + ]); + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertNotEmpty($response['body']); + $this->assertNotEmpty($response['body']['users']); + $this->assertGreaterThan(0, $response['body']['users']); + $this->assertEquals($response['body']['users'][0]['$id'], $data['userId']); + + $response = $this->client->call(Client::METHOD_GET, '/users', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'search' => $data['userId'] + ]); + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertNotEmpty($response['body']); + $this->assertNotEmpty($response['body']['users']); + $this->assertGreaterThan(0, $response['body']['users']); + $this->assertEquals($response['body']['users'][0]['$id'], $data['userId']); + + $response = $this->client->call(Client::METHOD_GET, '/users', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'search' => 'first user - first.user@example.com ' . $data['userId'] + ]); + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertNotEmpty($response['body']); + $this->assertNotEmpty($response['body']['users']); + $this->assertGreaterThan(0, $response['body']['users']); + $this->assertEquals($response['body']['users'][0]['$id'], $data['userId']); } /** @@ -100,8 +164,8 @@ trait UsersBase ], $this->getHeaders())); $this->assertEquals($user['headers']['status-code'], 200); - $this->assertEquals($user['body']['name'], 'Project User'); - $this->assertEquals($user['body']['email'], 'users.service@example.com'); + $this->assertEquals($user['body']['name'], 'First Name'); + $this->assertEquals($user['body']['email'], 'first.user@example.com'); $this->assertEquals($user['body']['status'], true); $this->assertGreaterThan(0, $user['body']['registration']); From 9d1b1a17d951a4ee050dbf99ce740622f50f6050 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Tue, 21 Sep 2021 10:26:28 +0200 Subject: [PATCH 163/206] Removed PHPStorm-generated stuff --- tests/e2e/Services/Functions/FunctionsCustomClientTest.php | 1 - tests/e2e/Services/Functions/FunctionsCustomServerTest.php | 2 -- tests/e2e/Services/Storage/StorageBase.php | 1 - tests/e2e/Services/Users/UsersBase.php | 2 -- 4 files changed, 6 deletions(-) diff --git a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php index 0d229aafce..86f6fde482 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomClientTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomClientTest.php @@ -7,7 +7,6 @@ use Tests\E2E\Client; use Tests\E2E\Scopes\ProjectCustom; use Tests\E2E\Scopes\Scope; use Tests\E2E\Scopes\SideClient; -use function var_dump; class FunctionsCustomClientTest extends Scope { diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index e118df8a5c..62198293a7 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -7,8 +7,6 @@ use Tests\E2E\Client; use Tests\E2E\Scopes\ProjectCustom; use Tests\E2E\Scopes\Scope; use Tests\E2E\Scopes\SideServer; -use function array_merge; -use function var_dump; class FunctionsCustomServerTest extends Scope { diff --git a/tests/e2e/Services/Storage/StorageBase.php b/tests/e2e/Services/Storage/StorageBase.php index fde1fc9fb0..47a81444cb 100644 --- a/tests/e2e/Services/Storage/StorageBase.php +++ b/tests/e2e/Services/Storage/StorageBase.php @@ -7,7 +7,6 @@ use Tests\E2E\Client; use Utopia\Image\Image; use function array_merge; use function realpath; -use function var_dump; trait StorageBase { diff --git a/tests/e2e/Services/Users/UsersBase.php b/tests/e2e/Services/Users/UsersBase.php index e85e2a44a6..4bf344d95d 100644 --- a/tests/e2e/Services/Users/UsersBase.php +++ b/tests/e2e/Services/Users/UsersBase.php @@ -3,8 +3,6 @@ namespace Tests\E2E\Services\Users; use Tests\E2E\Client; -use function array_merge; -use function var_dump; trait UsersBase { From fce5e3e96c54488a2250c8c801ba35cd1e1cf7ed Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Tue, 21 Sep 2021 10:27:31 +0200 Subject: [PATCH 164/206] Removed PHPStorm-generated stuff 2 --- tests/e2e/Services/Projects/ProjectsConsoleClientTest.php | 1 - tests/e2e/Services/Storage/StorageBase.php | 2 -- tests/e2e/Services/Teams/TeamsBase.php | 1 - 3 files changed, 4 deletions(-) diff --git a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php index 6c0813cc21..9edee076af 100644 --- a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php +++ b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php @@ -7,7 +7,6 @@ use Tests\E2E\Scopes\ProjectConsole; use Tests\E2E\Scopes\SideClient; use Tests\E2E\Services\Projects\ProjectsBase; use Tests\E2E\Client; -use function array_merge; class ProjectsConsoleClientTest extends Scope { diff --git a/tests/e2e/Services/Storage/StorageBase.php b/tests/e2e/Services/Storage/StorageBase.php index 47a81444cb..8d9b2d9c8c 100644 --- a/tests/e2e/Services/Storage/StorageBase.php +++ b/tests/e2e/Services/Storage/StorageBase.php @@ -5,8 +5,6 @@ namespace Tests\E2E\Services\Storage; use CURLFile; use Tests\E2E\Client; use Utopia\Image\Image; -use function array_merge; -use function realpath; trait StorageBase { diff --git a/tests/e2e/Services/Teams/TeamsBase.php b/tests/e2e/Services/Teams/TeamsBase.php index fd1b064a72..b594bbfa67 100644 --- a/tests/e2e/Services/Teams/TeamsBase.php +++ b/tests/e2e/Services/Teams/TeamsBase.php @@ -3,7 +3,6 @@ namespace Tests\E2E\Services\Teams; use Tests\E2E\Client; -use function array_merge; trait TeamsBase { From 77793b4e01160922e0154a46d49b2260ea0ff020 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Tue, 21 Sep 2021 13:18:54 +0200 Subject: [PATCH 165/206] update instead of delete, adjust code to new attribute --- app/config/collections2.php | 11 +++ app/controllers/api/account.php | 23 ++++-- app/controllers/api/users.php | 47 ++++++++----- composer.lock | 119 ++++++++++++++++---------------- 4 files changed, 115 insertions(+), 85 deletions(-) diff --git a/app/config/collections2.php b/app/config/collections2.php index 9ba9635457..866d1cb1b6 100644 --- a/app/config/collections2.php +++ b/app/config/collections2.php @@ -695,6 +695,17 @@ $collections = [ 'array' => true, 'filters' => ['json'], ], + [ + '$id' => 'deleted', + 'type' => Database::VAR_BOOLEAN, + 'format' => '', + 'size' => 0, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], ], 'indexes' => [ [ diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index bc34f81dda..3ad04efe81 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -78,7 +78,9 @@ App::post('/v1/account') $limit = $project->getAttribute('auths', [])['limit'] ?? 0; if ($limit !== 0) { - $sum = $dbForInternal->count('users', [], APP_LIMIT_USERS); + $sum = $dbForInternal->count('users', [ + new Query('deleted', Query::TYPE_EQUAL, [false]), + ], APP_LIMIT_USERS); if ($sum >= $limit) { throw new Exception('Project registration is restricted. Contact your administrator for more information.', 501); @@ -105,6 +107,7 @@ App::post('/v1/account') 'sessions' => [], 'tokens' => [], 'memberships' => [], + 'deleted' => false ])); } catch (Duplicate $th) { throw new Exception('Account already exists', 409); @@ -165,7 +168,7 @@ App::post('/v1/account/sessions') $email = \strtolower($email); $protocol = $request->getProtocol(); - $profile = $dbForInternal->findOne('users', [new Query('email', Query::TYPE_EQUAL, [$email])]); // Get user by email address + $profile = $dbForInternal->findOne('users', [new Query('deleted', Query::TYPE_EQUAL, [false]), new Query('email', Query::TYPE_EQUAL, [$email])]); // Get user by email address if (!$profile || !Auth::passwordVerify($password, $profile->getAttribute('password'))) { $audits @@ -462,13 +465,13 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') $name = $oauth2->getUserName($accessToken); $email = $oauth2->getUserEmail($accessToken); - $user = $dbForInternal->findOne('users', [new Query('email', Query::TYPE_EQUAL, [$email])]); // Get user by email address + $user = $dbForInternal->findOne('users', [new Query('deleted', Query::TYPE_EQUAL, [false]), new Query('email', Query::TYPE_EQUAL, [$email])]); // Get user by email address if ($user === false || $user->isEmpty()) { // Last option -> create the user, generate random password $limit = $project->getAttribute('auths', [])['limit'] ?? 0; if ($limit !== 0) { - $sum = $dbForInternal->count('users', [], APP_LIMIT_COUNT); + $sum = $dbForInternal->count('users', [ new Query('deleted', Query::TYPE_EQUAL, [false]),], APP_LIMIT_COUNT); if ($sum >= $limit) { throw new Exception('Project registration is restricted. Contact your administrator for more information.', 501); @@ -495,6 +498,7 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect') 'sessions' => [], 'tokens' => [], 'memberships' => [], + 'deleted' => false ])); } catch (Duplicate $th) { throw new Exception('Account already exists', 409); @@ -639,7 +643,9 @@ App::post('/v1/account/sessions/anonymous') $limit = $project->getAttribute('auths', [])['limit'] ?? 0; if ($limit !== 0) { - $sum = $dbForInternal->count('users', [], APP_LIMIT_COUNT); + $sum = $dbForInternal->count('users', [ + new Query('deleted', Query::TYPE_EQUAL, [false]), + ], APP_LIMIT_COUNT); if ($sum >= $limit) { throw new Exception('Project registration is restricted. Contact your administrator for more information.', 501); @@ -665,6 +671,7 @@ App::post('/v1/account/sessions/anonymous') 'sessions' => [], 'tokens' => [], 'memberships' => [], + 'deleted' => false ])); Authorization::reset(); @@ -1221,6 +1228,8 @@ App::delete('/v1/account') $protocol = $request->getProtocol(); $user = $dbForInternal->updateDocument('users', $user->getId(), $user->setAttribute('status', false)); + // TODO Seems to be related to users.php/App::delete('/v1/users/:userId'). Can we share code between these two? Do todos below apply to users.php? + // TODO delete all tokens or only current session? // TODO delete all user data according to GDPR. Make sure everything is backed up and backups are deleted later /* @@ -1463,7 +1472,7 @@ App::post('/v1/account/recovery') $isAppUser = Auth::isAppUser(Authorization::$roles); $email = \strtolower($email); - $profile = $dbForInternal->findOne('users', [new Query('email', Query::TYPE_EQUAL, [$email])]); // Get user by email address + $profile = $dbForInternal->findOne('users', [new Query('deleted', Query::TYPE_EQUAL, [false]), new Query('email', Query::TYPE_EQUAL, [$email])]); // Get user by email address if (!$profile) { throw new Exception('User not found', 404); @@ -1566,7 +1575,7 @@ App::put('/v1/account/recovery') $profile = $dbForInternal->getDocument('users', $userId); - if ($profile->isEmpty()) { + if ($profile->isEmpty() || $profile->getAttribute('deleted')) { throw new Exception('User not found', 404); } diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index c6c8207e21..20266d1fda 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -66,6 +66,7 @@ App::post('/v1/users') 'sessions' => [], 'tokens' => [], 'memberships' => [], + 'deleted' => false ])); } catch (Duplicate $th) { throw new Exception('Account already exists', 409); @@ -106,13 +107,17 @@ App::get('/v1/users') if (!empty($after)) { $afterUser = $dbForInternal->getDocument('users', $after); - if ($afterUser->isEmpty()) { + if ($afterUser->isEmpty() || $afterUser->getAttribute('deleted')) { throw new Exception('User for after not found', 400); } } - $results = $dbForInternal->find('users', [], $limit, $offset, [], [$orderType], $afterUser ?? null); - $sum = $dbForInternal->count('users', [], APP_LIMIT_COUNT); + $results = $dbForInternal->find('users', [ + new Query('deleted', Query::TYPE_EQUAL, [false]), + ], $limit, $offset, [], [$orderType], $afterUser ?? null); + $sum = $dbForInternal->count('users', [ + new Query('deleted', Query::TYPE_EQUAL, [false]), + ], APP_LIMIT_COUNT); $usage ->setParam('users.read', 1) @@ -146,7 +151,7 @@ App::get('/v1/users/:userId') $user = $dbForInternal->getDocument('users', $userId); - if ($user->isEmpty()) { + if ($user->isEmpty() || $user->getAttribute('deleted')) { throw new Exception('User not found', 404); } @@ -178,7 +183,7 @@ App::get('/v1/users/:userId/prefs') $user = $dbForInternal->getDocument('users', $userId); - if ($user->isEmpty()) { + if ($user->isEmpty() || $user->getAttribute('deleted')) { throw new Exception('User not found', 404); } @@ -214,7 +219,7 @@ App::get('/v1/users/:userId/sessions') $user = $dbForInternal->getDocument('users', $userId); - if ($user->isEmpty()) { + if ($user->isEmpty() || $user->getAttribute('deleted')) { throw new Exception('User not found', 404); } @@ -266,7 +271,7 @@ App::get('/v1/users/:userId/logs') $user = $dbForInternal->getDocument('users', $userId); - if ($user->isEmpty()) { + if ($user->isEmpty() || $user->getAttribute('deleted')) { throw new Exception('User not found', 404); } @@ -374,7 +379,7 @@ App::patch('/v1/users/:userId/status') $user = $dbForInternal->getDocument('users', $userId); - if ($user->isEmpty()) { + if ($user->isEmpty() || $user->getAttribute('deleted')) { throw new Exception('User not found', 404); } @@ -410,7 +415,7 @@ App::patch('/v1/users/:userId/verification') $user = $dbForInternal->getDocument('users', $userId); - if ($user->isEmpty()) { + if ($user->isEmpty() || $user->getAttribute('deleted')) { throw new Exception('User not found', 404); } @@ -446,7 +451,7 @@ App::patch('/v1/users/:userId/name') $user = $dbForInternal->getDocument('users', $userId); - if ($user->isEmpty()) { + if ($user->isEmpty() || $user->getAttribute('deleted')) { throw new Exception('User not found', 404); } @@ -485,7 +490,7 @@ App::patch('/v1/users/:userId/password') $user = $dbForInternal->getDocument('users', $userId); - if ($user->isEmpty()) { + if ($user->isEmpty() || $user->getAttribute('deleted')) { throw new Exception('User not found', 404); } @@ -525,7 +530,7 @@ App::patch('/v1/users/:userId/email') $user = $dbForInternal->getDocument('users', $userId); - if ($user->isEmpty()) { + if ($user->isEmpty() || $user->getAttribute('deleted')) { throw new Exception('User not found', 404); } @@ -569,7 +574,7 @@ App::patch('/v1/users/:userId/prefs') $user = $dbForInternal->getDocument('users', $userId); - if ($user->isEmpty()) { + if ($user->isEmpty() || $user->getAttribute('deleted')) { throw new Exception('User not found', 404); } @@ -606,7 +611,7 @@ App::delete('/v1/users/:userId/sessions/:sessionId') $user = $dbForInternal->getDocument('users', $userId); - if ($user->isEmpty()) { + if ($user->isEmpty() || $user->getAttribute('deleted')) { throw new Exception('User not found', 404); } @@ -661,7 +666,7 @@ App::delete('/v1/users/:userId/sessions') $user = $dbForInternal->getDocument('users', $userId); - if ($user->isEmpty()) { + if ($user->isEmpty() || $user->getAttribute('deleted')) { throw new Exception('User not found', 404); } @@ -710,13 +715,17 @@ App::delete('/v1/users/:userId') $user = $dbForInternal->getDocument('users', $userId); - if ($user->isEmpty()) { + if ($user->isEmpty() || $user->getAttribute('deleted')) { throw new Exception('User not found', 404); } - if (!$dbForInternal->deleteDocument('users', $userId)) { - throw new Exception('Failed to remove user from DB', 500); - } + $emptyUser = clone $user; + $emptyUser->setAttribute("name", null); + $emptyUser->setAttribute("email", null); + $emptyUser->setAttribute("password", null); + $emptyUser->setAttribute("deleted", true); + + $dbForInternal->updateDocument('users', $userId, $emptyUser); $deletes ->setParam('type', DELETE_TYPE_DOCUMENT) diff --git a/composer.lock b/composer.lock index 8dac7bcc6d..c319daa5a3 100644 --- a/composer.lock +++ b/composer.lock @@ -248,16 +248,16 @@ }, { "name": "chillerlan/php-settings-container", - "version": "2.1.1", + "version": "2.1.2", "source": { "type": "git", "url": "https://github.com/chillerlan/php-settings-container.git", - "reference": "98ccc1b31b31a53bcb563465c4961879b2b93096" + "reference": "ec834493a88682dd69652a1eeaf462789ed0c5f5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/chillerlan/php-settings-container/zipball/98ccc1b31b31a53bcb563465c4961879b2b93096", - "reference": "98ccc1b31b31a53bcb563465c4961879b2b93096", + "url": "https://api.github.com/repos/chillerlan/php-settings-container/zipball/ec834493a88682dd69652a1eeaf462789ed0c5f5", + "reference": "ec834493a88682dd69652a1eeaf462789ed0c5f5", "shasum": "" }, "require": { @@ -307,7 +307,7 @@ "type": "ko_fi" } ], - "time": "2021-01-06T15:57:03+00:00" + "time": "2021-09-06T15:17:01+00:00" }, { "name": "colinmollenhour/credis", @@ -355,16 +355,16 @@ }, { "name": "composer/package-versions-deprecated", - "version": "1.11.99.3", + "version": "1.11.99.4", "source": { "type": "git", "url": "https://github.com/composer/package-versions-deprecated.git", - "reference": "fff576ac850c045158a250e7e27666e146e78d18" + "reference": "b174585d1fe49ceed21928a945138948cb394600" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/composer/package-versions-deprecated/zipball/fff576ac850c045158a250e7e27666e146e78d18", - "reference": "fff576ac850c045158a250e7e27666e146e78d18", + "url": "https://api.github.com/repos/composer/package-versions-deprecated/zipball/b174585d1fe49ceed21928a945138948cb394600", + "reference": "b174585d1fe49ceed21928a945138948cb394600", "shasum": "" }, "require": { @@ -408,7 +408,7 @@ "description": "Composer plugin that provides efficient querying for installed package versions (no runtime IO)", "support": { "issues": "https://github.com/composer/package-versions-deprecated/issues", - "source": "https://github.com/composer/package-versions-deprecated/tree/1.11.99.3" + "source": "https://github.com/composer/package-versions-deprecated/tree/1.11.99.4" }, "funding": [ { @@ -424,7 +424,7 @@ "type": "tidelift" } ], - "time": "2021-08-17T13:49:14+00:00" + "time": "2021-09-13T08:41:34+00:00" }, { "name": "dragonmantank/cron-expression", @@ -3383,16 +3383,16 @@ }, { "name": "nikic/php-parser", - "version": "v4.12.0", + "version": "v4.13.0", "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "6608f01670c3cc5079e18c1dab1104e002579143" + "reference": "50953a2691a922aa1769461637869a0a2faa3f53" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/6608f01670c3cc5079e18c1dab1104e002579143", - "reference": "6608f01670c3cc5079e18c1dab1104e002579143", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/50953a2691a922aa1769461637869a0a2faa3f53", + "reference": "50953a2691a922aa1769461637869a0a2faa3f53", "shasum": "" }, "require": { @@ -3433,9 +3433,9 @@ ], "support": { "issues": "https://github.com/nikic/PHP-Parser/issues", - "source": "https://github.com/nikic/PHP-Parser/tree/v4.12.0" + "source": "https://github.com/nikic/PHP-Parser/tree/v4.13.0" }, - "time": "2021-07-21T10:44:31+00:00" + "time": "2021-09-20T12:20:58+00:00" }, { "name": "openlss/lib-array2xml", @@ -3712,16 +3712,16 @@ }, { "name": "phpdocumentor/type-resolver", - "version": "1.4.0", + "version": "1.5.0", "source": { "type": "git", "url": "https://github.com/phpDocumentor/TypeResolver.git", - "reference": "6a467b8989322d92aa1c8bf2bebcc6e5c2ba55c0" + "reference": "30f38bffc6f24293dadd1823936372dfa9e86e2f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/6a467b8989322d92aa1c8bf2bebcc6e5c2ba55c0", - "reference": "6a467b8989322d92aa1c8bf2bebcc6e5c2ba55c0", + "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/30f38bffc6f24293dadd1823936372dfa9e86e2f", + "reference": "30f38bffc6f24293dadd1823936372dfa9e86e2f", "shasum": "" }, "require": { @@ -3729,7 +3729,8 @@ "phpdocumentor/reflection-common": "^2.0" }, "require-dev": { - "ext-tokenizer": "*" + "ext-tokenizer": "*", + "psalm/phar": "^4.8" }, "type": "library", "extra": { @@ -3755,39 +3756,39 @@ "description": "A PSR-5 based resolver of Class names, Types and Structural Element Names", "support": { "issues": "https://github.com/phpDocumentor/TypeResolver/issues", - "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.4.0" + "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.5.0" }, - "time": "2020-09-17T18:55:26+00:00" + "time": "2021-09-17T15:28:14+00:00" }, { "name": "phpspec/prophecy", - "version": "1.13.0", + "version": "1.14.0", "source": { "type": "git", "url": "https://github.com/phpspec/prophecy.git", - "reference": "be1996ed8adc35c3fd795488a653f4b518be70ea" + "reference": "d86dfc2e2a3cd366cee475e52c6bb3bbc371aa0e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpspec/prophecy/zipball/be1996ed8adc35c3fd795488a653f4b518be70ea", - "reference": "be1996ed8adc35c3fd795488a653f4b518be70ea", + "url": "https://api.github.com/repos/phpspec/prophecy/zipball/d86dfc2e2a3cd366cee475e52c6bb3bbc371aa0e", + "reference": "d86dfc2e2a3cd366cee475e52c6bb3bbc371aa0e", "shasum": "" }, "require": { "doctrine/instantiator": "^1.2", - "php": "^7.2 || ~8.0, <8.1", + "php": "^7.2 || ~8.0, <8.2", "phpdocumentor/reflection-docblock": "^5.2", "sebastian/comparator": "^3.0 || ^4.0", "sebastian/recursion-context": "^3.0 || ^4.0" }, "require-dev": { - "phpspec/phpspec": "^6.0", + "phpspec/phpspec": "^6.0 || ^7.0", "phpunit/phpunit": "^8.0 || ^9.0" }, "type": "library", "extra": { "branch-alias": { - "dev-master": "1.11.x-dev" + "dev-master": "1.x-dev" } }, "autoload": { @@ -3822,29 +3823,29 @@ ], "support": { "issues": "https://github.com/phpspec/prophecy/issues", - "source": "https://github.com/phpspec/prophecy/tree/1.13.0" + "source": "https://github.com/phpspec/prophecy/tree/1.14.0" }, - "time": "2021-03-17T13:42:18+00:00" + "time": "2021-09-10T09:02:12+00:00" }, { "name": "phpunit/php-code-coverage", - "version": "9.2.6", + "version": "9.2.7", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "f6293e1b30a2354e8428e004689671b83871edde" + "reference": "d4c798ed8d51506800b441f7a13ecb0f76f12218" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/f6293e1b30a2354e8428e004689671b83871edde", - "reference": "f6293e1b30a2354e8428e004689671b83871edde", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/d4c798ed8d51506800b441f7a13ecb0f76f12218", + "reference": "d4c798ed8d51506800b441f7a13ecb0f76f12218", "shasum": "" }, "require": { "ext-dom": "*", "ext-libxml": "*", "ext-xmlwriter": "*", - "nikic/php-parser": "^4.10.2", + "nikic/php-parser": "^4.12.0", "php": ">=7.3", "phpunit/php-file-iterator": "^3.0.3", "phpunit/php-text-template": "^2.0.2", @@ -3893,7 +3894,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/php-code-coverage/issues", - "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.6" + "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.7" }, "funding": [ { @@ -3901,7 +3902,7 @@ "type": "github" } ], - "time": "2021-03-28T07:26:59+00:00" + "time": "2021-09-17T05:39:03+00:00" }, { "name": "phpunit/php-file-iterator", @@ -5313,16 +5314,16 @@ }, { "name": "symfony/console", - "version": "v5.3.6", + "version": "v5.3.7", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "51b71afd6d2dc8f5063199357b9880cea8d8bfe2" + "reference": "8b1008344647462ae6ec57559da166c2bfa5e16a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/51b71afd6d2dc8f5063199357b9880cea8d8bfe2", - "reference": "51b71afd6d2dc8f5063199357b9880cea8d8bfe2", + "url": "https://api.github.com/repos/symfony/console/zipball/8b1008344647462ae6ec57559da166c2bfa5e16a", + "reference": "8b1008344647462ae6ec57559da166c2bfa5e16a", "shasum": "" }, "require": { @@ -5392,7 +5393,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v5.3.6" + "source": "https://github.com/symfony/console/tree/v5.3.7" }, "funding": [ { @@ -5408,7 +5409,7 @@ "type": "tidelift" } ], - "time": "2021-07-27T19:10:22+00:00" + "time": "2021-08-25T20:02:16+00:00" }, { "name": "symfony/deprecation-contracts", @@ -5882,16 +5883,16 @@ }, { "name": "symfony/string", - "version": "v5.3.3", + "version": "v5.3.7", "source": { "type": "git", "url": "https://github.com/symfony/string.git", - "reference": "bd53358e3eccec6a670b5f33ab680d8dbe1d4ae1" + "reference": "8d224396e28d30f81969f083a58763b8b9ceb0a5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/string/zipball/bd53358e3eccec6a670b5f33ab680d8dbe1d4ae1", - "reference": "bd53358e3eccec6a670b5f33ab680d8dbe1d4ae1", + "url": "https://api.github.com/repos/symfony/string/zipball/8d224396e28d30f81969f083a58763b8b9ceb0a5", + "reference": "8d224396e28d30f81969f083a58763b8b9ceb0a5", "shasum": "" }, "require": { @@ -5945,7 +5946,7 @@ "utf8" ], "support": { - "source": "https://github.com/symfony/string/tree/v5.3.3" + "source": "https://github.com/symfony/string/tree/v5.3.7" }, "funding": [ { @@ -5961,7 +5962,7 @@ "type": "tidelift" } ], - "time": "2021-06-27T11:44:38+00:00" + "time": "2021-08-26T08:00:08+00:00" }, { "name": "theseer/tokenizer", @@ -6015,16 +6016,16 @@ }, { "name": "twig/twig", - "version": "v2.14.6", + "version": "v2.14.7", "source": { "type": "git", "url": "https://github.com/twigphp/Twig.git", - "reference": "27e5cf2b05e3744accf39d4c68a3235d9966d260" + "reference": "8e202327ee1ed863629de9b18a5ec70ac614d88f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/twigphp/Twig/zipball/27e5cf2b05e3744accf39d4c68a3235d9966d260", - "reference": "27e5cf2b05e3744accf39d4c68a3235d9966d260", + "url": "https://api.github.com/repos/twigphp/Twig/zipball/8e202327ee1ed863629de9b18a5ec70ac614d88f", + "reference": "8e202327ee1ed863629de9b18a5ec70ac614d88f", "shasum": "" }, "require": { @@ -6034,7 +6035,7 @@ }, "require-dev": { "psr/container": "^1.0", - "symfony/phpunit-bridge": "^4.4.9|^5.0.9" + "symfony/phpunit-bridge": "^4.4.9|^5.0.9|^6.0" }, "type": "library", "extra": { @@ -6078,7 +6079,7 @@ ], "support": { "issues": "https://github.com/twigphp/Twig/issues", - "source": "https://github.com/twigphp/Twig/tree/v2.14.6" + "source": "https://github.com/twigphp/Twig/tree/v2.14.7" }, "funding": [ { @@ -6090,7 +6091,7 @@ "type": "tidelift" } ], - "time": "2021-05-16T12:12:47+00:00" + "time": "2021-09-17T08:39:54+00:00" }, { "name": "vimeo/psalm", From e8f77ae3633873f922fd7ae922f4c48c6d6897a8 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Tue, 21 Sep 2021 14:01:18 +0200 Subject: [PATCH 166/206] OpenAPI3 support for anyOf --- composer.lock | 119 +++++++++--------- .../Specification/Format/OpenAPI3.php | 28 ++++- 2 files changed, 84 insertions(+), 63 deletions(-) diff --git a/composer.lock b/composer.lock index c867efc33a..27c2af756b 100644 --- a/composer.lock +++ b/composer.lock @@ -248,16 +248,16 @@ }, { "name": "chillerlan/php-settings-container", - "version": "2.1.1", + "version": "2.1.2", "source": { "type": "git", "url": "https://github.com/chillerlan/php-settings-container.git", - "reference": "98ccc1b31b31a53bcb563465c4961879b2b93096" + "reference": "ec834493a88682dd69652a1eeaf462789ed0c5f5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/chillerlan/php-settings-container/zipball/98ccc1b31b31a53bcb563465c4961879b2b93096", - "reference": "98ccc1b31b31a53bcb563465c4961879b2b93096", + "url": "https://api.github.com/repos/chillerlan/php-settings-container/zipball/ec834493a88682dd69652a1eeaf462789ed0c5f5", + "reference": "ec834493a88682dd69652a1eeaf462789ed0c5f5", "shasum": "" }, "require": { @@ -307,7 +307,7 @@ "type": "ko_fi" } ], - "time": "2021-01-06T15:57:03+00:00" + "time": "2021-09-06T15:17:01+00:00" }, { "name": "colinmollenhour/credis", @@ -355,16 +355,16 @@ }, { "name": "composer/package-versions-deprecated", - "version": "1.11.99.3", + "version": "1.11.99.4", "source": { "type": "git", "url": "https://github.com/composer/package-versions-deprecated.git", - "reference": "fff576ac850c045158a250e7e27666e146e78d18" + "reference": "b174585d1fe49ceed21928a945138948cb394600" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/composer/package-versions-deprecated/zipball/fff576ac850c045158a250e7e27666e146e78d18", - "reference": "fff576ac850c045158a250e7e27666e146e78d18", + "url": "https://api.github.com/repos/composer/package-versions-deprecated/zipball/b174585d1fe49ceed21928a945138948cb394600", + "reference": "b174585d1fe49ceed21928a945138948cb394600", "shasum": "" }, "require": { @@ -408,7 +408,7 @@ "description": "Composer plugin that provides efficient querying for installed package versions (no runtime IO)", "support": { "issues": "https://github.com/composer/package-versions-deprecated/issues", - "source": "https://github.com/composer/package-versions-deprecated/tree/1.11.99.3" + "source": "https://github.com/composer/package-versions-deprecated/tree/1.11.99.4" }, "funding": [ { @@ -424,7 +424,7 @@ "type": "tidelift" } ], - "time": "2021-08-17T13:49:14+00:00" + "time": "2021-09-13T08:41:34+00:00" }, { "name": "dragonmantank/cron-expression", @@ -3383,16 +3383,16 @@ }, { "name": "nikic/php-parser", - "version": "v4.12.0", + "version": "v4.13.0", "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "6608f01670c3cc5079e18c1dab1104e002579143" + "reference": "50953a2691a922aa1769461637869a0a2faa3f53" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/6608f01670c3cc5079e18c1dab1104e002579143", - "reference": "6608f01670c3cc5079e18c1dab1104e002579143", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/50953a2691a922aa1769461637869a0a2faa3f53", + "reference": "50953a2691a922aa1769461637869a0a2faa3f53", "shasum": "" }, "require": { @@ -3433,9 +3433,9 @@ ], "support": { "issues": "https://github.com/nikic/PHP-Parser/issues", - "source": "https://github.com/nikic/PHP-Parser/tree/v4.12.0" + "source": "https://github.com/nikic/PHP-Parser/tree/v4.13.0" }, - "time": "2021-07-21T10:44:31+00:00" + "time": "2021-09-20T12:20:58+00:00" }, { "name": "openlss/lib-array2xml", @@ -3712,16 +3712,16 @@ }, { "name": "phpdocumentor/type-resolver", - "version": "1.4.0", + "version": "1.5.0", "source": { "type": "git", "url": "https://github.com/phpDocumentor/TypeResolver.git", - "reference": "6a467b8989322d92aa1c8bf2bebcc6e5c2ba55c0" + "reference": "30f38bffc6f24293dadd1823936372dfa9e86e2f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/6a467b8989322d92aa1c8bf2bebcc6e5c2ba55c0", - "reference": "6a467b8989322d92aa1c8bf2bebcc6e5c2ba55c0", + "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/30f38bffc6f24293dadd1823936372dfa9e86e2f", + "reference": "30f38bffc6f24293dadd1823936372dfa9e86e2f", "shasum": "" }, "require": { @@ -3729,7 +3729,8 @@ "phpdocumentor/reflection-common": "^2.0" }, "require-dev": { - "ext-tokenizer": "*" + "ext-tokenizer": "*", + "psalm/phar": "^4.8" }, "type": "library", "extra": { @@ -3755,39 +3756,39 @@ "description": "A PSR-5 based resolver of Class names, Types and Structural Element Names", "support": { "issues": "https://github.com/phpDocumentor/TypeResolver/issues", - "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.4.0" + "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.5.0" }, - "time": "2020-09-17T18:55:26+00:00" + "time": "2021-09-17T15:28:14+00:00" }, { "name": "phpspec/prophecy", - "version": "1.13.0", + "version": "1.14.0", "source": { "type": "git", "url": "https://github.com/phpspec/prophecy.git", - "reference": "be1996ed8adc35c3fd795488a653f4b518be70ea" + "reference": "d86dfc2e2a3cd366cee475e52c6bb3bbc371aa0e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpspec/prophecy/zipball/be1996ed8adc35c3fd795488a653f4b518be70ea", - "reference": "be1996ed8adc35c3fd795488a653f4b518be70ea", + "url": "https://api.github.com/repos/phpspec/prophecy/zipball/d86dfc2e2a3cd366cee475e52c6bb3bbc371aa0e", + "reference": "d86dfc2e2a3cd366cee475e52c6bb3bbc371aa0e", "shasum": "" }, "require": { "doctrine/instantiator": "^1.2", - "php": "^7.2 || ~8.0, <8.1", + "php": "^7.2 || ~8.0, <8.2", "phpdocumentor/reflection-docblock": "^5.2", "sebastian/comparator": "^3.0 || ^4.0", "sebastian/recursion-context": "^3.0 || ^4.0" }, "require-dev": { - "phpspec/phpspec": "^6.0", + "phpspec/phpspec": "^6.0 || ^7.0", "phpunit/phpunit": "^8.0 || ^9.0" }, "type": "library", "extra": { "branch-alias": { - "dev-master": "1.11.x-dev" + "dev-master": "1.x-dev" } }, "autoload": { @@ -3822,29 +3823,29 @@ ], "support": { "issues": "https://github.com/phpspec/prophecy/issues", - "source": "https://github.com/phpspec/prophecy/tree/1.13.0" + "source": "https://github.com/phpspec/prophecy/tree/1.14.0" }, - "time": "2021-03-17T13:42:18+00:00" + "time": "2021-09-10T09:02:12+00:00" }, { "name": "phpunit/php-code-coverage", - "version": "9.2.6", + "version": "9.2.7", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "f6293e1b30a2354e8428e004689671b83871edde" + "reference": "d4c798ed8d51506800b441f7a13ecb0f76f12218" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/f6293e1b30a2354e8428e004689671b83871edde", - "reference": "f6293e1b30a2354e8428e004689671b83871edde", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/d4c798ed8d51506800b441f7a13ecb0f76f12218", + "reference": "d4c798ed8d51506800b441f7a13ecb0f76f12218", "shasum": "" }, "require": { "ext-dom": "*", "ext-libxml": "*", "ext-xmlwriter": "*", - "nikic/php-parser": "^4.10.2", + "nikic/php-parser": "^4.12.0", "php": ">=7.3", "phpunit/php-file-iterator": "^3.0.3", "phpunit/php-text-template": "^2.0.2", @@ -3893,7 +3894,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/php-code-coverage/issues", - "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.6" + "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.7" }, "funding": [ { @@ -3901,7 +3902,7 @@ "type": "github" } ], - "time": "2021-03-28T07:26:59+00:00" + "time": "2021-09-17T05:39:03+00:00" }, { "name": "phpunit/php-file-iterator", @@ -5313,16 +5314,16 @@ }, { "name": "symfony/console", - "version": "v5.3.6", + "version": "v5.3.7", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "51b71afd6d2dc8f5063199357b9880cea8d8bfe2" + "reference": "8b1008344647462ae6ec57559da166c2bfa5e16a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/51b71afd6d2dc8f5063199357b9880cea8d8bfe2", - "reference": "51b71afd6d2dc8f5063199357b9880cea8d8bfe2", + "url": "https://api.github.com/repos/symfony/console/zipball/8b1008344647462ae6ec57559da166c2bfa5e16a", + "reference": "8b1008344647462ae6ec57559da166c2bfa5e16a", "shasum": "" }, "require": { @@ -5392,7 +5393,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v5.3.6" + "source": "https://github.com/symfony/console/tree/v5.3.7" }, "funding": [ { @@ -5408,7 +5409,7 @@ "type": "tidelift" } ], - "time": "2021-07-27T19:10:22+00:00" + "time": "2021-08-25T20:02:16+00:00" }, { "name": "symfony/deprecation-contracts", @@ -5882,16 +5883,16 @@ }, { "name": "symfony/string", - "version": "v5.3.3", + "version": "v5.3.7", "source": { "type": "git", "url": "https://github.com/symfony/string.git", - "reference": "bd53358e3eccec6a670b5f33ab680d8dbe1d4ae1" + "reference": "8d224396e28d30f81969f083a58763b8b9ceb0a5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/string/zipball/bd53358e3eccec6a670b5f33ab680d8dbe1d4ae1", - "reference": "bd53358e3eccec6a670b5f33ab680d8dbe1d4ae1", + "url": "https://api.github.com/repos/symfony/string/zipball/8d224396e28d30f81969f083a58763b8b9ceb0a5", + "reference": "8d224396e28d30f81969f083a58763b8b9ceb0a5", "shasum": "" }, "require": { @@ -5945,7 +5946,7 @@ "utf8" ], "support": { - "source": "https://github.com/symfony/string/tree/v5.3.3" + "source": "https://github.com/symfony/string/tree/v5.3.7" }, "funding": [ { @@ -5961,7 +5962,7 @@ "type": "tidelift" } ], - "time": "2021-06-27T11:44:38+00:00" + "time": "2021-08-26T08:00:08+00:00" }, { "name": "theseer/tokenizer", @@ -6015,16 +6016,16 @@ }, { "name": "twig/twig", - "version": "v2.14.6", + "version": "v2.14.7", "source": { "type": "git", "url": "https://github.com/twigphp/Twig.git", - "reference": "27e5cf2b05e3744accf39d4c68a3235d9966d260" + "reference": "8e202327ee1ed863629de9b18a5ec70ac614d88f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/twigphp/Twig/zipball/27e5cf2b05e3744accf39d4c68a3235d9966d260", - "reference": "27e5cf2b05e3744accf39d4c68a3235d9966d260", + "url": "https://api.github.com/repos/twigphp/Twig/zipball/8e202327ee1ed863629de9b18a5ec70ac614d88f", + "reference": "8e202327ee1ed863629de9b18a5ec70ac614d88f", "shasum": "" }, "require": { @@ -6034,7 +6035,7 @@ }, "require-dev": { "psr/container": "^1.0", - "symfony/phpunit-bridge": "^4.4.9|^5.0.9" + "symfony/phpunit-bridge": "^4.4.9|^5.0.9|^6.0" }, "type": "library", "extra": { @@ -6078,7 +6079,7 @@ ], "support": { "issues": "https://github.com/twigphp/Twig/issues", - "source": "https://github.com/twigphp/Twig/tree/v2.14.6" + "source": "https://github.com/twigphp/Twig/tree/v2.14.7" }, "funding": [ { @@ -6090,7 +6091,7 @@ "type": "tidelift" } ], - "time": "2021-05-16T12:12:47+00:00" + "time": "2021-09-17T08:39:54+00:00" }, { "name": "vimeo/psalm", diff --git a/src/Appwrite/Specification/Format/OpenAPI3.php b/src/Appwrite/Specification/Format/OpenAPI3.php index 5095c5395f..f032e6652e 100644 --- a/src/Appwrite/Specification/Format/OpenAPI3.php +++ b/src/Appwrite/Specification/Format/OpenAPI3.php @@ -6,6 +6,8 @@ use Appwrite\Specification\Format; use Appwrite\Template\Template; use stdClass; use Utopia\Validator; +use function array_map; +use function var_dump; class OpenAPI3 extends Format { @@ -39,7 +41,13 @@ class OpenAPI3 extends Format } if (!is_object($model)) return; foreach ($model->getRules() as $rule) { - $this->getUsedModels($rule['type'], $usedModels); + if(\is_array($rule['type'])) { + foreach ($rule['type'] as $type) { + $this->getUsedModels($type, $usedModels); + } + } else { + $this->getUsedModels($rule['type'], $usedModels); + } } } @@ -430,12 +438,24 @@ class OpenAPI3 extends Format $type = 'object'; $rule['type'] = ($rule['type']) ? $rule['type'] : 'none'; - $items = [ - '$ref' => '#/components/schemas/'.$rule['type'], - ]; + if(\is_array($rule['type'])) { + $items = [ + 'oneOf' => \array_map(function($type) { + return ['$ref' => '#/components/schemas/'.$type]; + }, $rule['type']) + ]; + } else { + $items = [ + '$ref' => '#/components/schemas/'.$rule['type'], + ]; + } + + break; } + + if($rule['array']) { $output['components']['schemas'][$model->getType()]['properties'][$name] = [ 'type' => 'array', From a87d482e495a41c625a2ab4d7fc3ac2a89e7d4dc Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Tue, 21 Sep 2021 14:12:43 +0200 Subject: [PATCH 167/206] Swagger2 support for arrays (work-around) --- .../Specification/Format/OpenAPI3.php | 6 --- .../Specification/Format/Swagger2.php | 46 +++++++++++++------ 2 files changed, 31 insertions(+), 21 deletions(-) diff --git a/src/Appwrite/Specification/Format/OpenAPI3.php b/src/Appwrite/Specification/Format/OpenAPI3.php index f032e6652e..54e668fbe3 100644 --- a/src/Appwrite/Specification/Format/OpenAPI3.php +++ b/src/Appwrite/Specification/Format/OpenAPI3.php @@ -6,8 +6,6 @@ use Appwrite\Specification\Format; use Appwrite\Template\Template; use stdClass; use Utopia\Validator; -use function array_map; -use function var_dump; class OpenAPI3 extends Format { @@ -449,13 +447,9 @@ class OpenAPI3 extends Format '$ref' => '#/components/schemas/'.$rule['type'], ]; } - - break; } - - if($rule['array']) { $output['components']['schemas'][$model->getType()]['properties'][$name] = [ 'type' => 'array', diff --git a/src/Appwrite/Specification/Format/Swagger2.php b/src/Appwrite/Specification/Format/Swagger2.php index 2a3be45d17..5e14bd1f31 100644 --- a/src/Appwrite/Specification/Format/Swagger2.php +++ b/src/Appwrite/Specification/Format/Swagger2.php @@ -39,7 +39,13 @@ class Swagger2 extends Format } if (!is_object($model)) return; foreach ($model->getRules() as $rule) { - $this->getUsedModels($rule['type'], $usedModels); + if(\is_array($rule['type'])) { + foreach ($rule['type'] as $type) { + $this->getUsedModels($type, $usedModels); + } + } else { + $this->getUsedModels($rule['type'], $usedModels); + } } } @@ -91,15 +97,15 @@ class Swagger2 extends Format if (isset($output['securityDefinitions']['Project'])) { $output['securityDefinitions']['Project']['x-appwrite'] = ['demo' => '5df5acd0d48c2']; } - + if (isset($output['securityDefinitions']['Key'])) { $output['securityDefinitions']['Key']['x-appwrite'] = ['demo' => '919c2d18fb5d4...a2ae413da83346ad2']; } - + if (isset($output['securityDefinitions']['JWT'])) { $output['securityDefinitions']['JWT']['x-appwrite'] = ['demo' => 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ...']; } - + if (isset($output['securityDefinitions']['Locale'])) { $output['securityDefinitions']['Locale']['x-appwrite'] = ['demo' => 'en']; } @@ -147,7 +153,7 @@ class Swagger2 extends Format if(empty($routeSecurity)) { $sdkPlatofrms[] = APP_PLATFORM_CLIENT; } - + $temp = [ 'summary' => $route->getDesc(), 'operationId' => $route->getLabel('sdk.namespace', 'default').ucfirst($id), @@ -216,7 +222,7 @@ class Swagger2 extends Format if ((!empty($scope))) { // && 'public' != $scope $securities = ['Project' => []]; - + foreach($route->getLabel('sdk.auth', []) as $security) { if(array_key_exists($security, $this->keys)) { $securities[$security] = []; @@ -226,7 +232,7 @@ class Swagger2 extends Format $temp['x-appwrite']['auth'] = array_slice($securities, 0, $this->authCount); $temp['security'][] = $securities; } - + $body = [ 'name' => 'payload', 'in' => 'body', @@ -399,7 +405,7 @@ class Swagger2 extends Format if($model->isAny()) { $output['definitions'][$model->getType()]['additionalProperties'] = true; } - + if(!empty($required)) { $output['definitions'][$model->getType()]['required'] = $required; } @@ -414,7 +420,7 @@ class Swagger2 extends Format case 'json': $type = 'string'; break; - + case 'integer': $type = 'integer'; $format = 'int32'; @@ -424,19 +430,29 @@ class Swagger2 extends Format $type = 'number'; $format = 'float'; break; - + case 'boolean': $type = 'boolean'; break; - + default: $type = 'object'; $rule['type'] = ($rule['type']) ? $rule['type'] : 'none'; - $items = [ - 'type' => $type, - '$ref' => '#/definitions/'.$rule['type'], - ]; + if(\is_array($rule['type'])) { + // THIS IS NOT SUPPORTED IN 2.0!!! +// $items = [ +// 'oneOf' => \array_map(function($type) { +// return ['$ref' => '#/definitions/'.$type]; +// }, $rule['type']) +// ]; + + $items = []; + } else { + $items = [ + '$ref' => '#/definitions/'.$rule['type'], + ]; + } break; } From b97542e581910e452da8a6d01a1be77282e7b5d3 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Tue, 21 Sep 2021 14:16:24 +0200 Subject: [PATCH 168/206] Swagger2 array fix --- src/Appwrite/Specification/Format/Swagger2.php | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/Appwrite/Specification/Format/Swagger2.php b/src/Appwrite/Specification/Format/Swagger2.php index 5e14bd1f31..493dac0bb5 100644 --- a/src/Appwrite/Specification/Format/Swagger2.php +++ b/src/Appwrite/Specification/Format/Swagger2.php @@ -440,14 +440,11 @@ class Swagger2 extends Format $rule['type'] = ($rule['type']) ? $rule['type'] : 'none'; if(\is_array($rule['type'])) { - // THIS IS NOT SUPPORTED IN 2.0!!! -// $items = [ -// 'oneOf' => \array_map(function($type) { -// return ['$ref' => '#/definitions/'.$type]; -// }, $rule['type']) -// ]; - - $items = []; + $items = [ + 'oneOf' => \array_map(function($type) { + return ['$ref' => '#/definitions/'.$type]; + }, $rule['type']) + ]; } else { $items = [ '$ref' => '#/definitions/'.$rule['type'], From 1eeddc80151fb3bc2bf8ea393955134b4dd4566c Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Tue, 21 Sep 2021 14:19:25 +0200 Subject: [PATCH 169/206] This should not be removed --- src/Appwrite/Specification/Format/Swagger2.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Appwrite/Specification/Format/Swagger2.php b/src/Appwrite/Specification/Format/Swagger2.php index 493dac0bb5..70811536ca 100644 --- a/src/Appwrite/Specification/Format/Swagger2.php +++ b/src/Appwrite/Specification/Format/Swagger2.php @@ -447,6 +447,7 @@ class Swagger2 extends Format ]; } else { $items = [ + 'type' => $type, '$ref' => '#/definitions/'.$rule['type'], ]; } From d0e777edd5e27e8ca8c7ea5b7668865c61090a50 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Tue, 21 Sep 2021 14:25:41 +0200 Subject: [PATCH 170/206] Swagger Double type fix --- src/Appwrite/Specification/Format/OpenAPI3.php | 7 ++++++- src/Appwrite/Specification/Format/Swagger2.php | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Specification/Format/OpenAPI3.php b/src/Appwrite/Specification/Format/OpenAPI3.php index 54e668fbe3..82eb695338 100644 --- a/src/Appwrite/Specification/Format/OpenAPI3.php +++ b/src/Appwrite/Specification/Format/OpenAPI3.php @@ -33,7 +33,7 @@ class OpenAPI3 extends Format */ protected function getUsedModels($model, array &$usedModels) { - if (is_string($model) && !in_array($model, ['string', 'integer', 'boolean', 'json', 'float'])) { + if (is_string($model) && !in_array($model, ['string', 'integer', 'boolean', 'json', 'float', 'double'])) { $usedModels[] = $model; return; } @@ -427,6 +427,11 @@ class OpenAPI3 extends Format $type = 'number'; $format = 'float'; break; + + case 'double': + $type = 'number'; + $format = 'double'; + break; case 'boolean': $type = 'boolean'; diff --git a/src/Appwrite/Specification/Format/Swagger2.php b/src/Appwrite/Specification/Format/Swagger2.php index 70811536ca..397b97f960 100644 --- a/src/Appwrite/Specification/Format/Swagger2.php +++ b/src/Appwrite/Specification/Format/Swagger2.php @@ -33,7 +33,7 @@ class Swagger2 extends Format */ protected function getUsedModels($model, array &$usedModels) { - if (is_string($model) && !in_array($model, ['string', 'integer', 'boolean', 'json', 'float'])) { + if (is_string($model) && !in_array($model, ['string', 'integer', 'boolean', 'json', 'float', 'double'])) { $usedModels[] = $model; return; } @@ -431,6 +431,11 @@ class Swagger2 extends Format $format = 'float'; break; + case 'double': + $type = 'number'; + $format = 'double'; + break; + case 'boolean': $type = 'boolean'; break; From 9c9a17a2a44ac06113e9ec54b1e9248425b16091 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 21 Sep 2021 20:33:11 +0200 Subject: [PATCH 171/206] Apply suggestions from code review Co-authored-by: Torsten Dittmann --- src/Appwrite/Specification/Format/OpenAPI3.php | 2 +- src/Appwrite/Specification/Format/Swagger2.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Appwrite/Specification/Format/OpenAPI3.php b/src/Appwrite/Specification/Format/OpenAPI3.php index 82eb695338..0c055fe709 100644 --- a/src/Appwrite/Specification/Format/OpenAPI3.php +++ b/src/Appwrite/Specification/Format/OpenAPI3.php @@ -443,7 +443,7 @@ class OpenAPI3 extends Format if(\is_array($rule['type'])) { $items = [ - 'oneOf' => \array_map(function($type) { + 'anyOf' => \array_map(function($type) { return ['$ref' => '#/components/schemas/'.$type]; }, $rule['type']) ]; diff --git a/src/Appwrite/Specification/Format/Swagger2.php b/src/Appwrite/Specification/Format/Swagger2.php index 397b97f960..88cec0b2a0 100644 --- a/src/Appwrite/Specification/Format/Swagger2.php +++ b/src/Appwrite/Specification/Format/Swagger2.php @@ -446,7 +446,7 @@ class Swagger2 extends Format if(\is_array($rule['type'])) { $items = [ - 'oneOf' => \array_map(function($type) { + 'anyOf' => \array_map(function($type) { return ['$ref' => '#/definitions/'.$type]; }, $rule['type']) ]; From c87e686ca2224b202dcf4da5b6cc25dfa43c72d8 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Wed, 22 Sep 2021 21:29:56 -0400 Subject: [PATCH 172/206] Fix createUrlAttribute description --- app/controllers/api/database.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index a031888ce1..0451287a2a 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -785,7 +785,7 @@ App::post('/v1/database/collections/:collectionId/attributes/ip') }); App::post('/v1/database/collections/:collectionId/attributes/url') - ->desc('Create IP Address Attribute') + ->desc('Create URL Attribute') ->groups(['api', 'database']) ->label('event', 'database.attributes.create') ->label('scope', 'collections.write') From 4bbf0e93037e03edd072c29f0432a68f45485cf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Thu, 23 Sep 2021 08:44:57 +0200 Subject: [PATCH 173/206] Remove unnecessary whitespace Co-authored-by: kodumbeats --- tests/e2e/Services/Storage/StorageBase.php | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/e2e/Services/Storage/StorageBase.php b/tests/e2e/Services/Storage/StorageBase.php index 8d9b2d9c8c..c2f2483a50 100644 --- a/tests/e2e/Services/Storage/StorageBase.php +++ b/tests/e2e/Services/Storage/StorageBase.php @@ -210,7 +210,6 @@ trait StorageBase $this->assertCount(1, $response['body']['files']); $this->assertEquals('logo.png', $response['body']['files'][0]['name']); - $response = $this->client->call(Client::METHOD_GET, '/storage/files', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], From d43987fd7a569eb3470900be8f3ed26cce248758 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Thu, 23 Sep 2021 08:49:45 +0200 Subject: [PATCH 174/206] Fix test assert bug Co-authored-by: kodumbeats --- tests/e2e/Services/Storage/StorageBase.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/Services/Storage/StorageBase.php b/tests/e2e/Services/Storage/StorageBase.php index c2f2483a50..2bd69ec1a7 100644 --- a/tests/e2e/Services/Storage/StorageBase.php +++ b/tests/e2e/Services/Storage/StorageBase.php @@ -205,7 +205,7 @@ trait StorageBase ]); $this->assertEquals(200, $response['headers']['status-code']); - $this->assertGreaterThan(0, $response['body']['sum']); + $this->assertEquals(1, $response['body']['sum']); $this->assertIsInt($response['body']['sum']); $this->assertCount(1, $response['body']['files']); $this->assertEquals('logo.png', $response['body']['files'][0]['name']); From 8f1847f87109c617dadd41f1bbe6a109199f329a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Thu, 23 Sep 2021 08:50:47 +0200 Subject: [PATCH 175/206] Test assert bug fix Co-authored-by: kodumbeats --- tests/e2e/Services/Storage/StorageBase.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/Services/Storage/StorageBase.php b/tests/e2e/Services/Storage/StorageBase.php index 2bd69ec1a7..ecfc55d1a0 100644 --- a/tests/e2e/Services/Storage/StorageBase.php +++ b/tests/e2e/Services/Storage/StorageBase.php @@ -218,7 +218,7 @@ trait StorageBase ]); $this->assertEquals(200, $response['headers']['status-code']); - $this->assertGreaterThan(0, $response['body']['sum']); + $this->assertEquals(1, $response['body']['sum']); $this->assertIsInt($response['body']['sum']); $this->assertGreaterThan(0, $response['body']['files']); $this->assertEquals($data['fileId'], $response['body']['files'][0]['$id']); From 1f2f0c2443618366256522566d0645e4d5c98e95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Thu, 23 Sep 2021 08:57:16 +0200 Subject: [PATCH 176/206] Apply suggestions from code review Co-authored-by: kodumbeats --- tests/e2e/Services/Storage/StorageBase.php | 2 +- tests/e2e/Services/Users/UsersBase.php | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/e2e/Services/Storage/StorageBase.php b/tests/e2e/Services/Storage/StorageBase.php index ecfc55d1a0..ab1a08c682 100644 --- a/tests/e2e/Services/Storage/StorageBase.php +++ b/tests/e2e/Services/Storage/StorageBase.php @@ -220,7 +220,7 @@ trait StorageBase $this->assertEquals(200, $response['headers']['status-code']); $this->assertEquals(1, $response['body']['sum']); $this->assertIsInt($response['body']['sum']); - $this->assertGreaterThan(0, $response['body']['files']); + $this->assertCount(1, $response['body']['files']); $this->assertEquals($data['fileId'], $response['body']['files'][0]['$id']); /** diff --git a/tests/e2e/Services/Users/UsersBase.php b/tests/e2e/Services/Users/UsersBase.php index 1fd4444b35..564b3e7868 100644 --- a/tests/e2e/Services/Users/UsersBase.php +++ b/tests/e2e/Services/Users/UsersBase.php @@ -96,7 +96,7 @@ trait UsersBase $this->assertEquals($response['headers']['status-code'], 200); $this->assertNotEmpty($response['body']); $this->assertNotEmpty($response['body']['users']); - $this->assertGreaterThan(0, $response['body']['users']); + $this->assertCount(1, $response['body']['users']); $this->assertEquals($response['body']['users'][0]['$id'], $data['userId']); $response = $this->client->call(Client::METHOD_GET, '/users', array_merge([ @@ -108,7 +108,7 @@ trait UsersBase $this->assertEquals($response['headers']['status-code'], 200); $this->assertNotEmpty($response['body']); $this->assertNotEmpty($response['body']['users']); - $this->assertGreaterThan(0, $response['body']['users']); + $this->assertCount(1, $response['body']['users']); $this->assertEquals($response['body']['users'][0]['$id'], $data['userId']); $response = $this->client->call(Client::METHOD_GET, '/users', array_merge([ @@ -120,7 +120,7 @@ trait UsersBase $this->assertEquals($response['headers']['status-code'], 200); $this->assertNotEmpty($response['body']); $this->assertNotEmpty($response['body']['users']); - $this->assertGreaterThan(0, $response['body']['users']); + $this->assertCount(1, $response['body']['users']); $this->assertEquals($response['body']['users'][0]['$id'], $data['userId']); $response = $this->client->call(Client::METHOD_GET, '/users', array_merge([ From c61b00995c643bc95a1fc968adcc40dbc21858cf Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Thu, 23 Sep 2021 09:01:10 +0200 Subject: [PATCH 177/206] Manual code review suggestion changes --- .../Functions/FunctionsCustomServerTest.php | 12 +++++++++-- .../Projects/ProjectsConsoleClientTest.php | 20 +++++++++++++------ tests/e2e/Services/Users/UsersBase.php | 4 ++-- 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index 62198293a7..a0b0754b0b 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -77,6 +77,10 @@ class FunctionsCustomServerTest extends Scope /** * Test for SUCCESS */ + + /** + * Test search queries + */ $response = $this->client->call(Client::METHOD_GET, '/functions', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], @@ -110,7 +114,9 @@ class FunctionsCustomServerTest extends Scope $this->assertCount(1, $response['body']['functions']); $this->assertEquals($response['body']['functions'][0]['$id'], $data['functionId']); - + /** + * Test pagination + */ $response = $this->client->call(Client::METHOD_POST, '/functions', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], @@ -155,7 +161,6 @@ class FunctionsCustomServerTest extends Scope $this->assertCount(1, $response['body']['functions']); $this->assertEquals($response['body']['functions'][0]['name'], 'Test 2'); - return $data; } @@ -316,6 +321,9 @@ class FunctionsCustomServerTest extends Scope $this->assertIsArray($function['body']['tags']); $this->assertCount(1, $function['body']['tags']); + /** + * Test search queries + */ $function = $this->client->call(Client::METHOD_GET, '/functions/'.$data['functionId'].'/tags', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], diff --git a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php index 5de323cc4d..56daab688d 100644 --- a/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php +++ b/tests/e2e/Services/Projects/ProjectsConsoleClientTest.php @@ -87,6 +87,7 @@ class ProjectsConsoleClientTest extends Scope /** * Test for SUCCESS */ + $response = $this->client->call(Client::METHOD_GET, '/projects', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], @@ -97,6 +98,9 @@ class ProjectsConsoleClientTest extends Scope $this->assertEquals($id, $response['body']['projects'][0]['$id']); $this->assertEquals('Project Test', $response['body']['projects'][0]['name']); + /** + * Test search queries + */ $response = $this->client->call(Client::METHOD_GET, '/projects', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], @@ -104,9 +108,11 @@ class ProjectsConsoleClientTest extends Scope 'search' => $id ])); - $this->assertEquals(200, $response['headers']['status-code']); - $this->assertNotEmpty($response['body']); - $this->assertEquals('Project Test', $response['body']['projects'][0]['name']); + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertEquals($response['body']['sum'], 1); + $this->assertIsArray($response['body']['projects']); + $this->assertCount(1, $response['body']['projects']); + $this->assertEquals($response['body']['projects'][0]['name'], 'Project Test'); $response = $this->client->call(Client::METHOD_GET, '/projects', array_merge([ 'content-type' => 'application/json', @@ -115,9 +121,11 @@ class ProjectsConsoleClientTest extends Scope 'search' => 'Project Test' ])); - $this->assertEquals(200, $response['headers']['status-code']); - $this->assertNotEmpty($response['body']); - $this->assertEquals($id, $response['body']['projects'][0]['$id']); + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertEquals($response['body']['sum'], 1); + $this->assertIsArray($response['body']['projects']); + $this->assertCount(1, $response['body']['projects']); + $this->assertEquals($response['body']['projects'][0]['$id'], $data['projectId']); /** * Test after pagination diff --git a/tests/e2e/Services/Users/UsersBase.php b/tests/e2e/Services/Users/UsersBase.php index 564b3e7868..7db500ddc8 100644 --- a/tests/e2e/Services/Users/UsersBase.php +++ b/tests/e2e/Services/Users/UsersBase.php @@ -132,7 +132,7 @@ trait UsersBase $this->assertEquals($response['headers']['status-code'], 200); $this->assertNotEmpty($response['body']); $this->assertNotEmpty($response['body']['users']); - $this->assertGreaterThan(0, $response['body']['users']); + $this->assertCount(1, $response['body']['users']); $this->assertEquals($response['body']['users'][0]['$id'], $data['userId']); $response = $this->client->call(Client::METHOD_GET, '/users', array_merge([ @@ -144,7 +144,7 @@ trait UsersBase $this->assertEquals($response['headers']['status-code'], 200); $this->assertNotEmpty($response['body']); $this->assertNotEmpty($response['body']['users']); - $this->assertGreaterThan(0, $response['body']['users']); + $this->assertCount(1, $response['body']['users']); $this->assertEquals($response['body']['users'][0]['$id'], $data['userId']); } From c97b15325507b6eca319fca1fa38ab741be4e43a Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Thu, 23 Sep 2021 13:12:50 +0200 Subject: [PATCH 178/206] Implemented oneOf for non-array results with multiple types --- .../Specification/Format/OpenAPI3.php | 19 ++++++++++++++----- .../Specification/Format/Swagger2.php | 18 +++++++++++++----- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/src/Appwrite/Specification/Format/OpenAPI3.php b/src/Appwrite/Specification/Format/OpenAPI3.php index 0c055fe709..7bcfdd00a2 100644 --- a/src/Appwrite/Specification/Format/OpenAPI3.php +++ b/src/Appwrite/Specification/Format/OpenAPI3.php @@ -6,6 +6,7 @@ use Appwrite\Specification\Format; use Appwrite\Template\Template; use stdClass; use Utopia\Validator; +use function var_dump; class OpenAPI3 extends Format { @@ -442,11 +443,19 @@ class OpenAPI3 extends Format $rule['type'] = ($rule['type']) ? $rule['type'] : 'none'; if(\is_array($rule['type'])) { - $items = [ - 'anyOf' => \array_map(function($type) { - return ['$ref' => '#/components/schemas/'.$type]; - }, $rule['type']) - ]; + if($rule['array']) { + $items = [ + 'anyOf' => \array_map(function($type) { + return ['$ref' => '#/components/schemas/'.$type]; + }, $rule['type']) + ]; + } else { + $items = [ + 'oneOf' => \array_map(function($type) { + return ['$ref' => '#/components/schemas/'.$type]; + }, $rule['type']) + ]; + } } else { $items = [ '$ref' => '#/components/schemas/'.$rule['type'], diff --git a/src/Appwrite/Specification/Format/Swagger2.php b/src/Appwrite/Specification/Format/Swagger2.php index 88cec0b2a0..66324567bc 100644 --- a/src/Appwrite/Specification/Format/Swagger2.php +++ b/src/Appwrite/Specification/Format/Swagger2.php @@ -445,11 +445,19 @@ class Swagger2 extends Format $rule['type'] = ($rule['type']) ? $rule['type'] : 'none'; if(\is_array($rule['type'])) { - $items = [ - 'anyOf' => \array_map(function($type) { - return ['$ref' => '#/definitions/'.$type]; - }, $rule['type']) - ]; + if($rule['array']) { + $items = [ + 'anyOf' => \array_map(function($type) { + return ['$ref' => '#/definitions/'.$type]; + }, $rule['type']) + ]; + } else { + $items = [ + 'oneOf' => \array_map(function($type) { + return ['$ref' => '#/definitions/'.$type]; + }, $rule['type']) + ]; + } } else { $items = [ 'type' => $type, From 36f55c8726364fdda25c629647046ff6fd3f8acc Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Thu, 23 Sep 2021 13:13:22 +0200 Subject: [PATCH 179/206] Removed var dump --- src/Appwrite/Specification/Format/OpenAPI3.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Appwrite/Specification/Format/OpenAPI3.php b/src/Appwrite/Specification/Format/OpenAPI3.php index 7bcfdd00a2..1d462669c9 100644 --- a/src/Appwrite/Specification/Format/OpenAPI3.php +++ b/src/Appwrite/Specification/Format/OpenAPI3.php @@ -6,7 +6,6 @@ use Appwrite\Specification\Format; use Appwrite\Template\Template; use stdClass; use Utopia\Validator; -use function var_dump; class OpenAPI3 extends Format { From add7a01fac568ab7e78220ef0659a1f6cbc6b44e Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Thu, 23 Sep 2021 15:40:05 +0200 Subject: [PATCH 180/206] Added database key for performence --- app/config/collections2.php | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/app/config/collections2.php b/app/config/collections2.php index 866d1cb1b6..71dc7e9f69 100644 --- a/app/config/collections2.php +++ b/app/config/collections2.php @@ -715,6 +715,13 @@ $collections = [ 'lengths' => [1024], 'orders' => [Database::ORDER_ASC], ], + [ + '$id' => '_key_deleted_email', + 'type' => Database::INDEX_KEY, + 'attributes' => ['deleted', 'email'], + 'lengths' => [0, 1024], + 'orders' => [Database::ORDER_ASC, Database::ORDER_ASC], + ], ], ], From d0b70a9dee75d1a1f6026f325e1cce6374a2ab5e Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Mon, 27 Sep 2021 12:12:42 +0200 Subject: [PATCH 181/206] Re-added function execution search + added missing tests --- app/config/collections2.php | 18 +++++++++++ app/controllers/api/functions.php | 24 +++++++++----- .../Functions/FunctionsCustomServerTest.php | 32 +++++++++++++++++++ 3 files changed, 66 insertions(+), 8 deletions(-) diff --git a/app/config/collections2.php b/app/config/collections2.php index ea99ae47d1..da79e3684c 100644 --- a/app/config/collections2.php +++ b/app/config/collections2.php @@ -1711,6 +1711,17 @@ $collections = [ 'array' => false, 'filters' => [], ], + [ + '$id' => 'search', + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 16384, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ], ], 'indexes' => [ [ @@ -1720,6 +1731,13 @@ $collections = [ 'lengths' => [Database::LENGTH_KEY], 'orders' => [Database::ORDER_ASC], ], + [ + '$id' => '_fulltext_search', + 'type' => Database::INDEX_FULLTEXT, + 'attributes' => ['search'], + 'lengths' => [1024], + 'orders' => [Database::ORDER_ASC], + ], ], ], diff --git a/app/controllers/api/functions.php b/app/controllers/api/functions.php index 28ff911c10..00ed181239 100644 --- a/app/controllers/api/functions.php +++ b/app/controllers/api/functions.php @@ -680,8 +680,10 @@ App::post('/v1/functions/:functionId/executions') Authorization::disable(); + $executionId = $dbForInternal->getId(); + $execution = $dbForInternal->createDocument('executions', new Document([ - '$id' => $dbForInternal->getId(), + '$id' => $executionId, '$read' => (!$user->isEmpty()) ? ['user:' . $user->getId()] : [], '$write' => [], 'dateCreated' => time(), @@ -693,6 +695,7 @@ App::post('/v1/functions/:functionId/executions') 'stdout' => '', 'stderr' => '', 'time' => 0.0, + 'search' => implode(' ', [$functionId, $executionId]), ])); Authorization::reset(); @@ -747,10 +750,11 @@ App::get('/v1/functions/:functionId/executions') ->param('functionId', '', new UID(), 'Function unique ID.') ->param('limit', 25, new Range(0, 100), 'Results limit value. By default will return maximum 25 results. Maximum of 100 results allowed per request.', true) ->param('offset', 0, new Range(0, 2000), 'Results offset. The default value is 0. Use this param to manage pagination.', true) + ->param('search', '', new Text(256), 'Search term to filter your list results. Max length: 256 chars.', true) ->param('after', '', new UID(), 'ID of the execution used as the starting point for the query, excluding the execution itself. Should be used for efficient pagination when working with large sets of data.', true) ->inject('response') ->inject('dbForInternal') - ->action(function ($functionId, $limit, $offset, $after, $response, $dbForInternal) { + ->action(function ($functionId, $limit, $offset, $search, $after, $response, $dbForInternal) { /** @var Appwrite\Utopia\Response $response */ /** @var Utopia\Database\Database $dbForInternal */ @@ -770,13 +774,17 @@ App::get('/v1/functions/:functionId/executions') } } - $results = $dbForInternal->find('executions', [ - new Query('functionId', Query::TYPE_EQUAL, [$function->getId()]), - ], $limit, $offset, [], [Database::ORDER_DESC], $afterExecution ?? null); + $queries = [ + new Query('functionId', Query::TYPE_EQUAL, [$function->getId()]) + ]; - $sum = $dbForInternal->count('executions', [ - new Query('functionId', Query::TYPE_EQUAL, [$function->getId()]), - ], APP_LIMIT_COUNT); + if (!empty($search)) { + $queries[] = new Query('search', Query::TYPE_SEARCH, [$search]); + } + + $results = $dbForInternal->find('executions', $queries, $limit, $offset, [], [Database::ORDER_DESC], $afterExecution ?? null); + + $sum = $dbForInternal->count('executions', $queries, APP_LIMIT_COUNT); $response->dynamic(new Document([ 'executions' => $results, diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index a0b0754b0b..aa64133884 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -7,6 +7,8 @@ use Tests\E2E\Client; use Tests\E2E\Scopes\ProjectCustom; use Tests\E2E\Scopes\Scope; use Tests\E2E\Scopes\SideServer; +use function array_merge; +use function var_dump; class FunctionsCustomServerTest extends Scope { @@ -474,6 +476,36 @@ class FunctionsCustomServerTest extends Scope $this->assertCount(1, $function['body']['executions']); $this->assertEquals($function['body']['executions'][0]['$id'], $data['executionId']); + /** + * Test search queries + */ + + $response = $this->client->call(Client::METHOD_GET, '/functions/'.$data['functionId'].'/executions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'search' => $data['executionId'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals(1, $response['body']['sum']); + $this->assertIsInt($response['body']['sum']); + $this->assertCount(1, $response['body']['executions']); + $this->assertEquals($data['functionId'], $response['body']['executions'][0]['functionId']); + + $response = $this->client->call(Client::METHOD_GET, '/functions/'.$data['functionId'].'/executions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'search' => $data['functionId'], + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals(1, $response['body']['sum']); + $this->assertIsInt($response['body']['sum']); + $this->assertCount(1, $response['body']['executions']); + $this->assertEquals($data['executionId'], $response['body']['executions'][0]['$id']); + return $data; } From 24f9c2518bf54f68c15329abb46f58f15fd31054 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Mon, 27 Sep 2021 12:20:32 +0200 Subject: [PATCH 182/206] Removed imports --- tests/e2e/Services/Functions/FunctionsCustomServerTest.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index aa64133884..57b8ad9218 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -7,8 +7,6 @@ use Tests\E2E\Client; use Tests\E2E\Scopes\ProjectCustom; use Tests\E2E\Scopes\Scope; use Tests\E2E\Scopes\SideServer; -use function array_merge; -use function var_dump; class FunctionsCustomServerTest extends Scope { From 429da848c6ccb1ee41f15dbe82c2b023f42b6989 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Mon, 27 Sep 2021 14:11:06 +0200 Subject: [PATCH 183/206] Added missing tests for user deprecation feature --- composer.lock | 12 ++--- docker-compose.yml | 2 +- .../Services/Users/UsersCustomServerTest.php | 48 +++++++++++++++++++ 3 files changed, 55 insertions(+), 7 deletions(-) diff --git a/composer.lock b/composer.lock index c319daa5a3..6e2046e5e8 100644 --- a/composer.lock +++ b/composer.lock @@ -2576,16 +2576,16 @@ "packages-dev": [ { "name": "amphp/amp", - "version": "v2.6.0", + "version": "v2.6.1", "source": { "type": "git", "url": "https://github.com/amphp/amp.git", - "reference": "caa95edeb1ca1bf7532e9118ede4a3c3126408cc" + "reference": "c5fc66a78ee38d7ac9195a37bacaf940eb3f65ae" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/amphp/amp/zipball/caa95edeb1ca1bf7532e9118ede4a3c3126408cc", - "reference": "caa95edeb1ca1bf7532e9118ede4a3c3126408cc", + "url": "https://api.github.com/repos/amphp/amp/zipball/c5fc66a78ee38d7ac9195a37bacaf940eb3f65ae", + "reference": "c5fc66a78ee38d7ac9195a37bacaf940eb3f65ae", "shasum": "" }, "require": { @@ -2653,7 +2653,7 @@ "support": { "irc": "irc://irc.freenode.org/amphp", "issues": "https://github.com/amphp/amp/issues", - "source": "https://github.com/amphp/amp/tree/v2.6.0" + "source": "https://github.com/amphp/amp/tree/v2.6.1" }, "funding": [ { @@ -2661,7 +2661,7 @@ "type": "github" } ], - "time": "2021-07-16T20:06:06+00:00" + "time": "2021-09-23T18:43:08+00:00" }, { "name": "amphp/byte-stream", diff --git a/docker-compose.yml b/docker-compose.yml index 539799aa9a..27c2a125d9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -63,7 +63,7 @@ services: - ./psalm.xml:/usr/src/code/psalm.xml - ./tests:/usr/src/code/tests - ./app:/usr/src/code/app - # - ./vendor/utopia-php/database:/usr/src/code/vendor/utopia-php/database + - ./vendor:/usr/src/code/vendor - ./docs:/usr/src/code/docs - ./public:/usr/src/code/public - ./src:/usr/src/code/src diff --git a/tests/e2e/Services/Users/UsersCustomServerTest.php b/tests/e2e/Services/Users/UsersCustomServerTest.php index c5e4ff8c1a..3acd4330af 100644 --- a/tests/e2e/Services/Users/UsersCustomServerTest.php +++ b/tests/e2e/Services/Users/UsersCustomServerTest.php @@ -2,6 +2,7 @@ namespace Tests\E2E\Services\Users; +use Tests\E2E\Client; use Tests\E2E\Scopes\ProjectCustom; use Tests\E2E\Scopes\Scope; use Tests\E2E\Scopes\SideServer; @@ -11,4 +12,51 @@ class UsersCustomServerTest extends Scope use UsersBase; use ProjectCustom; use SideServer; + + public function testDeprecatedUsers():array + { + /** + * Test for FAILURE (don't allow recreating account with same custom ID) + */ + + // Create user with custom ID 'meldiron' + $response = $this->client->call(Client::METHOD_POST, '/users', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'userId' => 'meldiron', + 'email' => 'matej@appwrite.io', + 'password' => 'my-superstr0ng-password', + 'name' => 'Matej Bačo' + ]); + + $this->assertEquals(201, $response['headers']['status-code']); + + // Delete user with custom ID 'meldiron' + $response = $this->client->call(Client::METHOD_DELETE, '/users/meldiron', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + + ]); + + $this->assertEquals(204, $response['headers']['status-code']); + + // Try to create user with custom ID 'meldiron' again, but now it should fail + $response1 = $this->client->call(Client::METHOD_POST, '/users', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'userId' => 'meldiron', + 'email' => 'matej@appwrite.io', + 'password' => 'my-superstr0ng-password', + 'name' => 'Matej Bačo' + ]); + + $this->assertEquals(409, $response1['headers']['status-code']); + $this->assertEquals('Account already exists', $response1['body']['message']); + + return []; + } + } \ No newline at end of file From f4d07e826b34129c0baf7123487d221a419e2310 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Mon, 27 Sep 2021 19:38:03 -0400 Subject: [PATCH 184/206] Fix tests by testing cleanup duplicate behavior on fresh collection --- .../Database/DatabaseCustomServerTest.php | 38 ++++++++++++------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/tests/e2e/Services/Database/DatabaseCustomServerTest.php b/tests/e2e/Services/Database/DatabaseCustomServerTest.php index fd4a12b5a1..50d8af29f4 100644 --- a/tests/e2e/Services/Database/DatabaseCustomServerTest.php +++ b/tests/e2e/Services/Database/DatabaseCustomServerTest.php @@ -336,12 +336,26 @@ class DatabaseCustomServerTest extends Scope return $data; } - /** - * @depends testDeleteIndex - */ - public function testCleanupDuplicateIndexOnDeleteAttribute($data) + public function testCleanupDuplicateIndexOnDeleteAttribute() { - $attribute1 = $this->client->call(Client::METHOD_POST, '/database/collections/' . $data['collectionId'] . '/attributes/string', array_merge([ + $collection = $this->client->call(Client::METHOD_POST, '/database/collections', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]), [ + 'collectionId' => 'unique()', + 'name' => 'TestCleanupDuplicateIndexOnDeleteAttribute', + 'read' => ['role:all'], + 'write' => ['role:all'], + 'permission' => 'document', + ]); + + $this->assertEquals(201, $collection['headers']['status-code']); + $this->assertNotEmpty($collection['body']['$id']); + + $collectionId = $collection['body']['$id']; + + $attribute1 = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/attributes/string', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] @@ -351,7 +365,7 @@ class DatabaseCustomServerTest extends Scope 'required' => true, ]); - $attribute2 = $this->client->call(Client::METHOD_POST, '/database/collections/' . $data['collectionId'] . '/attributes/string', array_merge([ + $attribute2 = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/attributes/string', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] @@ -368,7 +382,7 @@ class DatabaseCustomServerTest extends Scope sleep(2); - $index1 = $this->client->call(Client::METHOD_POST, '/database/collections/' . $data['collectionId'] . '/indexes', array_merge([ + $index1 = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/indexes', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] @@ -379,7 +393,7 @@ class DatabaseCustomServerTest extends Scope 'orders' => ['ASC', 'ASC'], ]); - $index2 = $this->client->call(Client::METHOD_POST, '/database/collections/' . $data['collectionId'] . '/indexes', array_merge([ + $index2 = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/indexes', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] @@ -397,7 +411,7 @@ class DatabaseCustomServerTest extends Scope sleep(2); // Expected behavior: deleting attribute1 would cause index1 to be a duplicate of index2 and automatically removed - $deleted = $this->client->call(Client::METHOD_DELETE, '/database/collections/' . $data['collectionId'] . '/attributes/'. $attribute1['body']['key'], array_merge([ + $deleted = $this->client->call(Client::METHOD_DELETE, '/database/collections/' . $collectionId . '/attributes/'. $attribute1['body']['key'], array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] @@ -408,7 +422,7 @@ class DatabaseCustomServerTest extends Scope // wait for database worker to complete sleep(2); - $collection = $this->client->call(Client::METHOD_GET, '/database/collections/' . $data['collectionId'], array_merge([ + $collection = $this->client->call(Client::METHOD_GET, '/database/collections/' . $collectionId, array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] @@ -423,15 +437,13 @@ class DatabaseCustomServerTest extends Scope $this->assertEquals($attribute2['body']['key'], $collection['body']['indexes'][0]['attributes'][0]); // Delete attribute - $deleted = $this->client->call(Client::METHOD_DELETE, '/database/collections/' . $data['collectionId'] . '/attributes/' . $attribute2['body']['key'], array_merge([ + $deleted = $this->client->call(Client::METHOD_DELETE, '/database/collections/' . $collectionId . '/attributes/' . $attribute2['body']['key'], array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], 'x-appwrite-key' => $this->getProject()['apiKey'] ])); $this->assertEquals($deleted['headers']['status-code'], 204); - - return $data; } /** From 8fb83d9605dcced707ef7af1df835c8d96616da1 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Tue, 28 Sep 2021 08:51:53 +0200 Subject: [PATCH 185/206] Fixed column width --- app/config/collections2.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/config/collections2.php b/app/config/collections2.php index 71dc7e9f69..72b8e00671 100644 --- a/app/config/collections2.php +++ b/app/config/collections2.php @@ -578,7 +578,7 @@ $collections = [ '$id' => 'email', 'type' => Database::VAR_STRING, 'format' => '', - 'size' => 1024, + 'size' => 320, 'signed' => true, 'required' => false, 'default' => null, @@ -712,14 +712,14 @@ $collections = [ '$id' => '_key_email', 'type' => Database::INDEX_UNIQUE, 'attributes' => ['email'], - 'lengths' => [1024], + 'lengths' => [320], 'orders' => [Database::ORDER_ASC], ], [ '$id' => '_key_deleted_email', 'type' => Database::INDEX_KEY, 'attributes' => ['deleted', 'email'], - 'lengths' => [0, 1024], + 'lengths' => [0, 320], 'orders' => [Database::ORDER_ASC, Database::ORDER_ASC], ], ], From 73c0b23680c310cdde8ab8010d9599cf07db1210 Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Thu, 30 Sep 2021 11:59:01 +0200 Subject: [PATCH 186/206] Swagger (2,3) support for multiple types of response --- app/controllers/web/home.php | 9 +- .../Specification/Format/OpenAPI3.php | 86 +++++++++++++------ .../Specification/Format/Swagger2.php | 59 ++++++++++--- 3 files changed, 114 insertions(+), 40 deletions(-) diff --git a/app/controllers/web/home.php b/app/controllers/web/home.php index 1d9e86d873..9f4d2412a8 100644 --- a/app/controllers/web/home.php +++ b/app/controllers/web/home.php @@ -387,11 +387,10 @@ App::get('/specs/:format') } $routes[] = $route; - $model = $response->getModel($route->getLabel('sdk.response.model', 'none')); - - if($model) { - $models[$model->getType()] = $model; - } + $modelLabel = $route->getLabel('sdk.response.model', 'none'); + $model = \is_array($modelLabel) ? \array_map(function($m) use($response) { + return $response->getModel($m); + }, $modelLabel) : $response->getModel($modelLabel); } } diff --git a/src/Appwrite/Specification/Format/OpenAPI3.php b/src/Appwrite/Specification/Format/OpenAPI3.php index 1d462669c9..b0a61114cc 100644 --- a/src/Appwrite/Specification/Format/OpenAPI3.php +++ b/src/Appwrite/Specification/Format/OpenAPI3.php @@ -6,6 +6,7 @@ use Appwrite\Specification\Format; use Appwrite\Template\Template; use stdClass; use Utopia\Validator; +use function gettype; class OpenAPI3 extends Format { @@ -25,14 +26,14 @@ class OpenAPI3 extends Format * Get Used Models * * Recursively get all used models - * + * * @param object $model * @param array $models * * @return void */ protected function getUsedModels($model, array &$usedModels) - { + { if (is_string($model) && !in_array($model, ['string', 'integer', 'boolean', 'json', 'float', 'double'])) { $usedModels[] = $model; return; @@ -99,7 +100,7 @@ class OpenAPI3 extends Format if (isset($output['components']['securitySchemes']['Project'])) { $output['components']['securitySchemes']['Project']['x-appwrite'] = ['demo' => '5df5acd0d48c2']; } - + if (isset($output['components']['securitySchemes']['Key'])) { $output['components']['securitySchemes']['Key']['x-appwrite'] = ['demo' => '919c2d18fb5d4...a2ae413da83346ad2']; } @@ -107,7 +108,7 @@ class OpenAPI3 extends Format if (isset($output['securityDefinitions']['JWT'])) { $output['securityDefinitions']['JWT']['x-appwrite'] = ['demo' => 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ...']; } - + if (isset($output['components']['securitySchemes']['Locale'])) { $output['components']['securitySchemes']['Locale']['x-appwrite'] = ['demo' => 'en']; } @@ -131,7 +132,7 @@ class OpenAPI3 extends Format $id = $route->getLabel('sdk.method', \uniqid()); $desc = (!empty($route->getLabel('sdk.description', ''))) ? \realpath(__DIR__.'/../../../../'.$route->getLabel('sdk.description', '')) : null; $produces = $route->getLabel('sdk.response.type', null); - $model = $route->getLabel('sdk.response.model', 'none'); + $model = $route->getLabel('sdk.response.model', 'none'); $routeSecurity = $route->getLabel('sdk.auth', []); $sdkPlatofrms = []; @@ -155,7 +156,7 @@ class OpenAPI3 extends Format if(empty($routeSecurity)) { $sdkPlatofrms[] = APP_PLATFORM_CLIENT; } - + $temp = [ 'summary' => $route->getDesc(), 'operationId' => $route->getLabel('sdk.namespace', 'default').ucfirst($id), @@ -181,13 +182,24 @@ class OpenAPI3 extends Format ]; foreach ($this->models as $key => $value) { - if($value->getType() === $model) { - $model = $value; - break; + if(\is_array($model)) { + $model = \array_map(function($m) use($value) { + if($m === $value->getType()) { + return $value; + } + + return $m; + }, $model); + } else { + if($value->getType() === $model) { + $model = $value; + break; + } } + } - if($model->isNone()) { + if(!(\is_array($model)) && $model->isNone()) { $temp['responses'][(string)$route->getLabel('sdk.response.code', '500')] = [ 'description' => (in_array($produces, [ 'image/*', @@ -204,17 +216,43 @@ class OpenAPI3 extends Format // ], ]; } else { - $usedModels[] = $model->getType(); - $temp['responses'][(string)$route->getLabel('sdk.response.code', '500')] = [ - 'description' => $model->getName(), - 'content' => [ - $produces => [ - 'schema' => [ - '$ref' => '#/components/schemas/'.$model->getType(), + if(\is_array($model)) { + $modelDescription = \join(', or ', \array_map(function ($m) { + return $m->getName(); + }, $model)); + + // model has multiple possible responses, we will use oneOf + foreach ($model as $m) { + $usedModels[] = $m->getType(); + } + + $temp['responses'][(string)$route->getLabel('sdk.response.code', '500')] = [ + 'description' => $modelDescription, + 'content' => [ + $produces => [ + 'schema' => [ + 'oneOf' => \array_map(function($m) { + return ['$ref' => '#/components/schemas/'.$m->getType()]; + }, $model) + ], ], ], - ], - ]; + ]; + } else { + // Response definition using one type + $usedModels[] = $model->getType(); + $temp['responses'][(string)$route->getLabel('sdk.response.code', '500')] = [ + 'description' => $model->getName(), + 'content' => [ + $produces => [ + 'schema' => [ + '$ref' => '#/components/schemas/'.$model->getType(), + ], + ], + ], + ]; + } + } if($route->getLabel('sdk.response.code', 500) === 204) { @@ -224,7 +262,7 @@ class OpenAPI3 extends Format if ((!empty($scope))) { // && 'public' != $scope $securities = ['Project' => []]; - + foreach($route->getLabel('sdk.auth', []) as $security) { if(array_key_exists($security, $this->keys)) { $securities[$security] = []; @@ -402,7 +440,7 @@ class OpenAPI3 extends Format if($model->isAny()) { $output['components']['schemas'][$model->getType()]['additionalProperties'] = true; } - + if(!empty($required)) { $output['components']['schemas'][$model->getType()]['required'] = $required; } @@ -417,7 +455,7 @@ class OpenAPI3 extends Format case 'json': $type = 'string'; break; - + case 'integer': $type = 'integer'; $format = 'int32'; @@ -432,11 +470,11 @@ class OpenAPI3 extends Format $type = 'number'; $format = 'double'; break; - + case 'boolean': $type = 'boolean'; break; - + default: $type = 'object'; $rule['type'] = ($rule['type']) ? $rule['type'] : 'none'; diff --git a/src/Appwrite/Specification/Format/Swagger2.php b/src/Appwrite/Specification/Format/Swagger2.php index 66324567bc..a8dcffb1e4 100644 --- a/src/Appwrite/Specification/Format/Swagger2.php +++ b/src/Appwrite/Specification/Format/Swagger2.php @@ -183,13 +183,22 @@ class Swagger2 extends Format } foreach ($this->models as $key => $value) { - if($value->getType() === $model) { - $model = $value; - break; + if(\is_array($model)) { + $model = \array_map(function($m) use($value) { + if($m === $value->getType()) { + return $value; + } + return $m; + }, $model); + } else { + if($value->getType() === $model) { + $model = $value; + break; + } } } - if($model->isNone()) { + if(!(\is_array($model)) && $model->isNone()) { $temp['responses'][(string)$route->getLabel('sdk.response.code', '500')] = [ 'description' => (in_array($produces, [ 'image/*', @@ -206,13 +215,41 @@ class Swagger2 extends Format ], ]; } else { - $usedModels[] = $model->getType(); - $temp['responses'][(string)$route->getLabel('sdk.response.code', '500')] = [ - 'description' => $model->getName(), - 'schema' => [ - '$ref' => '#/definitions/'.$model->getType(), - ], - ]; + + if(\is_array($model)) { + $modelDescription = \join(', or ', \array_map(function ($m) { + return $m->getName(); + }, $model)); + // model has multiple possible responses, we will use oneOf + foreach ($model as $m) { + $usedModels[] = $m->getType(); + } + $temp['responses'][(string)$route->getLabel('sdk.response.code', '500')] = [ + 'description' => $modelDescription, + 'content' => [ + $produces => [ + 'schema' => [ + 'oneOf' => \array_map(function($m) { + return ['$ref' => '#/definitions/'.$m->getType()]; + }, $model) + ], + ], + ], + ]; + } else { + // Response definition using one type + $usedModels[] = $model->getType(); + $temp['responses'][(string)$route->getLabel('sdk.response.code', '500')] = [ + 'description' => $model->getName(), + 'content' => [ + $produces => [ + 'schema' => [ + '$ref' => '#/definitions/'.$model->getType(), + ], + ], + ], + ]; + } } if(in_array($route->getLabel('sdk.response.code', 500), [204, 301, 302, 308], true)) { From dd04158ae1baad6e940393484f33e8e1261623f2 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Thu, 30 Sep 2021 15:03:18 -0400 Subject: [PATCH 187/206] Return oneOf models for getAttribute --- app/controllers/api/database.php | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index a031888ce1..2973adc404 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -1040,7 +1040,14 @@ App::get('/v1/database/collections/:collectionId/attributes/:attributeId') ->label('sdk.description', '/docs/references/database/get-attribute.md') ->label('sdk.response.code', Response::STATUS_CODE_OK) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) - ->label('sdk.response.model', Response::MODEL_ATTRIBUTE) + ->label('sdk.response.model', [ + Response::MODEL_ATTRIBUTE_BOOLEAN, + Response::MODEL_ATTRIBUTE_INTEGER, + Response::MODEL_ATTRIBUTE_FLOAT, + Response::MODEL_ATTRIBUTE_EMAIL, + Response::MODEL_ATTRIBUTE_URL, + Response::MODEL_ATTRIBUTE_IP, + Response::MODEL_ATTRIBUTE_STRING,])// needs to be last, since its condition would dominate any other string attribute ->param('collectionId', '', new UID(), 'Collection unique ID. You can create a new collection using the Database service [server integration](/docs/server/database#createCollection).') ->param('attributeId', '', new Key(), 'Attribute ID.') ->inject('response') From 7ca035960dc9019dcaca2b1fa95fe751ec59d155 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Thu, 30 Sep 2021 15:37:21 -0400 Subject: [PATCH 188/206] Fix attribute response model spec definitions --- app/controllers/api/database.php | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 0451287a2a..950095b111 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -663,7 +663,7 @@ App::post('/v1/database/collections/:collectionId/attributes/string') ->label('sdk.description', '/docs/references/database/create-attribute-string.md') ->label('sdk.response.code', Response::STATUS_CODE_CREATED) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) - ->label('sdk.response.model', Response::MODEL_ATTRIBUTE) + ->label('sdk.response.model', Response::MODEL_ATTRIBUTE_STRING) ->param('collectionId', '', new UID(), 'Collection unique ID. You can create a new collection using the Database service [server integration](/docs/server/database#createCollection).') ->param('attributeId', '', new Key(), 'Attribute ID.') ->param('size', null, new Integer(), 'Attribute size for text attributes, in number of characters.') @@ -711,7 +711,7 @@ App::post('/v1/database/collections/:collectionId/attributes/email') ->label('sdk.description', '/docs/references/database/create-attribute-email.md') ->label('sdk.response.code', Response::STATUS_CODE_CREATED) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) - ->label('sdk.response.model', Response::MODEL_ATTRIBUTE) + ->label('sdk.response.model', Response::MODEL_ATTRIBUTE_EMAIL) ->param('collectionId', '', new UID(), 'Collection unique ID. You can create a new collection using the Database service [server integration](/docs/server/database#createCollection).') ->param('attributeId', '', new Key(), 'Attribute ID.') ->param('required', null, new Boolean(), 'Is attribute required?') @@ -753,7 +753,7 @@ App::post('/v1/database/collections/:collectionId/attributes/ip') ->label('sdk.description', '/docs/references/database/create-attribute-ip.md') ->label('sdk.response.code', Response::STATUS_CODE_CREATED) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) - ->label('sdk.response.model', Response::MODEL_ATTRIBUTE) + ->label('sdk.response.model', Response::MODEL_ATTRIBUTE_IP) ->param('collectionId', '', new UID(), 'Collection unique ID. You can create a new collection using the Database service [server integration](/docs/server/database#createCollection).') ->param('attributeId', '', new Key(), 'Attribute ID.') ->param('required', null, new Boolean(), 'Is attribute required?') @@ -795,7 +795,7 @@ App::post('/v1/database/collections/:collectionId/attributes/url') ->label('sdk.description', '/docs/references/database/create-attribute-url.md') ->label('sdk.response.code', Response::STATUS_CODE_CREATED) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) - ->label('sdk.response.model', Response::MODEL_ATTRIBUTE) + ->label('sdk.response.model', Response::MODEL_ATTRIBUTE_URL) ->param('collectionId', '', new UID(), 'Collection unique ID. You can create a new collection using the Database service [server integration](/docs/server/database#createCollection).') ->param('attributeId', '', new Key(), 'Attribute ID.') ->param('required', null, new Boolean(), 'Is attribute required?') @@ -837,7 +837,7 @@ App::post('/v1/database/collections/:collectionId/attributes/integer') ->label('sdk.description', '/docs/references/database/create-attribute-integer.md') ->label('sdk.response.code', Response::STATUS_CODE_CREATED) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) - ->label('sdk.response.model', Response::MODEL_ATTRIBUTE) + ->label('sdk.response.model', Response::MODEL_ATTRIBUTE_INTEGER) ->param('collectionId', '', new UID(), 'Collection unique ID. You can create a new collection using the Database service [server integration](/docs/server/database#createCollection).') ->param('attributeId', '', new Key(), 'Attribute ID.') ->param('required', null, new Boolean(), 'Is attribute required?') @@ -901,7 +901,7 @@ App::post('/v1/database/collections/:collectionId/attributes/float') ->label('sdk.description', '/docs/references/database/create-attribute-float.md') ->label('sdk.response.code', Response::STATUS_CODE_CREATED) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) - ->label('sdk.response.model', Response::MODEL_ATTRIBUTE) + ->label('sdk.response.model', Response::MODEL_ATTRIBUTE_FLOAT) ->param('collectionId', '', new UID(), 'Collection unique ID. You can create a new collection using the Database service [server integration](/docs/server/database#createCollection).') ->param('attributeId', '', new Key(), 'Attribute ID.') ->param('required', null, new Boolean(), 'Is attribute required?') @@ -965,7 +965,7 @@ App::post('/v1/database/collections/:collectionId/attributes/boolean') ->label('sdk.description', '/docs/references/database/create-attribute-boolean.md') ->label('sdk.response.code', Response::STATUS_CODE_CREATED) ->label('sdk.response.type', Response::CONTENT_TYPE_JSON) - ->label('sdk.response.model', Response::MODEL_ATTRIBUTE) + ->label('sdk.response.model', Response::MODEL_ATTRIBUTE_BOOLEAN) ->param('collectionId', '', new UID(), 'Collection unique ID. You can create a new collection using the Database service [server integration](/docs/server/database#createCollection).') ->param('attributeId', '', new Key(), 'Attribute ID.') ->param('required', null, new Boolean(), 'Is attribute required?') From 6ada84cb53f91eb733cc729ed669c551075d95d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 4 Oct 2021 12:58:33 +0200 Subject: [PATCH 189/206] Update tests/e2e/Services/Users/UsersCustomServerTest.php Co-authored-by: kodumbeats --- tests/e2e/Services/Users/UsersCustomServerTest.php | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/e2e/Services/Users/UsersCustomServerTest.php b/tests/e2e/Services/Users/UsersCustomServerTest.php index 3acd4330af..9d7e60a730 100644 --- a/tests/e2e/Services/Users/UsersCustomServerTest.php +++ b/tests/e2e/Services/Users/UsersCustomServerTest.php @@ -36,9 +36,7 @@ class UsersCustomServerTest extends Scope $response = $this->client->call(Client::METHOD_DELETE, '/users/meldiron', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], - ], $this->getHeaders()), [ - - ]); + ], $this->getHeaders())); $this->assertEquals(204, $response['headers']['status-code']); From b068f4969cde06c8e5a7a095238f4642626bbc1d Mon Sep 17 00:00:00 2001 From: Matej Baco Date: Mon, 4 Oct 2021 13:01:08 +0200 Subject: [PATCH 190/206] Review changes --- docker-compose.yml | 2 +- tests/e2e/Services/Users/UsersCustomServerTest.php | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 27c2a125d9..ade61dc1f8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -63,7 +63,7 @@ services: - ./psalm.xml:/usr/src/code/psalm.xml - ./tests:/usr/src/code/tests - ./app:/usr/src/code/app - - ./vendor:/usr/src/code/vendor + # - ./vendor:/usr/src/code/vendor - ./docs:/usr/src/code/docs - ./public:/usr/src/code/public - ./src:/usr/src/code/src diff --git a/tests/e2e/Services/Users/UsersCustomServerTest.php b/tests/e2e/Services/Users/UsersCustomServerTest.php index 9d7e60a730..cee654e6c3 100644 --- a/tests/e2e/Services/Users/UsersCustomServerTest.php +++ b/tests/e2e/Services/Users/UsersCustomServerTest.php @@ -46,9 +46,9 @@ class UsersCustomServerTest extends Scope 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ 'userId' => 'meldiron', - 'email' => 'matej@appwrite.io', - 'password' => 'my-superstr0ng-password', - 'name' => 'Matej Bačo' + 'email' => 'matej2@appwrite.io', + 'password' => 'someones-superstr0ng-password', + 'name' => 'Matej Bačo Second' ]); $this->assertEquals(409, $response1['headers']['status-code']); From 7497540db0c9c7a48dbb618b2679e9d371e7067a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 4 Oct 2021 13:02:54 +0200 Subject: [PATCH 191/206] Update app/config/collections2.php Co-authored-by: kodumbeats --- app/config/collections2.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/config/collections2.php b/app/config/collections2.php index da79e3684c..bab2414aea 100644 --- a/app/config/collections2.php +++ b/app/config/collections2.php @@ -1735,7 +1735,7 @@ $collections = [ '$id' => '_fulltext_search', 'type' => Database::INDEX_FULLTEXT, 'attributes' => ['search'], - 'lengths' => [1024], + 'lengths' => [16384], 'orders' => [Database::ORDER_ASC], ], ], From 22132ef42004cc56b7a846444dcd82f6636c6199 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Tue, 5 Oct 2021 09:05:19 -0400 Subject: [PATCH 192/206] Update to utopia-php/database:0.10.0 --- composer.json | 11 +++------ composer.lock | 63 ++++++++++++++++++++++++--------------------------- 2 files changed, 33 insertions(+), 41 deletions(-) diff --git a/composer.json b/composer.json index b251fb13a9..b3d21a4b02 100644 --- a/composer.json +++ b/composer.json @@ -45,7 +45,7 @@ "utopia-php/cache": "0.4.*", "utopia-php/cli": "0.11.*", "utopia-php/config": "0.2.*", - "utopia-php/database": "dev-feat-adjust-encodeAttribute as 0.10.0", + "utopia-php/database": "0.10.0", "utopia-php/locale": "0.4.*", "utopia-php/registry": "0.5.*", "utopia-php/preloader": "0.2.*", @@ -63,12 +63,7 @@ "adhocore/jwt": "1.1.2", "slickdeals/statsd": "3.1.0" }, - "repositories": [ - { - "type": "git", - "url": "https://github.com/utopia-php/database" - } - ], + "repositories": [], "require-dev": { "appwrite/sdk-generator": "0.13.0", "swoole/ide-helper": "4.6.7", @@ -83,4 +78,4 @@ "php": "8.0" } } -} \ No newline at end of file +} diff --git a/composer.lock b/composer.lock index 27c2af756b..7eee04c864 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "31670d6cc60a22007b4ed59a7dc9448f", + "content-hash": "dfb8fa19daa736b3687617c98f309983", "packages": [ { "name": "adhocore/jwt", @@ -1984,11 +1984,17 @@ }, { "name": "utopia-php/database", - "version": "dev-feat-adjust-encodeAttribute", + "version": "0.10.0", "source": { "type": "git", - "url": "https://github.com/utopia-php/database", - "reference": "5ef32ec85143daf78796e7826453244d24f8a86a" + "url": "https://github.com/utopia-php/database.git", + "reference": "b7c60b0ec769a9050dd2b939b78ff1f5d4fa27e8" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/database/zipball/b7c60b0ec769a9050dd2b939b78ff1f5d4fa27e8", + "reference": "b7c60b0ec769a9050dd2b939b78ff1f5d4fa27e8", + "shasum": "" }, "require": { "ext-mongodb": "*", @@ -2011,11 +2017,7 @@ "Utopia\\Database\\": "src/Database" } }, - "autoload-dev": { - "psr-4": { - "Utopia\\Tests\\": "tests/Database" - } - }, + "notification-url": "https://packagist.org/downloads/", "license": [ "MIT" ], @@ -2037,7 +2039,11 @@ "upf", "utopia" ], - "time": "2021-08-27T20:39:51+00:00" + "support": { + "issues": "https://github.com/utopia-php/database/issues", + "source": "https://github.com/utopia-php/database/tree/0.10.0" + }, + "time": "2021-10-04T17:23:25+00:00" }, { "name": "utopia-php/domains", @@ -2576,16 +2582,16 @@ "packages-dev": [ { "name": "amphp/amp", - "version": "v2.6.0", + "version": "v2.6.1", "source": { "type": "git", "url": "https://github.com/amphp/amp.git", - "reference": "caa95edeb1ca1bf7532e9118ede4a3c3126408cc" + "reference": "c5fc66a78ee38d7ac9195a37bacaf940eb3f65ae" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/amphp/amp/zipball/caa95edeb1ca1bf7532e9118ede4a3c3126408cc", - "reference": "caa95edeb1ca1bf7532e9118ede4a3c3126408cc", + "url": "https://api.github.com/repos/amphp/amp/zipball/c5fc66a78ee38d7ac9195a37bacaf940eb3f65ae", + "reference": "c5fc66a78ee38d7ac9195a37bacaf940eb3f65ae", "shasum": "" }, "require": { @@ -2653,7 +2659,7 @@ "support": { "irc": "irc://irc.freenode.org/amphp", "issues": "https://github.com/amphp/amp/issues", - "source": "https://github.com/amphp/amp/tree/v2.6.0" + "source": "https://github.com/amphp/amp/tree/v2.6.1" }, "funding": [ { @@ -2661,7 +2667,7 @@ "type": "github" } ], - "time": "2021-07-16T20:06:06+00:00" + "time": "2021-09-23T18:43:08+00:00" }, { "name": "amphp/byte-stream", @@ -3712,16 +3718,16 @@ }, { "name": "phpdocumentor/type-resolver", - "version": "1.5.0", + "version": "1.5.1", "source": { "type": "git", "url": "https://github.com/phpDocumentor/TypeResolver.git", - "reference": "30f38bffc6f24293dadd1823936372dfa9e86e2f" + "reference": "a12f7e301eb7258bb68acd89d4aefa05c2906cae" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/30f38bffc6f24293dadd1823936372dfa9e86e2f", - "reference": "30f38bffc6f24293dadd1823936372dfa9e86e2f", + "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/a12f7e301eb7258bb68acd89d4aefa05c2906cae", + "reference": "a12f7e301eb7258bb68acd89d4aefa05c2906cae", "shasum": "" }, "require": { @@ -3756,9 +3762,9 @@ "description": "A PSR-5 based resolver of Class names, Types and Structural Element Names", "support": { "issues": "https://github.com/phpDocumentor/TypeResolver/issues", - "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.5.0" + "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.5.1" }, - "time": "2021-09-17T15:28:14+00:00" + "time": "2021-10-02T14:08:47+00:00" }, { "name": "phpspec/prophecy", @@ -6249,18 +6255,9 @@ "time": "2015-12-17T08:42:14+00:00" } ], - "aliases": [ - { - "package": "utopia-php/database", - "version": "dev-feat-adjust-encodeAttribute", - "alias": "0.10.0", - "alias_normalized": "0.10.0.0" - } - ], + "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/database": 20 - }, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { From 481ff1e2f969eecc306cbe5d360595a0ffe459c0 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Tue, 5 Oct 2021 09:05:40 -0400 Subject: [PATCH 193/206] Refactor purgeDocument calls to deleteCachedDocument --- app/controllers/api/database.php | 8 ++++---- app/workers/database.php | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index 6d7268c249..0b288a9ab8 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -96,7 +96,7 @@ function createAttribute($collectionId, $attribute, $response, $dbForInternal, $ throw new Exception('Attribute already exists', 409); } - $dbForInternal->purgeDocument('collections', $collectionId); + $dbForInternal->deleteCachedDocument('collections', $collectionId); // Pass clone of $attribute object to workers // so we can later modify Document to fit response model @@ -1131,7 +1131,7 @@ App::delete('/v1/database/collections/:collectionId/attributes/:attributeId') } $attribute = $dbForInternal->updateDocument('attributes', $attribute->getId(), $attribute->setAttribute('status', 'deleting')); - $dbForInternal->purgeDocument('collections', $collectionId); + $dbForInternal->deleteCachedDocument('collections', $collectionId); $database ->setParam('type', DATABASE_TYPE_DELETE_ATTRIBUTE) @@ -1238,7 +1238,7 @@ App::post('/v1/database/collections/:collectionId/indexes') throw new Exception('Index already exists', 409); } - $dbForInternal->purgeDocument('collections', $collectionId); + $dbForInternal->deleteCachedDocument('collections', $collectionId); $database ->setParam('type', DATABASE_TYPE_CREATE_INDEX) @@ -1383,7 +1383,7 @@ App::delete('/v1/database/collections/:collectionId/indexes/:indexId') } $index = $dbForInternal->updateDocument('indexes', $index->getId(), $index->setAttribute('status', 'deleting')); - $dbForInternal->purgeDocument('collections', $collectionId); + $dbForInternal->deleteCachedDocument('collections', $collectionId); $database ->setParam('type', DATABASE_TYPE_DELETE_INDEX) diff --git a/app/workers/database.php b/app/workers/database.php index 2366b37998..fb841c6445 100644 --- a/app/workers/database.php +++ b/app/workers/database.php @@ -94,7 +94,7 @@ class DatabaseV1 extends Worker $dbForInternal->updateDocument('attributes', $attribute->getId(), $attribute->setAttribute('status', 'failed')); } - $dbForInternal->purgeDocument('collections', $collectionId); + $dbForInternal->deleteCachedDocument('collections', $collectionId); } /** @@ -120,7 +120,7 @@ class DatabaseV1 extends Worker $dbForInternal->updateDocument('attributes', $attribute->getId(), $attribute->setAttribute('status', 'failed')); } - $dbForInternal->purgeDocument('collections', $collectionId); + $dbForInternal->deleteCachedDocument('collections', $collectionId); } /** @@ -150,7 +150,7 @@ class DatabaseV1 extends Worker $dbForInternal->updateDocument('indexes', $index->getId(), $index->setAttribute('status', 'failed')); } - $dbForInternal->purgeDocument('collections', $collectionId); + $dbForInternal->deleteCachedDocument('collections', $collectionId); } /** @@ -177,6 +177,6 @@ class DatabaseV1 extends Worker $dbForInternal->updateDocument('indexes', $index->getId(), $index->setAttribute('status', 'failed')); } - $dbForInternal->purgeDocument('collections', $collectionId); + $dbForInternal->deleteCachedDocument('collections', $collectionId); } } From dc06e42348e4f0ae5f7ad9355ab13caf9d47a2dc Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Tue, 5 Oct 2021 09:16:27 -0400 Subject: [PATCH 194/206] Remove standard lib imports --- src/Appwrite/Specification/Format/OpenAPI3.php | 2 -- src/Appwrite/Utopia/Response/Model/Collection.php | 1 - 2 files changed, 3 deletions(-) diff --git a/src/Appwrite/Specification/Format/OpenAPI3.php b/src/Appwrite/Specification/Format/OpenAPI3.php index b0a61114cc..8d375d2f9f 100644 --- a/src/Appwrite/Specification/Format/OpenAPI3.php +++ b/src/Appwrite/Specification/Format/OpenAPI3.php @@ -4,9 +4,7 @@ namespace Appwrite\Specification\Format; use Appwrite\Specification\Format; use Appwrite\Template\Template; -use stdClass; use Utopia\Validator; -use function gettype; class OpenAPI3 extends Format { diff --git a/src/Appwrite/Utopia/Response/Model/Collection.php b/src/Appwrite/Utopia/Response/Model/Collection.php index e52539691d..e46603fd9a 100644 --- a/src/Appwrite/Utopia/Response/Model/Collection.php +++ b/src/Appwrite/Utopia/Response/Model/Collection.php @@ -5,7 +5,6 @@ namespace Appwrite\Utopia\Response\Model; use Appwrite\Utopia\Response; use Appwrite\Utopia\Response\Model; use Utopia\Database\Document; -use stdClass; class Collection extends Model { From 0f5931555aa3024e2d1b6e68b4d9610a5e2a24d2 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Tue, 5 Oct 2021 09:57:57 -0400 Subject: [PATCH 195/206] Fix issues from merge --- app/controllers/api/database.php | 5 ++--- src/Appwrite/Utopia/Response/Model/Collection.php | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/app/controllers/api/database.php b/app/controllers/api/database.php index ecbc896d01..255293c040 100644 --- a/app/controllers/api/database.php +++ b/app/controllers/api/database.php @@ -77,7 +77,7 @@ function createAttribute($collectionId, $attribute, $response, $dbForInternal, $ } try { - $attribute = $dbForInternal->createDocument('attributes', new Document([ + $attribute = new Document([ '$id' => $collectionId.'_'.$attributeId, 'key' => $attributeId, 'collectionId' => $collectionId, @@ -90,9 +90,8 @@ function createAttribute($collectionId, $attribute, $response, $dbForInternal, $ 'array' => $array, 'format' => $format, 'formatOptions' => $formatOptions, - ]); + ]); - try { $dbForInternal->checkAttribute($collection, $attribute); $attribute = $dbForInternal->createDocument('attributes', $attribute); } diff --git a/src/Appwrite/Utopia/Response/Model/Collection.php b/src/Appwrite/Utopia/Response/Model/Collection.php index e46603fd9a..331dd48b0a 100644 --- a/src/Appwrite/Utopia/Response/Model/Collection.php +++ b/src/Appwrite/Utopia/Response/Model/Collection.php @@ -55,14 +55,14 @@ class Collection extends Model ], 'description' => 'Collection attributes.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true, ]) ->addRule('indexes', [ 'type' => Response::MODEL_INDEX, 'description' => 'Collection indexes.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true ]) ; From 0960a54169364e4f219f6d1db57716c612996405 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Tue, 5 Oct 2021 10:10:40 -0400 Subject: [PATCH 196/206] Clean up unneeded sleeps in db tests --- tests/e2e/Services/Database/DatabaseBase.php | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/tests/e2e/Services/Database/DatabaseBase.php b/tests/e2e/Services/Database/DatabaseBase.php index 96339b1fd2..4111aca118 100644 --- a/tests/e2e/Services/Database/DatabaseBase.php +++ b/tests/e2e/Services/Database/DatabaseBase.php @@ -654,8 +654,6 @@ trait DatabaseBase 'required' => false, ]); - sleep(2); - $ip = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/attributes/ip', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], @@ -665,8 +663,6 @@ trait DatabaseBase 'required' => false, ]); - sleep(2); - $url = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/attributes/url', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], @@ -677,8 +673,6 @@ trait DatabaseBase 'required' => false, ]); - sleep(2); - $range = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/attributes/integer', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], @@ -690,8 +684,6 @@ trait DatabaseBase 'max' => 10, ]); - sleep(2); - // TODO@kodumbeats min and max are rounded in error message $floatRange = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/attributes/float', array_merge([ 'content-type' => 'application/json', @@ -704,8 +696,6 @@ trait DatabaseBase 'max' => 1.4, ]); - sleep(2); - // TODO@kodumbeats float validator rejects 0.0 and 1.0 as floats // $probability = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/attributes/float', array_merge([ // 'content-type' => 'application/json', @@ -728,8 +718,6 @@ trait DatabaseBase 'max' => 10, ]); - sleep(2); - $lowerBound = $this->client->call(Client::METHOD_POST, '/database/collections/' . $collectionId . '/attributes/integer', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], @@ -766,7 +754,7 @@ trait DatabaseBase // $this->assertEquals('Minimum value must be lesser than maximum value', $invalidRange['body']['message']); // wait for worker to add attributes - sleep(2); + sleep(3); $collection = $this->client->call(Client::METHOD_GET, '/database/collections/' . $collectionId, array_merge([ 'content-type' => 'application/json', From bc0a15e38bbd356275ee02c21a84cd9720a725ab Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Tue, 5 Oct 2021 16:27:06 +0200 Subject: [PATCH 197/206] fix usage worker --- app/tasks/usage.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 4ff5e8cad9..701037fcca 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -342,7 +342,7 @@ $cli do { // list projects try { $attempts++; - $projects = $dbForConsole->find('projects', [], 100, orderAfter:$latestProject); + $projects = $dbForConsole->find('projects', [], 100, cursor:$latestProject); break; // leave the do-while if successful } catch (\Exception $e) { Console::warning("Console DB not ready yet. Retrying ({$attempts})..."); @@ -472,7 +472,7 @@ $cli do { // Loop over all the parent collection document for each sub collection $dbForProject->setNamespace("project_{$projectId}_{$options['namespace']}"); - $parents = $dbForProject->find($collection, [], 100, orderAfter:$latestParent); // Get all the parents for the sub collections for example for documents, this will get all the collections + $parents = $dbForProject->find($collection, [], 100, cursor:$latestParent); // Get all the parents for the sub collections for example for documents, this will get all the collections if (empty($parents)) { continue; From 88ef9dfd3c8ebe6f2a70bf3885f5a256a01d9203 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Tue, 5 Oct 2021 10:29:43 -0400 Subject: [PATCH 198/206] style - pass clone of user object to workers --- app/controllers/api/users.php | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 20266d1fda..5023ccd040 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -719,26 +719,31 @@ App::delete('/v1/users/:userId') throw new Exception('User not found', 404); } - $emptyUser = clone $user; - $emptyUser->setAttribute("name", null); - $emptyUser->setAttribute("email", null); - $emptyUser->setAttribute("password", null); - $emptyUser->setAttribute("deleted", true); + // clone user object to send to workers + $clone = clone $user; - $dbForInternal->updateDocument('users', $userId, $emptyUser); + $user + ->setAttribute("name", null) + ->setAttribute("email", null); + ->setAttribute("password", null); + ->setAttribute("deleted", true) + ; + + $dbForInternal->updateDocument('users', $userId, $user); $deletes ->setParam('type', DELETE_TYPE_DOCUMENT) - ->setParam('document', $user) + ->setParam('document', $clone) ; $events - ->setParam('eventData', $response->output($user, Response::MODEL_USER)) + ->setParam('eventData', $response->output($clone, Response::MODEL_USER)) ; $usage ->setParam('users.delete', 1) ; + $response->noContent(); }); From cdaec34c6a5427c0926dfd6b02dceb6f040bb541 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Tue, 5 Oct 2021 16:36:34 +0200 Subject: [PATCH 199/206] fix usage worker --- app/tasks/usage.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 4ff5e8cad9..701037fcca 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -342,7 +342,7 @@ $cli do { // list projects try { $attempts++; - $projects = $dbForConsole->find('projects', [], 100, orderAfter:$latestProject); + $projects = $dbForConsole->find('projects', [], 100, cursor:$latestProject); break; // leave the do-while if successful } catch (\Exception $e) { Console::warning("Console DB not ready yet. Retrying ({$attempts})..."); @@ -472,7 +472,7 @@ $cli do { // Loop over all the parent collection document for each sub collection $dbForProject->setNamespace("project_{$projectId}_{$options['namespace']}"); - $parents = $dbForProject->find($collection, [], 100, orderAfter:$latestParent); // Get all the parents for the sub collections for example for documents, this will get all the collections + $parents = $dbForProject->find($collection, [], 100, cursor:$latestParent); // Get all the parents for the sub collections for example for documents, this will get all the collections if (empty($parents)) { continue; From 41857f0bda8c65df0817cec29308badee57bf2c7 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Tue, 5 Oct 2021 16:37:19 +0200 Subject: [PATCH 200/206] fix usage worker --- app/tasks/usage.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/tasks/usage.php b/app/tasks/usage.php index 4ff5e8cad9..701037fcca 100644 --- a/app/tasks/usage.php +++ b/app/tasks/usage.php @@ -342,7 +342,7 @@ $cli do { // list projects try { $attempts++; - $projects = $dbForConsole->find('projects', [], 100, orderAfter:$latestProject); + $projects = $dbForConsole->find('projects', [], 100, cursor:$latestProject); break; // leave the do-while if successful } catch (\Exception $e) { Console::warning("Console DB not ready yet. Retrying ({$attempts})..."); @@ -472,7 +472,7 @@ $cli do { // Loop over all the parent collection document for each sub collection $dbForProject->setNamespace("project_{$projectId}_{$options['namespace']}"); - $parents = $dbForProject->find($collection, [], 100, orderAfter:$latestParent); // Get all the parents for the sub collections for example for documents, this will get all the collections + $parents = $dbForProject->find($collection, [], 100, cursor:$latestParent); // Get all the parents for the sub collections for example for documents, this will get all the collections if (empty($parents)) { continue; From c3d1dad82cdaaeb95f0fc42ed87de76c240bf3a4 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Tue, 5 Oct 2021 17:39:39 +0200 Subject: [PATCH 201/206] fix stdClass usage --- app/controllers/api/projects.php | 2 +- .../Specification/Format/OpenAPI3.php | 2 +- .../Specification/Format/Swagger2.php | 3 +-- src/Appwrite/Utopia/Response.php | 3 +-- .../Utopia/Response/Model/Collection.php | 5 ++--- .../Utopia/Response/Model/Project.php | 9 ++++---- .../Utopia/Response/Model/UsageBuckets.php | 11 +++++----- .../Utopia/Response/Model/UsageCollection.php | 11 +++++----- .../Utopia/Response/Model/UsageDatabase.php | 21 +++++++++---------- .../Utopia/Response/Model/UsageFunctions.php | 7 +++---- .../Utopia/Response/Model/UsageProject.php | 15 +++++++------ .../Utopia/Response/Model/UsageStorage.php | 5 ++--- .../Utopia/Response/Model/UsageUsers.php | 17 +++++++-------- 13 files changed, 50 insertions(+), 61 deletions(-) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index 3c7dbb8e5e..4baf99e084 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -94,7 +94,7 @@ App::post('/v1/projects') 'legalCity' => $legalCity, 'legalAddress' => $legalAddress, 'legalTaxId' => $legalTaxId, - 'services' => new stdClass(), + 'services' => new \stdClass(), 'platforms' => [], 'webhooks' => [], 'keys' => [], diff --git a/src/Appwrite/Specification/Format/OpenAPI3.php b/src/Appwrite/Specification/Format/OpenAPI3.php index 8d375d2f9f..863a7a9627 100644 --- a/src/Appwrite/Specification/Format/OpenAPI3.php +++ b/src/Appwrite/Specification/Format/OpenAPI3.php @@ -319,7 +319,7 @@ class OpenAPI3 extends Format case 'Utopia\Validator\JSON': case 'Utopia\Validator\Mock': case 'Utopia\Validator\Assoc': - $param['default'] = (empty($param['default'])) ? new stdClass() : $param['default']; + $param['default'] = (empty($param['default'])) ? new \stdClass() : $param['default']; $node['schema']['type'] = 'object'; $node['schema']['x-example'] = '{}'; //$node['schema']['format'] = 'json'; diff --git a/src/Appwrite/Specification/Format/Swagger2.php b/src/Appwrite/Specification/Format/Swagger2.php index a8dcffb1e4..3ab5f184a2 100644 --- a/src/Appwrite/Specification/Format/Swagger2.php +++ b/src/Appwrite/Specification/Format/Swagger2.php @@ -4,7 +4,6 @@ namespace Appwrite\Specification\Format; use Appwrite\Specification\Format; use Appwrite\Template\Template; -use stdClass; use Utopia\Validator; class Swagger2 extends Format @@ -317,7 +316,7 @@ class Swagger2 extends Format case 'Utopia\Validator\Mock': case 'Utopia\Validator\Assoc': $node['type'] = 'object'; - $param['default'] = (empty($param['default'])) ? new stdClass() : $param['default']; + $param['default'] = (empty($param['default'])) ? new \stdClass() : $param['default']; $node['x-example'] = '{}'; //$node['format'] = 'json'; break; diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index 8258758353..f92023d8dd 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -59,7 +59,6 @@ use Appwrite\Utopia\Response\Model\UsageFunctions; use Appwrite\Utopia\Response\Model\UsageProject; use Appwrite\Utopia\Response\Model\UsageStorage; use Appwrite\Utopia\Response\Model\UsageUsers; -use stdClass; /** * @method Response public function setStatusCode(int $code = 200) @@ -328,7 +327,7 @@ class Response extends SwooleResponse $output = self::getFilter()->parse($output, $model); } - $this->json(!empty($output) ? $output : new stdClass()); + $this->json(!empty($output) ? $output : new \stdClass()); } /** diff --git a/src/Appwrite/Utopia/Response/Model/Collection.php b/src/Appwrite/Utopia/Response/Model/Collection.php index e46603fd9a..b2a05846c5 100644 --- a/src/Appwrite/Utopia/Response/Model/Collection.php +++ b/src/Appwrite/Utopia/Response/Model/Collection.php @@ -4,7 +4,6 @@ namespace Appwrite\Utopia\Response\Model; use Appwrite\Utopia\Response; use Appwrite\Utopia\Response\Model; -use Utopia\Database\Document; class Collection extends Model { @@ -55,14 +54,14 @@ class Collection extends Model ], 'description' => 'Collection attributes.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true, ]) ->addRule('indexes', [ 'type' => Response::MODEL_INDEX, 'description' => 'Collection indexes.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true ]) ; diff --git a/src/Appwrite/Utopia/Response/Model/Project.php b/src/Appwrite/Utopia/Response/Model/Project.php index adbd922666..ed2fd60f14 100644 --- a/src/Appwrite/Utopia/Response/Model/Project.php +++ b/src/Appwrite/Utopia/Response/Model/Project.php @@ -2,7 +2,6 @@ namespace Appwrite\Utopia\Response\Model; -use stdClass; use Appwrite\Utopia\Response; use Appwrite\Utopia\Response\Model; use Utopia\Config\Config; @@ -100,28 +99,28 @@ class Project extends Model 'type' => Response::MODEL_PLATFORM, 'description' => 'List of Platforms.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true, ]) ->addRule('webhooks', [ 'type' => Response::MODEL_WEBHOOK, 'description' => 'List of Webhooks.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true, ]) ->addRule('keys', [ 'type' => Response::MODEL_KEY, 'description' => 'List of API Keys.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true, ]) ->addRule('domains', [ 'type' => Response::MODEL_DOMAIN, 'description' => 'List of Domains.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true, ]) ; diff --git a/src/Appwrite/Utopia/Response/Model/UsageBuckets.php b/src/Appwrite/Utopia/Response/Model/UsageBuckets.php index 8c66ff47c4..cafa679ab4 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageBuckets.php +++ b/src/Appwrite/Utopia/Response/Model/UsageBuckets.php @@ -4,7 +4,6 @@ namespace Appwrite\Utopia\Response\Model; use Appwrite\Utopia\Response; use Appwrite\Utopia\Response\Model; -use stdClass; class UsageBuckets extends Model { @@ -21,35 +20,35 @@ class UsageBuckets extends Model 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for total number of files in this bucket.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true ]) ->addRule('files.create', [ 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for files created.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true ]) ->addRule('files.read', [ 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for files read.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true ]) ->addRule('files.update', [ 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for files updated.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true ]) ->addRule('files.delete', [ 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for files deleted.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true ]) ; diff --git a/src/Appwrite/Utopia/Response/Model/UsageCollection.php b/src/Appwrite/Utopia/Response/Model/UsageCollection.php index f2815831a2..cf9e3e4541 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageCollection.php +++ b/src/Appwrite/Utopia/Response/Model/UsageCollection.php @@ -4,7 +4,6 @@ namespace Appwrite\Utopia\Response\Model; use Appwrite\Utopia\Response; use Appwrite\Utopia\Response\Model; -use stdClass; class UsageCollection extends Model { @@ -21,35 +20,35 @@ class UsageCollection extends Model 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for total number of documents.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true ]) ->addRule('documents.create', [ 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for documents created.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true ]) ->addRule('documents.read', [ 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for documents read.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true ]) ->addRule('documents.update', [ 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for documents updated.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true ]) ->addRule('documents.delete', [ 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for documents deleted.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true ]) ; diff --git a/src/Appwrite/Utopia/Response/Model/UsageDatabase.php b/src/Appwrite/Utopia/Response/Model/UsageDatabase.php index 2ebe0b64c0..eab025ba14 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageDatabase.php +++ b/src/Appwrite/Utopia/Response/Model/UsageDatabase.php @@ -4,7 +4,6 @@ namespace Appwrite\Utopia\Response\Model; use Appwrite\Utopia\Response; use Appwrite\Utopia\Response\Model; -use stdClass; class UsageDatabase extends Model { @@ -21,70 +20,70 @@ class UsageDatabase extends Model 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for total number of documents.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true ]) ->addRule('collections.count', [ 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for total number of collections.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true ]) ->addRule('documents.create', [ 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for documents created.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true ]) ->addRule('documents.read', [ 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for documents read.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true ]) ->addRule('documents.update', [ 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for documents updated.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true ]) ->addRule('documents.delete', [ 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for documents deleted.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true ]) ->addRule('collections.create', [ 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for collections created.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true ]) ->addRule('collections.read', [ 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for collections read.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true ]) ->addRule('collections.update', [ 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for collections updated.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true ]) ->addRule('collections.delete', [ 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for collections delete.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true ]) ; diff --git a/src/Appwrite/Utopia/Response/Model/UsageFunctions.php b/src/Appwrite/Utopia/Response/Model/UsageFunctions.php index 625bdab71c..6183b1aff8 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageFunctions.php +++ b/src/Appwrite/Utopia/Response/Model/UsageFunctions.php @@ -4,7 +4,6 @@ namespace Appwrite\Utopia\Response\Model; use Appwrite\Utopia\Response; use Appwrite\Utopia\Response\Model; -use stdClass; class UsageFunctions extends Model { @@ -21,21 +20,21 @@ class UsageFunctions extends Model 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for function executions.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true ]) ->addRule('functions.failures', [ 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for function execution failures.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true ]) ->addRule('functions.compute', [ 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for function execution duration.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true ]) ; diff --git a/src/Appwrite/Utopia/Response/Model/UsageProject.php b/src/Appwrite/Utopia/Response/Model/UsageProject.php index c44af4cf6d..5593def1d8 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageProject.php +++ b/src/Appwrite/Utopia/Response/Model/UsageProject.php @@ -4,7 +4,6 @@ namespace Appwrite\Utopia\Response\Model; use Appwrite\Utopia\Response; use Appwrite\Utopia\Response\Model; -use stdClass; class UsageProject extends Model { @@ -21,49 +20,49 @@ class UsageProject extends Model 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for number of requests.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true ]) ->addRule('network', [ 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for consumed bandwidth.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true ]) ->addRule('functions', [ 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for function executions.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true ]) ->addRule('documents', [ 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for number of documents.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true ]) ->addRule('collections', [ 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for number of collections.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true ]) ->addRule('users', [ 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for number of users.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true ]) ->addRule('storage', [ 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for the occupied storage size (in bytes).', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true ]) ; diff --git a/src/Appwrite/Utopia/Response/Model/UsageStorage.php b/src/Appwrite/Utopia/Response/Model/UsageStorage.php index 4462a219a9..db40ccf3be 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageStorage.php +++ b/src/Appwrite/Utopia/Response/Model/UsageStorage.php @@ -4,7 +4,6 @@ namespace Appwrite\Utopia\Response\Model; use Appwrite\Utopia\Response; use Appwrite\Utopia\Response\Model; -use stdClass; class UsageStorage extends Model { @@ -21,14 +20,14 @@ class UsageStorage extends Model 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for the occupied storage size (in bytes).', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true ]) ->addRule('files', [ 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for total number of files.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true ]) ; diff --git a/src/Appwrite/Utopia/Response/Model/UsageUsers.php b/src/Appwrite/Utopia/Response/Model/UsageUsers.php index 8813615062..3854fdec75 100644 --- a/src/Appwrite/Utopia/Response/Model/UsageUsers.php +++ b/src/Appwrite/Utopia/Response/Model/UsageUsers.php @@ -4,7 +4,6 @@ namespace Appwrite\Utopia\Response\Model; use Appwrite\Utopia\Response; use Appwrite\Utopia\Response\Model; -use stdClass; class UsageUsers extends Model { @@ -21,56 +20,56 @@ class UsageUsers extends Model 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for total number of users.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true ]) ->addRule('users.create', [ 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for users created.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true ]) ->addRule('users.read', [ 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for users read.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true ]) ->addRule('users.update', [ 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for users updated.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true ]) ->addRule('users.delete', [ 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for users deleted.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true ]) ->addRule('sessions.create', [ 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for sessions created.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true ]) ->addRule('sessions.provider.create', [ 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for sessions created for a provider ( email, anonymous or oauth2 ).', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true ]) ->addRule('sessions.delete', [ 'type' => Response::MODEL_METRIC_LIST, 'description' => 'Aggregated stats for sessions deleted.', 'default' => [], - 'example' => new stdClass, + 'example' => new \stdClass, 'array' => true ]) ; From 3d6de5da4774f60719526f6ebe2a8306bb435158 Mon Sep 17 00:00:00 2001 From: kodumbeats Date: Tue, 5 Oct 2021 12:02:54 -0400 Subject: [PATCH 202/206] Throw correct response code for duplicate user --- app/controllers/api/account.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index e01bb07314..ac7179972b 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -1140,7 +1140,7 @@ App::patch('/v1/account/email') $profile = $dbForInternal->findOne('users', [new Query('email', Query::TYPE_EQUAL, [$email])]); // Get user by email address if ($profile) { - throw new Exception('User already registered', 400); + throw new Exception('User already registered', 409); } try { From 895da9dbef9c65a244b1b899145f6d65beadae80 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Tue, 5 Oct 2021 22:37:17 +0200 Subject: [PATCH 203/206] fix semi-colons --- app/controllers/api/users.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 5023ccd040..86319ea5ec 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -724,8 +724,8 @@ App::delete('/v1/users/:userId') $user ->setAttribute("name", null) - ->setAttribute("email", null); - ->setAttribute("password", null); + ->setAttribute("email", null) + ->setAttribute("password", null) ->setAttribute("deleted", true) ; From 4a054559a54ddc43601caeaf5a5af11e92646b5d Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Tue, 5 Oct 2021 23:15:43 +0200 Subject: [PATCH 204/206] fix search with cristiano --- app/controllers/api/users.php | 2 +- tests/e2e/Services/Users/UsersBase.php | 67 +++++++++++--------------- 2 files changed, 28 insertions(+), 41 deletions(-) diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index a688572730..7adaa747c2 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -111,7 +111,7 @@ App::get('/v1/users') throw new Exception("User '{$after}' for the 'after' value not found.", 400); } } - + $queries = []; if (!empty($search)) { diff --git a/tests/e2e/Services/Users/UsersBase.php b/tests/e2e/Services/Users/UsersBase.php index 7db500ddc8..ee35e66fc3 100644 --- a/tests/e2e/Services/Users/UsersBase.php +++ b/tests/e2e/Services/Users/UsersBase.php @@ -16,14 +16,14 @@ trait UsersBase 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ 'userId' => 'unique()', - 'email' => 'first.user@example.com', + 'email' => 'cristiano.ronaldo@manchester-united.co.uk', 'password' => 'password', - 'name' => 'First Name', + 'name' => 'Cristiano Ronaldo', ]); $this->assertEquals($user['headers']['status-code'], 201); - $this->assertEquals($user['body']['name'], 'First Name'); - $this->assertEquals($user['body']['email'], 'first.user@example.com'); + $this->assertEquals($user['body']['name'], 'Cristiano Ronaldo'); + $this->assertEquals($user['body']['email'], 'cristiano.ronaldo@manchester-united.co.uk'); $this->assertEquals($user['body']['status'], true); $this->assertGreaterThan(0, $user['body']['registration']); @@ -35,15 +35,15 @@ trait UsersBase 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ 'userId' => 'user1', - 'email' => 'users.service1@example.com', + 'email' => 'lionel.messi@psg.fr', 'password' => 'password', - 'name' => 'Project User', + 'name' => 'Lionel Messi', ]); $this->assertEquals($res['headers']['status-code'], 201); $this->assertEquals($res['body']['$id'], 'user1'); - $this->assertEquals($res['body']['name'], 'Project User'); - $this->assertEquals($res['body']['email'], 'users.service1@example.com'); + $this->assertEquals($res['body']['name'], 'Lionel Messi'); + $this->assertEquals($res['body']['email'], 'lionel.messi@psg.fr'); $this->assertEquals(true, $res['body']['status']); $this->assertGreaterThan(0, $res['body']['registration']); @@ -91,7 +91,7 @@ trait UsersBase 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ - 'search' => 'First Name' + 'search' => 'Ronaldo' ]); $this->assertEquals($response['headers']['status-code'], 200); $this->assertNotEmpty($response['body']); @@ -103,7 +103,7 @@ trait UsersBase 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ - 'search' => 'first.user' + 'search' => 'cristiano.ronaldo' ]); $this->assertEquals($response['headers']['status-code'], 200); $this->assertNotEmpty($response['body']); @@ -115,7 +115,7 @@ trait UsersBase 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], ], $this->getHeaders()), [ - 'search' => 'first user name' + 'search' => 'manchester' ]); $this->assertEquals($response['headers']['status-code'], 200); $this->assertNotEmpty($response['body']); @@ -123,6 +123,20 @@ trait UsersBase $this->assertCount(1, $response['body']['users']); $this->assertEquals($response['body']['users'][0]['$id'], $data['userId']); + $response = $this->client->call(Client::METHOD_GET, '/users', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'search' => 'manchester-united.co.uk' + ]); + + $this->assertEquals($response['headers']['status-code'], 200); + $this->assertIsArray($response['body']); + $this->assertIsArray($response['body']['users']); + $this->assertIsInt($response['body']['sum']); + $this->assertEquals(1, $response['body']['sum']); + $this->assertCount(1, $response['body']['users']); + $response = $this->client->call(Client::METHOD_GET, '/users', array_merge([ 'content-type' => 'application/json', 'x-appwrite-project' => $this->getProject()['$id'], @@ -134,18 +148,6 @@ trait UsersBase $this->assertNotEmpty($response['body']['users']); $this->assertCount(1, $response['body']['users']); $this->assertEquals($response['body']['users'][0]['$id'], $data['userId']); - - $response = $this->client->call(Client::METHOD_GET, '/users', array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - ], $this->getHeaders()), [ - 'search' => 'first user - first.user@example.com ' . $data['userId'] - ]); - $this->assertEquals($response['headers']['status-code'], 200); - $this->assertNotEmpty($response['body']); - $this->assertNotEmpty($response['body']['users']); - $this->assertCount(1, $response['body']['users']); - $this->assertEquals($response['body']['users'][0]['$id'], $data['userId']); } /** @@ -162,8 +164,8 @@ trait UsersBase ], $this->getHeaders())); $this->assertEquals($user['headers']['status-code'], 200); - $this->assertEquals($user['body']['name'], 'First Name'); - $this->assertEquals($user['body']['email'], 'first.user@example.com'); + $this->assertEquals($user['body']['name'], 'Cristiano Ronaldo'); + $this->assertEquals($user['body']['email'], 'cristiano.ronaldo@manchester-united.co.uk'); $this->assertEquals($user['body']['status'], true); $this->assertGreaterThan(0, $user['body']['registration']); @@ -194,21 +196,6 @@ trait UsersBase $this->assertIsInt($users['body']['sum']); $this->assertGreaterThan(0, $users['body']['sum']); - $users = $this->client->call(Client::METHOD_GET, '/users', array_merge([ - 'content-type' => 'application/json', - 'x-appwrite-project' => $this->getProject()['$id'], - ], $this->getHeaders()), [ - 'search' => 'example.com' - ]); - - $this->assertEquals($users['headers']['status-code'], 200); - $this->assertIsArray($users['body']); - $this->assertIsArray($users['body']['users']); - $this->assertIsInt($users['body']['sum']); - $this->assertEquals(2, $users['body']['sum']); - $this->assertGreaterThan(0, $users['body']['sum']); - $this->assertCount(2, $users['body']['users']); - return $data; } From 7ce2ca45b4405ba84c67ea53d698d018e64fefa6 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Wed, 6 Oct 2021 13:56:44 +0200 Subject: [PATCH 205/206] fix cache purge in projects api --- app/controllers/api/projects.php | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/app/controllers/api/projects.php b/app/controllers/api/projects.php index f87176af08..2acb095a2a 100644 --- a/app/controllers/api/projects.php +++ b/app/controllers/api/projects.php @@ -597,7 +597,7 @@ App::post('/v1/projects/:projectId/webhooks') $webhook = $dbForConsole->createDocument('webhooks', $webhook); - $dbForConsole->purgeDocument('projects', $project->getId()); + $dbForConsole->deleteCachedDocument('projects', $project->getId()); $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($webhook, Response::MODEL_WEBHOOK); @@ -724,7 +724,7 @@ App::put('/v1/projects/:projectId/webhooks/:webhookId') $dbForConsole->updateDocument('webhooks', $webhook->getId(), $webhook); - $dbForConsole->purgeDocument('projects', $project->getId()); + $dbForConsole->deleteCachedDocument('projects', $project->getId()); $response->dynamic($webhook, Response::MODEL_WEBHOOK); }); @@ -763,7 +763,7 @@ App::delete('/v1/projects/:projectId/webhooks/:webhookId') $dbForConsole->deleteDocument('webhooks', $webhook->getId()); - $dbForConsole->purgeDocument('projects', $project->getId()); + $dbForConsole->deleteCachedDocument('projects', $project->getId()); $response->noContent(); }); @@ -807,7 +807,7 @@ App::post('/v1/projects/:projectId/keys') $key = $dbForConsole->createDocument('keys', $key); - $dbForConsole->purgeDocument('projects', $project->getId()); + $dbForConsole->deleteCachedDocument('projects', $project->getId()); $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($key, Response::MODEL_KEY); @@ -924,7 +924,7 @@ App::put('/v1/projects/:projectId/keys/:keyId') $dbForConsole->updateDocument('keys', $key->getId(), $key); - $dbForConsole->purgeDocument('projects', $project->getId()); + $dbForConsole->deleteCachedDocument('projects', $project->getId()); $response->dynamic($key, Response::MODEL_KEY); }); @@ -963,7 +963,7 @@ App::delete('/v1/projects/:projectId/keys/:keyId') $dbForConsole->deleteDocument('keys', $key->getId()); - $dbForConsole->purgeDocument('projects', $project->getId()); + $dbForConsole->deleteCachedDocument('projects', $project->getId()); $response->noContent(); }); @@ -1014,7 +1014,7 @@ App::post('/v1/projects/:projectId/platforms') $platform = $dbForConsole->createDocument('platforms', $platform); - $dbForConsole->purgeDocument('projects', $project->getId()); + $dbForConsole->deleteCachedDocument('projects', $project->getId()); $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($platform, Response::MODEL_PLATFORM); @@ -1136,7 +1136,7 @@ App::put('/v1/projects/:projectId/platforms/:platformId') $dbForConsole->updateDocument('platforms', $platform->getId(), $platform); - $dbForConsole->purgeDocument('projects', $project->getId()); + $dbForConsole->deleteCachedDocument('projects', $project->getId()); $response->dynamic($platform, Response::MODEL_PLATFORM); }); @@ -1175,7 +1175,7 @@ App::delete('/v1/projects/:projectId/platforms/:platformId') $dbForConsole->deleteDocument('platforms', $platformId); - $dbForConsole->purgeDocument('projects', $project->getId()); + $dbForConsole->deleteCachedDocument('projects', $project->getId()); $response->noContent(); }); @@ -1238,7 +1238,7 @@ App::post('/v1/projects/:projectId/domains') $domain = $dbForConsole->createDocument('domains', $domain); - $dbForConsole->purgeDocument('projects', $project->getId()); + $dbForConsole->deleteCachedDocument('projects', $project->getId()); $response->setStatusCode(Response::STATUS_CODE_CREATED); $response->dynamic($domain, Response::MODEL_DOMAIN); @@ -1364,7 +1364,7 @@ App::patch('/v1/projects/:projectId/domains/:domainId/verification') $dbForConsole->updateDocument('domains', $domain->getId(), $domain->setAttribute('verification', true)); - $dbForConsole->purgeDocument('projects', $project->getId()); + $dbForConsole->deleteCachedDocument('projects', $project->getId()); // Issue a TLS certificate when domain is verified Resque::enqueue('v1-certificates', 'CertificatesV1', [ @@ -1410,7 +1410,7 @@ App::delete('/v1/projects/:projectId/domains/:domainId') $dbForConsole->deleteDocument('domains', $domain->getId()); - $dbForConsole->purgeDocument('projects', $project->getId()); + $dbForConsole->deleteCachedDocument('projects', $project->getId()); $deletes ->setParam('type', DELETE_TYPE_CERTIFICATES) From e0699932a03ad73b66b14e78459c5586d1313fe6 Mon Sep 17 00:00:00 2001 From: Torsten Dittmann Date: Wed, 6 Oct 2021 14:56:44 +0200 Subject: [PATCH 206/206] fix(users): query to ignore deleted users --- app/controllers/api/users.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/controllers/api/users.php b/app/controllers/api/users.php index 28da6b0a93..99748d40d5 100644 --- a/app/controllers/api/users.php +++ b/app/controllers/api/users.php @@ -113,7 +113,9 @@ App::get('/v1/users') } } - $queries = []; + $queries = [ + new Query('deleted', Query::TYPE_EQUAL, [false]) + ]; if (!empty($search)) { $queries[] = new Query('search', Query::TYPE_SEARCH, [$search]);