Merge pull request #5725 from appwrite/feat-git-integration

Feat: Functions G4

.env

@@ -1,12 +1,10 @@
 _APP_ENV=development
 _APP_LOCALE=en
-_APP_WORKER_PER_CORE=2
+_APP_WORKER_PER_CORE=6
 _APP_CONSOLE_WHITELIST_ROOT=disabled
 _APP_CONSOLE_WHITELIST_EMAILS=
-_APP_CONSOLE_WHITELIST_CODES=code-zero,code-one
 _APP_CONSOLE_WHITELIST_IPS=
-_APP_CONSOLE_INVITES=enabled
-_APP_CONSOLE_ROOT_SESSION=disabled
+_APP_CONSOLE_WHITELIST_CODES=code-zero,code-one
 _APP_SYSTEM_EMAIL_NAME=Appwrite
 _APP_SYSTEM_EMAIL_ADDRESS=team@appwrite.io
 _APP_SYSTEM_SECURITY_EMAIL_ADDRESS=security@appwrite.io
@@ -14,8 +12,9 @@ _APP_SYSTEM_RESPONSE_FORMAT=
 _APP_OPTIONS_ABUSE=disabled
 _APP_OPTIONS_FORCE_HTTPS=disabled
 _APP_OPENSSL_KEY_V1=your-secret-key
-_APP_DOMAIN=demo.appwrite.io
-_APP_DOMAIN_TARGET=demo.appwrite.io
+_APP_DOMAIN=localhost
+_APP_DOMAIN_FUNCTIONS=functions.localhost
+_APP_DOMAIN_TARGET=localhost
 _APP_REDIS_HOST=redis
 _APP_REDIS_PORT=6379
 _APP_REDIS_PASS=
@@ -26,26 +25,39 @@ _APP_DB_SCHEMA=appwrite
 _APP_DB_USER=user
 _APP_DB_PASS=password
 _APP_DB_ROOT_PASS=rootsecretpassword
-_APP_CONNECTIONS_MAX=3100
-_APP_POOL_CLIENTS=14
-_APP_CONNECTIONS_DB_PROJECT=db_fra1_02=mariadb://user:password@mariadb:3306/appwrite
-_APP_CONNECTIONS_DB_CONSOLE=db_fra1_01=mariadb://user:password@mariadb:3306/appwrite
-_APP_CONNECTIONS_CACHE=redis_fra1_01=redis://redis:6379
-_APP_CONNECTIONS_QUEUE=redis_fra1_01=redis://redis:6379
-_APP_CONNECTIONS_PUBSUB=redis_fra1_01=redis://redis:6379
-_APP_CONNECTIONS_STORAGE=local://localhost
+_APP_STORAGE_DEVICE=Local
+_APP_STORAGE_S3_ACCESS_KEY=
+_APP_STORAGE_S3_SECRET=
+_APP_STORAGE_S3_REGION=us-east-1
+_APP_STORAGE_S3_BUCKET=
+_APP_STORAGE_DO_SPACES_ACCESS_KEY=
+_APP_STORAGE_DO_SPACES_SECRET=
+_APP_STORAGE_DO_SPACES_REGION=us-east-1
+_APP_STORAGE_DO_SPACES_BUCKET=
+_APP_STORAGE_BACKBLAZE_ACCESS_KEY=
+_APP_STORAGE_BACKBLAZE_SECRET=
+_APP_STORAGE_BACKBLAZE_REGION=us-west-004
+_APP_STORAGE_BACKBLAZE_BUCKET=
+_APP_STORAGE_LINODE_ACCESS_KEY=
+_APP_STORAGE_LINODE_SECRET=
+_APP_STORAGE_LINODE_REGION=eu-central-1
+_APP_STORAGE_LINODE_BUCKET=
+_APP_STORAGE_WASABI_ACCESS_KEY=
+_APP_STORAGE_WASABI_SECRET=
+_APP_STORAGE_WASABI_REGION=eu-central-1
+_APP_STORAGE_WASABI_BUCKET=
 _APP_STORAGE_ANTIVIRUS=disabled
 _APP_STORAGE_ANTIVIRUS_HOST=clamav
 _APP_STORAGE_ANTIVIRUS_PORT=3310
+_APP_INFLUXDB_HOST=influxdb
+_APP_INFLUXDB_PORT=8086
+_APP_STATSD_HOST=telegraf
+_APP_STATSD_PORT=8125
 _APP_SMTP_HOST=maildev
 _APP_SMTP_PORT=1025
 _APP_SMTP_SECURE=
 _APP_SMTP_USERNAME=
 _APP_SMTP_PASSWORD=
-_APP_USERS_STATS_RECIPIENTS=
-_APP_HAMSTER_INTERVAL=86400
-_APP_HAMSTER_TIME=21:00
-_APP_MIXPANEL_TOKEN=
 _APP_SMS_PROVIDER=sms://username:password@mock
 _APP_SMS_FROM=+123456789
 _APP_STORAGE_LIMIT=30000000
@@ -54,32 +66,35 @@ _APP_FUNCTIONS_SIZE_LIMIT=30000000
 _APP_FUNCTIONS_TIMEOUT=900
 _APP_FUNCTIONS_BUILD_TIMEOUT=900
 _APP_FUNCTIONS_CPUS=1
-_APP_FUNCTIONS_MEMORY=512
+_APP_FUNCTIONS_MEMORY=1024
 _APP_FUNCTIONS_INACTIVE_THRESHOLD=600
 _APP_FUNCTIONS_MAINTENANCE_INTERVAL=600
 _APP_FUNCTIONS_RUNTIMES_NETWORK=runtimes
 _APP_EXECUTOR_SECRET=your-secret-key
-_APP_EXECUTOR_HOST=http://exc1/v1
+_APP_EXECUTOR_HOST=http://proxy/v1
 _APP_FUNCTIONS_RUNTIMES=
 _APP_MAINTENANCE_INTERVAL=86400
 _APP_MAINTENANCE_RETENTION_CACHE=2592000
 _APP_MAINTENANCE_RETENTION_EXECUTION=1209600
 _APP_MAINTENANCE_RETENTION_ABUSE=86400
 _APP_MAINTENANCE_RETENTION_AUDIT=1209600
-_APP_MAINTENANCE_RETENTION_SCHEDULES=86400
+_APP_USAGE_AGGREGATION_INTERVAL=5
 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000
+_APP_MAINTENANCE_RETENTION_SCHEDULES=86400
 _APP_USAGE_STATS=enabled
-_APP_USAGE_AGGREGATION_INTERVAL=30
 _APP_LOGGING_PROVIDER=
 _APP_LOGGING_CONFIG=
 _APP_GRAPHQL_MAX_BATCH_SIZE=10
 _APP_GRAPHQL_MAX_COMPLEXITY=250
 _APP_GRAPHQL_MAX_DEPTH=3
-_APP_REGION=default
 _APP_DOCKER_HUB_USERNAME=
 _APP_DOCKER_HUB_PASSWORD=
-_APP_CONSOLE_GITHUB_SECRET=
-_APP_CONSOLE_GITHUB_APP_ID=
+_APP_VCS_GITHUB_APP_NAME=
+_APP_VCS_GITHUB_PRIVATE_KEY=""
+_APP_VCS_GITHUB_APP_ID=
+_APP_VCS_GITHUB_CLIENT_ID=
+_APP_VCS_GITHUB_CLIENT_SECRET=
+_APP_VCS_GITHUB_WEBHOOK_SECRET=
 _APP_MIGRATIONS_FIREBASE_CLIENT_ID=
 _APP_MIGRATIONS_FIREBASE_CLIENT_SECRET=
-OPENAI_API_KEY=YOUR_OPENAI_API_KEY
+_APP_ASSISTANT_OPENAI_API_KEY=

.github/workflows/tests.yml

@@ -49,7 +49,9 @@ jobs:
         sleep 30
 
     - name: Doctor
-      run: docker compose exec -T appwrite doctor
+      run:  |
+        docker compose logs appwrite
+        docker compose exec -T appwrite doctor
 
     - name: Environment Variables
       run:  docker compose exec -T appwrite vars

.gitmodules

@@ -1,4 +1,4 @@
 [submodule "app/console"]
 	path = app/console
 	url = https://github.com/appwrite/console
-	branch = disallow-personal-data
+	branch = main

CHANGES.md

@@ -121,41 +121,6 @@
 - Get default region from environment on project create [#4780](https://github.com/appwrite/appwrite/pull/4780)
 - Fix french translation [#4782](https://github.com/appwrite/appwrite/pull/4782)
 - Fix max mimetype size [#4814](https://github.com/appwrite/appwrite/pull/4814)
-- New usage metrics collection flow [#4770](https://github.com/appwrite/appwrite/pull/4770)
-  - Deprecated influxdb, telegraf containers and removed all of their occurrences from the code.
-  - Removed _APP_INFLUXDB_HOST, _APP_INFLUXDB_PORT, _APP_STATSD_HOST, _APP_STATSD_PORT env variables.
-  - Removed usage labels dependency.
-  - Dropped type attribute from stats collection.
-  - Usage metrics are processed via new usage worker.
-  - Metrics changes: 
-    - Storage
-      - deprecated
-        - filesCreate, filesRead, filesUpdate, filesDelete, bucketsCreate, bucketsRead, bucketsUpdate, bucketsDelete.
-      - renamed
-        - filesCount to filesTotal, storage to filesStorage, bucketsCount to bucketsTotal.
-    - Auth 
-        - deprecated
-          - usersCreate, usersRead, usersUpdate, usersDelete, sessionsCreate sessionsProviderCreate, sessionsDelete.
-        - renamed
-          - usersCount to usersTotal.
-        - added
-          - sessionsTotal.
-    - Databases
-      - deprecated
-        - databasesCreate, databasesRead, databasesDelete, documentsCreate, documentsRead, documentsUpdate, documentsDelete, collectionsCreate, collectionsRead, collectionsUpdate, collectionsDelete.
-      - renamed
-        - databasesCount to databasesTotal, collectionsCount to collectionsTotal, documentsCount to documentsTotal.
-    - Functions
-      - deprecated 
-        - executionsFailure, executionsSuccess, buildsFailure, buildsSuccess, executionsFailure, executionsSuccess.
-      - renamed
-        - executionsTime to executionsCompute, buildsTime to buildsCompute, documentsCount to documentsTotal.
-      - added
-        - functionsTotal, buildsStorage, deploymentsTotal, deploymentsStorage.
-    - Project
-      - renamed
-        - executions to executionsTotal, builds to buildsTotal, requests to requestsTotal, storage to filesStorage, buckets to bucketsTotal, users to usersTotal, documents to documentsTotal, collections to collectionsTotal, databases to databasesTotal.
-
 ## Bugs
 - Fix invited account verified status [#4776](https://github.com/appwrite/appwrite/pull/4776)
 

CONTRIBUTING.md

@@ -238,6 +238,8 @@ Appwrite stack is a combination of a variety of open-source technologies and too
 
 - Redis - for managing cache and in-memory data (currently, we do not use Redis for persistent data).
 - MariaDB - for database storage and queries.
+- InfluxDB - for managing stats and time-series based data
+- Statsd - for sending data over UDP protocol (using Telegraf)
 - ClamAV - for validating and scanning storage files.
 - Imagemagick - for manipulating and managing image media files.
 - Webp - for better compression of images on supporting clients.

Dockerfile

@@ -29,7 +29,7 @@ ENV VITE_APPWRITE_GROWTH_ENDPOINT=$VITE_APPWRITE_GROWTH_ENDPOINT
 RUN npm ci
 RUN npm run build
 
-FROM appwrite/base:0.4.2 as final
+FROM appwrite/base:0.4.3 as final
 
 LABEL maintainer="team@appwrite.io"
 
@@ -43,14 +43,15 @@ ENV DOCKER_COMPOSE_VERSION=v2.5.0
 ENV _APP_SERVER=swoole \
     _APP_ENV=production \
     _APP_LOCALE=en \
+    _APP_WORKER_PER_CORE= \
     _APP_DOMAIN=localhost \
+    _APP_DOMAIN_FUNCTIONS=functions.localhost \
     _APP_DOMAIN_TARGET=localhost \
     _APP_HOME=https://appwrite.io \
     _APP_EDITION=community \
     _APP_CONSOLE_WHITELIST_ROOT=enabled \
     _APP_CONSOLE_WHITELIST_EMAILS= \
     _APP_CONSOLE_WHITELIST_IPS= \
-    _APP_CONSOLE_ROOT_SESSION= \
     _APP_SYSTEM_EMAIL_NAME= \
     _APP_SYSTEM_EMAIL_ADDRESS= \
     _APP_SYSTEM_RESPONSE_FORMAT= \
@@ -62,6 +63,27 @@ ENV _APP_SERVER=swoole \
     _APP_STORAGE_ANTIVIRUS=enabled \
     _APP_STORAGE_ANTIVIRUS_HOST=clamav \
     _APP_STORAGE_ANTIVIRUS_PORT=3310 \
+    _APP_STORAGE_DEVICE=Local \
+    _APP_STORAGE_S3_ACCESS_KEY= \
+    _APP_STORAGE_S3_SECRET= \
+    _APP_STORAGE_S3_REGION= \
+    _APP_STORAGE_S3_BUCKET= \
+    _APP_STORAGE_DO_SPACES_ACCESS_KEY= \
+    _APP_STORAGE_DO_SPACES_SECRET= \
+    _APP_STORAGE_DO_SPACES_REGION= \
+    _APP_STORAGE_DO_SPACES_BUCKET= \
+    _APP_STORAGE_BACKBLAZE_ACCESS_KEY= \
+    _APP_STORAGE_BACKBLAZE_SECRET= \
+    _APP_STORAGE_BACKBLAZE_REGION= \
+    _APP_STORAGE_BACKBLAZE_BUCKET= \
+    _APP_STORAGE_LINODE_ACCESS_KEY= \
+    _APP_STORAGE_LINODE_SECRET= \
+    _APP_STORAGE_LINODE_REGION= \
+    _APP_STORAGE_LINODE_BUCKET= \
+    _APP_STORAGE_WASABI_ACCESS_KEY= \
+    _APP_STORAGE_WASABI_SECRET= \
+    _APP_STORAGE_WASABI_REGION= \
+    _APP_STORAGE_WASABI_BUCKET= \
     _APP_REDIS_HOST=redis \
     _APP_REDIS_PORT=6379 \
     _APP_DB_HOST=mariadb \
@@ -69,22 +91,45 @@ ENV _APP_SERVER=swoole \
     _APP_DB_USER=root \
     _APP_DB_PASS=password \
     _APP_DB_SCHEMA=appwrite \
+    _APP_INFLUXDB_HOST=influxdb \
+    _APP_INFLUXDB_PORT=8086 \
+    _APP_STATSD_HOST=telegraf \
+    _APP_STATSD_PORT=8125 \
+    _APP_SMTP_HOST= \
+    _APP_SMTP_PORT= \
+    _APP_SMTP_SECURE= \
+    _APP_SMTP_USERNAME= \
+    _APP_SMTP_PASSWORD= \
+    _APP_SMS_PROVIDER= \
+    _APP_SMS_FROM= \
     _APP_FUNCTIONS_SIZE_LIMIT=30000000 \
     _APP_FUNCTIONS_TIMEOUT=900 \
+    _APP_FUNCTIONS_CONTAINERS=10 \
     _APP_FUNCTIONS_CPUS=1 \
     _APP_FUNCTIONS_MEMORY=128 \
+    _APP_FUNCTIONS_MEMORY_SWAP=128 \
     _APP_EXECUTOR_SECRET=a-random-secret \
-    _APP_EXECUTOR_HOST=http://exc1/v1 \
+    _APP_EXECUTOR_HOST=http://appwrite-executor/v1 \
+    _APP_EXECUTOR_RUNTIME_NETWORK=appwrite_runtimes \
     _APP_SETUP=self-hosted \
     _APP_VERSION=$VERSION \
     _APP_USAGE_STATS=enabled \
+    _APP_USAGE_AGGREGATION_INTERVAL=30 \
     # 14 Days = 1209600 s
     _APP_MAINTENANCE_RETENTION_EXECUTION=1209600 \
     _APP_MAINTENANCE_RETENTION_AUDIT=1209600 \
     # 1 Day = 86400 s
     _APP_MAINTENANCE_RETENTION_ABUSE=86400 \
     _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 \
-    _APP_MAINTENANCE_INTERVAL=86400
+    _APP_MAINTENANCE_INTERVAL=86400 \
+    _APP_LOGGING_PROVIDER= \
+    _APP_LOGGING_CONFIG= \
+    _APP_VCS_GITHUB_APP_NAME= \
+    _APP_VCS_GITHUB_PRIVATE_KEY= \
+    _APP_VCS_GITHUB_APP_ID= \
+    _APP_VCS_GITHUB_CLIENT_ID= \
+    _APP_VCS_GITHUB_CLIENT_SECRET= \
+    _APP_VCS_GITHUB_WEBHOOK_SECRET=
 
 RUN \
   if [ "$DEBUG" == "true" ]; then \
@@ -119,13 +164,8 @@ RUN mkdir -p /storage/uploads && \
 
 # Executables
 RUN chmod +x /usr/local/bin/doctor && \
-    chmod +x /usr/local/bin/patch-delete-schedule-updated-at-attribute && \
-    chmod +x /usr/local/bin/clear-card-cache && \
-    chmod +x /usr/local/bin/calc-users-stats && \
-    chmod +x /usr/local/bin/calc-tier-stats && \
-    chmod +x /usr/local/bin/patch-delete-project-collections && \
     chmod +x /usr/local/bin/maintenance &&  \
-    chmod +x /usr/local/bin/volume-sync && \
+    chmod +x /usr/local/bin/usage && \
     chmod +x /usr/local/bin/install && \
     chmod +x /usr/local/bin/migrate && \
     chmod +x /usr/local/bin/realtime && \
@@ -133,7 +173,6 @@ RUN chmod +x /usr/local/bin/doctor && \
     chmod +x /usr/local/bin/sdks && \
     chmod +x /usr/local/bin/specs && \
     chmod +x /usr/local/bin/ssl && \
-    chmod +x /usr/local/bin/hamster && \
     chmod +x /usr/local/bin/test && \
     chmod +x /usr/local/bin/vars && \
     chmod +x /usr/local/bin/worker-audits && \
@@ -145,8 +184,16 @@ RUN chmod +x /usr/local/bin/doctor && \
     chmod +x /usr/local/bin/worker-mails && \
     chmod +x /usr/local/bin/worker-messaging && \
     chmod +x /usr/local/bin/worker-webhooks && \
-    chmod +x /usr/local/bin/worker-migrations && \
-    chmod +x /usr/local/bin/worker-usage
+    chmod +x /usr/local/bin/worker-migrations
+
+# Cloud Executabless
+RUN chmod +x /usr/local/bin/hamster && \
+    chmod +x /usr/local/bin/volume-sync && \
+    chmod +x /usr/local/bin/patch-delete-schedule-updated-at-attribute && \
+    chmod +x /usr/local/bin/patch-delete-project-collections && \
+    chmod +x /usr/local/bin/clear-card-cache && \
+    chmod +x /usr/local/bin/calc-users-stats && \
+    chmod +x /usr/local/bin/calc-tier-stats
 
 # Letsencrypt Permissions
 RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/

README-CN.md

@@ -66,7 +66,7 @@ docker run -it --rm \
     --volume /var/run/docker.sock:/var/run/docker.sock \
     --volume "$(pwd)"/appwrite:/usr/src/code/appwrite:rw \
     --entrypoint="install" \
-    appwrite/appwrite:1.3.8
+    appwrite/appwrite:1.4.0
 ```
 
 ### Windows
@@ -78,7 +78,7 @@ docker run -it --rm ^
     --volume //var/run/docker.sock:/var/run/docker.sock ^
     --volume "%cd%"/appwrite:/usr/src/code/appwrite:rw ^
     --entrypoint="install" ^
-    appwrite/appwrite:1.3.8
+    appwrite/appwrite:1.4.0
 ```
 
 #### PowerShell
@@ -88,7 +88,7 @@ docker run -it --rm `
     --volume /var/run/docker.sock:/var/run/docker.sock `
     --volume ${pwd}/appwrite:/usr/src/code/appwrite:rw `
     --entrypoint="install" `
-    appwrite/appwrite:1.3.8
+    appwrite/appwrite:1.4.0
 ```
 
 运行后，可以在浏览器上访问 http://localhost 找到 Appwrite 控制台。在非 Linux 的本机主机上完成安装后，服务器可能需要几分钟才能启动。

README.md

@@ -75,7 +75,7 @@ docker run -it --rm \
     --volume /var/run/docker.sock:/var/run/docker.sock \
     --volume "$(pwd)"/appwrite:/usr/src/code/appwrite:rw \
     --entrypoint="install" \
-    appwrite/appwrite:1.3.8
+    appwrite/appwrite:1.4.0
 ```
 
 ### Windows
@@ -87,7 +87,7 @@ docker run -it --rm ^
     --volume //var/run/docker.sock:/var/run/docker.sock ^
     --volume "%cd%"/appwrite:/usr/src/code/appwrite:rw ^
     --entrypoint="install" ^
-    appwrite/appwrite:1.3.8
+    appwrite/appwrite:1.4.0
 ```
 
 #### PowerShell
@@ -97,7 +97,7 @@ docker run -it --rm `
     --volume /var/run/docker.sock:/var/run/docker.sock `
     --volume ${pwd}/appwrite:/usr/src/code/appwrite:rw `
     --entrypoint="install" `
-    appwrite/appwrite:1.3.8
+    appwrite/appwrite:1.4.0
 ```
 
 Once the Docker installation is complete, go to http://localhost to access the Appwrite console from your browser. Please note that on non-Linux native hosts, the server might take a few minutes to start after completing the installation.

SECURITY.md

@@ -9,6 +9,7 @@
 | 1.1.x     | :white_check_mark: |
 | 1.2.x     | :white_check_mark: |
 | 1.3.x     | :white_check_mark: |
+| 1.4.x     | :white_check_mark: |
 
 ## Reporting a Vulnerability
 

app/cli.php

@@ -58,7 +58,7 @@ CLI::setResource('dbForConsole', function ($pools, $cache) {
                 ->getResource();
 
             $dbForConsole = new Database($dbAdapter, $cache);
-            $dbForConsole->setNamespace('console');
+            $dbForConsole->setNamespace('_console');
 
             // Ensure tables exist
             $collections = Config::getParam('collections', [])['console'];
@@ -74,7 +74,7 @@ CLI::setResource('dbForConsole', function ($pools, $cache) {
             $pools->get('console')->reclaim();
             sleep($sleep);
         }
-    } while ($attempts < $maxAttempts);
+    } while ($attempts < $maxAttempts && !$ready);
 
     if (!$ready) {
         throw new Exception("Console is not ready yet. Please try again later.");
@@ -116,6 +116,29 @@ CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole,
     return $getProjectDB;
 }, ['pools', 'dbForConsole', 'cache']);
 
+CLI::setResource('influxdb', function (Registry $register) {
+    $client = $register->get('influxdb'); /** @var InfluxDB\Client $client */
+    $attempts = 0;
+    $max = 10;
+    $sleep = 1;
+
+    do { // check if telegraf database is ready
+        try {
+            $attempts++;
+            $database = $client->selectDB('telegraf');
+            if (in_array('telegraf', $client->listDatabases())) {
+                break; // leave the do-while if successful
+            }
+        } catch (\Throwable $th) {
+            Console::warning("InfluxDB not ready. Retrying connection ({$attempts})...");
+            if ($attempts >= $max) {
+                throw new \Exception('InfluxDB database not ready yet');
+            }
+            sleep($sleep);
+        }
+    } while ($attempts < $max);
+    return $database;
+}, ['register']);
 
 CLI::setResource('queueForFunctions', function (Group $pools) {
     return new Func($pools->get('queue')->pop()->getResource());

app/config/errors.php

@@ -98,6 +98,11 @@ return [
         'description' => 'Usage stats is not configured. Please check the value of the _APP_USAGE_STATS environment variable of your Appwrite server.',
         'code' => 501,
     ],
+    Exception::GENERAL_NOT_IMPLEMENTED => [
+        'name' => Exception::GENERAL_NOT_IMPLEMENTED,
+        'description' => 'This method was not fully implemented yet. If you believe this is a mistake, please upgrade your Appwrite server version.',
+        'code' => 405,
+    ],
 
     /** User Errors */
     Exception::USER_COUNT_EXCEEDED => [
@@ -112,12 +117,12 @@ return [
     ],
     Exception::USER_ALREADY_EXISTS => [
         'name' => Exception::USER_ALREADY_EXISTS,
-        'description' => 'A user with the same id, email, or phone already exists in your project.',
+        'description' => 'A user with the same id, email, or phone already exists in this project.',
         'code' => 409,
     ],
     Exception::USER_BLOCKED => [
         'name' => Exception::USER_BLOCKED,
-        'description' => 'The current user has been blocked.',
+        'description' => 'The current user has been blocked. You can unblock the user by making a request to the User API\'s "Update User Status" endpoint or in the Appwrite Console\'s Auth section.',
         'code' => 401,
     ],
     Exception::USER_INVALID_TOKEN => [
@@ -177,12 +182,12 @@ return [
     ],
     Exception::USER_PASSWORD_RECENTLY_USED => [
         'name' => Exception::USER_PASSWORD_RECENTLY_USED,
-        'description' => 'The password you are trying to use is similar to your previous password. Please choose a stronger password.',
+        'description' => 'The password you are trying to use is similar to your previous password. For your security, please choose a different password and try again.',
         'code' => 400,
     ],
     Exception::USER_PASSWORD_PERSONAL_DATA => [
         'name' => Exception::USER_PASSWORD_PERSONAL_DATA,
-        'description' => 'The password you are trying to use contains references to your name, email, phone or userID. Please choose a different password and try again.',
+        'description' => 'The password you are trying to use contains references to your name, email, phone or userID. For your security, please choose a different password and try again.',
         'code' => 400,
     ],
     Exception::USER_SESSION_NOT_FOUND => [
@@ -192,7 +197,7 @@ return [
     ],
     Exception::USER_IDENTITY_NOT_FOUND => [
         'name' => Exception::USER_IDENTITY_NOT_FOUND,
-        'description' => 'The identity could not be found.',
+        'description' => 'The identity could not be found. Please sign in with OAuth provider to create identity first.',
         'code' => 404,
     ],
     Exception::USER_UNAUTHORIZED => [
@@ -254,7 +259,7 @@ return [
     ],
     Exception::TEAM_INVALID_SECRET => [
         'name' => Exception::TEAM_INVALID_SECRET,
-        'description' => 'The team invitation secret is invalid.',
+        'description' => 'The team invitation secret is invalid. Please request  a new invitation and try again.',
         'code' => 401,
     ],
     Exception::TEAM_MEMBERSHIP_MISMATCH => [
@@ -269,7 +274,7 @@ return [
     ],
     Exception::TEAM_ALREADY_EXISTS => [
         'name' => Exception::TEAM_ALREADY_EXISTS,
-        'description' => 'Team with requested ID already exists.',
+        'description' => 'Team with requested ID already exists. Please choose a different ID and try again.',
         'code' => 409,
     ],
 
@@ -281,7 +286,7 @@ return [
     ],
     Exception::MEMBERSHIP_ALREADY_CONFIRMED => [
         'name' => Exception::MEMBERSHIP_ALREADY_CONFIRMED,
-        'description' => 'Membership already confirmed',
+        'description' => 'Membership is already confirmed.',
         'code' => 409,
     ],
 
@@ -350,7 +355,7 @@ return [
     ],
     Exception::STORAGE_BUCKET_ALREADY_EXISTS => [
         'name' => Exception::STORAGE_BUCKET_ALREADY_EXISTS,
-        'description' => 'A storage bucket with the requested ID already exists.',
+        'description' => 'A storage bucket with the requested ID already exists. Try again with a different ID or use "unique()" to generate a unique ID.',
         'code' => 409,
     ],
     Exception::STORAGE_BUCKET_NOT_FOUND => [
@@ -370,7 +375,34 @@ return [
     ],
     Exception::STORAGE_INVALID_APPWRITE_ID => [
         'name' => Exception::STORAGE_INVALID_APPWRITE_ID,
-        'description' => 'The value for x-appwrite-id header is invalid. Please check the value of the x-appwrite-id header is valid id and not unique().',
+        'description' => 'The value for x-appwrite-id header is invalid. Please check the value of the x-appwrite-id header is a valid id and not unique().',
+        'code' => 400,
+    ],
+
+    /** VCS */
+    Exception::INSTALLATION_NOT_FOUND => [
+        'name' => Exception::INSTALLATION_NOT_FOUND,
+        'description' => 'Installation with the requested ID could not be found. Check to see if the ID is correct, or create the installation.',
+        'code' => 404,
+    ],
+    Exception::PROVIDER_REPOSITORY_NOT_FOUND => [
+        'name' => Exception::PROVIDER_REPOSITORY_NOT_FOUND,
+        'description' => 'VCS (Version Control System) repository with the requested ID could not be found. Check to see if the ID is correct, and if it belongs to installationId you provided.',
+        'code' => 404,
+    ],
+    Exception::REPOSITORY_NOT_FOUND => [
+        'name' => Exception::REPOSITORY_NOT_FOUND,
+        'description' => 'Repository with the requested ID could not be found. Check to see if the ID is correct, or create the respository.',
+        'code' => 404,
+    ],
+    Exception::PROVIDER_CONTRIBUTION_CONFLICT => [
+        'name' => Exception::PROVIDER_CONTRIBUTION_CONFLICT,
+        'description' => 'External contribution is already authorized.',
+        'code' => 409,
+    ],
+    Exception::GENERAL_PROVIDER_FAILURE => [
+        'name' => Exception::GENERAL_PROVIDER_FAILURE,
+        'description' => 'VCS (Version Control System) provider failed to proccess the request. We believe this is an error with the VCS provider. Try again, or contact support for more information.',
         'code' => 400,
     ],
 
@@ -385,6 +417,11 @@ return [
         'description' => 'The requested runtime is either inactive or unsupported. Please check the value of the _APP_FUNCTIONS_RUNTIMES environment variable.',
         'code' => 404,
     ],
+    Exception::FUNCTION_ENTRYPOINT_MISSING => [
+        'name' => Exception::FUNCTION_RUNTIME_UNSUPPORTED,
+        'description' => 'Entrypoint for your Appwrite Function is missing. Please specify it when making deployment or update the entrypoint under your function\'s "Settings" > "Configuration" > "Entrypoint".',
+        'code' => 404,
+    ],
 
     /** Builds  */
     Exception::BUILD_NOT_FOUND => [
@@ -438,7 +475,7 @@ return [
     ],
     Exception::COLLECTION_ALREADY_EXISTS => [
         'name' => Exception::COLLECTION_ALREADY_EXISTS,
-        'description' => 'A collection with the requested ID already exists.',
+        'description' => 'A collection with the requested ID already exists. Try again with a different ID or use "unique()" to generate a unique ID.',
         'code' => 409,
     ],
     Exception::COLLECTION_LIMIT_EXCEEDED => [
@@ -460,17 +497,17 @@ return [
     ],
     Exception::DOCUMENT_MISSING_DATA => [
         'name' => Exception::DOCUMENT_MISSING_DATA,
-        'description' => 'The document data is missing. You must provide the document data.',
+        'description' => 'The document data is missing. Try again with document data populated',
         'code' => 400,
     ],
     Exception::DOCUMENT_MISSING_PAYLOAD => [
         'name' => Exception::DOCUMENT_MISSING_PAYLOAD,
-        'description' => 'The document data and permissions are missing. You must provide either the document data or permissions to be updated.',
+        'description' => 'The document data and permissions are missing. You must provide either document data or permissions to be updated.',
         'code' => 400,
     ],
     Exception::DOCUMENT_ALREADY_EXISTS => [
         'name' => Exception::DOCUMENT_ALREADY_EXISTS,
-        'description' => 'Document with the requested ID already exists.',
+        'description' => 'Document with the requested ID already exists. Try again with a different ID or use "unique()" to generate a unique ID.',
         'code' => 409,
     ],
     Exception::DOCUMENT_UPDATE_CONFLICT => [
@@ -512,7 +549,7 @@ return [
     ],
     Exception::ATTRIBUTE_ALREADY_EXISTS => [
         'name' => Exception::ATTRIBUTE_ALREADY_EXISTS,
-        'description' => 'Attribute with the requested ID already exists.',
+        'description' => 'Attribute with the requested ID already exists. Try again with a different ID or use "unique()" to generate a unique ID.',
         'code' => 409,
     ],
     Exception::ATTRIBUTE_LIMIT_EXCEEDED => [
@@ -544,7 +581,7 @@ return [
     ],
     Exception::INDEX_ALREADY_EXISTS => [
         'name' => Exception::INDEX_ALREADY_EXISTS,
-        'description' => 'Index with the requested ID already exists.',
+        'description' => 'Index with the requested ID already exists. Try again with a different ID or use "unique()" to generate a unique ID.',
         'code' => 409,
     ],
     Exception::INDEX_INVALID => [
@@ -561,7 +598,7 @@ return [
     ],
     Exception::PROJECT_ALREADY_EXISTS => [
         'name' => Exception::PROJECT_ALREADY_EXISTS,
-        'description' => 'Project with the requested ID already exists.',
+        'description' => 'Project with the requested ID already exists. Try again with a different ID or use "unique()" to generate a unique ID.',
         'code' => 409,
     ],
     Exception::PROJECT_UNKNOWN => [
@@ -599,14 +636,44 @@ return [
         'description' => 'The project key has expired. Please generate a new key using the Appwrite console.',
         'code' => 401,
     ],
+    Exception::ROUTER_HOST_NOT_FOUND => [
+        'name' => Exception::ROUTER_HOST_NOT_FOUND,
+        'description' => 'Host is not trusted. This could occur because you have not configured a custom domain. Add a custom domain to your project first and try again.',
+        'code' => 404,
+    ],
+    Exception::ROUTER_DOMAIN_NOT_CONFIGURED => [
+        'name' => Exception::ROUTER_DOMAIN_NOT_CONFIGURED,
+        'description' => 'Domain environment variables not configured. Please configure domain environment variables before using Appwrite outside of localhost.',
+        'code' => 500,
+    ],
+    Exception::RULE_RESOURCE_NOT_FOUND => [
+        'name' => Exception::RULE_RESOURCE_NOT_FOUND,
+        'description' => 'Resource could not be found. Please check if the resourceId and resourceType are correct, or if the resource actually exists.',
+        'code' => 404,
+    ],
+    Exception::RULE_NOT_FOUND => [
+        'name' => Exception::RULE_NOT_FOUND,
+        'description' => 'Rule with the requested ID could not be found. Please check if the ID provided is correct or if the rule actually exists.',
+        'code' => 404,
+    ],
+    Exception::RULE_ALREADY_EXISTS => [
+        'name' => Exception::RULE_ALREADY_EXISTS,
+        'description' => 'Domain is already used. Please try again with a different domain.',
+        'code' => 409,
+    ],
+    Exception::RULE_VERIFICATION_FAILED => [
+        'name' => Exception::RULE_VERIFICATION_FAILED,
+        'description' => 'Domain verification failed. Please check if your DNS records are correct and try again.',
+        'code' => 401,
+    ],
     Exception::PROJECT_SMTP_CONFIG_INVALID => [
         'name' => Exception::PROJECT_SMTP_CONFIG_INVALID,
-        'description' => 'Provided SMTP config is invalid.',
+        'description' => 'Provided SMTP config is invalid. Please check the configured values and try again.',
         'code' => 400,
     ],
     Exception::PROJECT_TEMPLATE_DEFAULT_DELETION => [
         'name' => Exception::PROJECT_TEMPLATE_DEFAULT_DELETION,
-        'description' => 'The default template for the project cannot be deleted.',
+        'description' => 'You can\'t delete default template. If you are trying to reset your template changes, you can ignore this error as it\'s already been reset.',
         'code' => 401,
     ],
     Exception::WEBHOOK_NOT_FOUND => [
@@ -624,21 +691,6 @@ return [
         'description' => 'Platform with the requested ID could not be found.',
         'code' => 404,
     ],
-    Exception::DOMAIN_NOT_FOUND => [
-        'name' => Exception::DOMAIN_NOT_FOUND,
-        'description' => 'Domain with the requested ID could not be found.',
-        'code' => 404,
-    ],
-    Exception::DOMAIN_ALREADY_EXISTS => [
-        'name' => Exception::DOMAIN_ALREADY_EXISTS,
-        'description' => 'The requested domain is currently in use by a project.',
-        'code' => 409,
-    ],
-    Exception::DOMAIN_FORBIDDEN => [
-        'name' => Exception::DOMAIN_FORBIDDEN,
-        'description' => 'The requested domain cannot be used as a custom domain.',
-        'code' => 403,
-    ],
     Exception::VARIABLE_NOT_FOUND => [
         'name' => Exception::VARIABLE_NOT_FOUND,
         'description' => 'Variable with the requested ID could not be found.',
@@ -646,19 +698,9 @@ return [
     ],
     Exception::VARIABLE_ALREADY_EXISTS => [
         'name' => Exception::VARIABLE_ALREADY_EXISTS,
-        'description' => 'Variable with the same ID already exists in your project.',
+        'description' => 'Variable with the same ID already exists in this project. Try again with a different ID.',
         'code' => 409,
     ],
-    Exception::DOMAIN_VERIFICATION_FAILED => [
-        'name' => Exception::DOMAIN_VERIFICATION_FAILED,
-        'description' => 'Domain verification for the requested domain has failed.',
-        'code' => 401,
-    ],
-    Exception::DOMAIN_TARGET_INVALID => [
-        'name' => Exception::DOMAIN_TARGET_INVALID,
-        'description' => 'Your Appwrite instance is not publicly accessible. Please check the _APP_DOMAIN_TARGET environment variable of your Appwrite server.',
-        'code' => 501,
-    ],
     Exception::GRAPHQL_NO_QUERY => [
         'name' => Exception::GRAPHQL_NO_QUERY,
         'description' => 'Param "query" is not optional.',
@@ -673,17 +715,17 @@ return [
     /** Migrations */
     Exception::MIGRATION_NOT_FOUND => [
         'name' => Exception::MIGRATION_NOT_FOUND,
-        'description' => 'Migration with the requested ID could not be found.',
+        'description' => 'Migration with the requested ID could not be found. Please verify that the provided ID is correct and try again.',
         'code' => 404,
     ],
     Exception::MIGRATION_ALREADY_EXISTS => [
         'name' => Exception::MIGRATION_ALREADY_EXISTS,
-        'description' => 'Migration with the requested ID already exists.',
+        'description' => 'Migration with the requested ID already exists. Try again with a different ID.',
         'code' => 409,
     ],
     Exception::MIGRATION_IN_PROGRESS => [
         'name' => Exception::MIGRATION_IN_PROGRESS,
-        'description' => 'Migration is already in progress.',
+        'description' => 'Migration is already in progress. You can check the status of the migration in your Appwrite Console\'s "Settings" > "Migrations".',
         'code' => 409,
     ],
 ];

app/config/events.php

@@ -236,5 +236,19 @@ return [
         'update' => [
             '$description' => 'This event triggers when a function is updated.',
         ]
+    ],
+    'rules' => [
+        '$model' => Response::MODEL_PROXY_RULE,
+        '$resource' => true,
+        '$description' => 'This event triggers on any proxy rule event.',
+        'create' => [
+            '$description' => 'This event triggers when a proxy rule is created.'
+        ],
+        'delete' => [
+            '$description' => 'This event triggers when a proxy rule is deleted.',
+        ],
+        'update' => [
+            '$description' => 'This event triggers when a proxy rule is updated.',
+        ]
     ]
 ];

app/config/locale/templates/email-base.tpl

@@ -42,6 +42,10 @@
             border: none;
             border-top: 1px solid #E8E9F0;
         }
+
+        p {
+            margin-bottom: 10px;
+        }
     </style>
 </head>
 
@@ -52,29 +56,7 @@
     <table style="margin-top: 32px">
         <tr>
             <td>
-                <h1>
-                    {{subject}}
-                </h1>
-            </td>
-        </tr>
-    </table>
-
-    <table style="margin-top: 40px">
-        <tr>
-            <td>
-                <p>{{hello}}</p>
-
-                <p>{{body}}</p>
-
-                <a href="{{redirect}}" target="_blank">{{redirect}}</a>
-
-                <p><br />{{footer}}</p>
-                <br />
-
-                <p>{{thanks}}
-                <br />
-                {{signature}}
-                </p>
+                {{body}}
             </td>
         </tr>
     </table>

app/config/locale/templates/email-inner-base.tpl

@@ -0,0 +1,15 @@
+<p>{{hello}}</p>
+
+<br>
+
+<p>{{body}}</p>
+
+<a href="{{redirect}}" target="_blank">{{redirect}}</a>
+
+<p>{{footer}}</p>
+
+<br>
+
+<p>{{thanks}}</p>
+
+<p>{{signature}}</p>

app/config/providers.php

@@ -1,6 +1,7 @@
 <?php
 
-return [ // Ordered by ABC.
+// Ordered by ABC.
+return [
     'amazon' => [
         'name' => 'Amazon',
         'developers' => 'https://developer.amazon.com/apps-and-games/services-and-apis',

app/config/roles.php

@@ -3,6 +3,7 @@
 use Appwrite\Auth\Auth;
 
 $member = [
+    'global',
     'public',
     'home',
     'console',
@@ -23,6 +24,7 @@ $member = [
 ];
 
 $admins = [
+    'global',
     'graphql',
     'teams.read',
     'teams.write',
@@ -51,6 +53,8 @@ $admins = [
     'functions.write',
     'execution.read',
     'execution.write',
+    'rules.read',
+    'rules.write',
     'migrations.read',
     'migrations.write',
 ];
@@ -74,22 +78,22 @@ return [
     ],
     Auth::USER_ROLE_USERS => [
         'label' => 'Users',
-        'scopes' => \array_merge($member, []),
+        'scopes' => \array_merge($member),
     ],
     Auth::USER_ROLE_ADMIN => [
         'label' => 'Admin',
-        'scopes' => \array_merge($admins, []),
+        'scopes' => \array_merge($admins),
     ],
     Auth::USER_ROLE_DEVELOPER => [
         'label' => 'Developer',
-        'scopes' => \array_merge($admins, []),
+        'scopes' => \array_merge($admins),
     ],
     Auth::USER_ROLE_OWNER => [
         'label' => 'Owner',
-        'scopes' => \array_merge($member, $admins, []),
+        'scopes' => \array_merge($member, $admins),
     ],
     Auth::USER_ROLE_APPS => [
         'label' => 'Applications',
-        'scopes' => ['health.read', 'graphql'],
+        'scopes' => ['global', 'health.read', 'graphql'],
     ],
 ];

app/config/runtimes.php

@@ -7,7 +7,7 @@
 use Utopia\App;
 use Appwrite\Runtimes\Runtimes;
 
-$runtimes = new Runtimes('v2');
+$runtimes = new Runtimes('v3');
 
 $allowList = empty(App::getEnv('_APP_FUNCTIONS_RUNTIMES')) ? [] : \explode(',', App::getEnv('_APP_FUNCTIONS_RUNTIMES'));
 

app/config/scopes.php

@@ -76,6 +76,12 @@ return [ // List of publicly visible scopes
     'health.read' => [
         'description' => 'Access to read your project\'s health status',
     ],
+    'rules.read' => [
+        'description' => 'Access to read your project\'s proxy rules',
+    ],
+    'rules.write' => [
+        'description' => 'Access to create, update, and delete your project\'s proxy rules',
+    ],
     'migrations.read' => [
         'description' => 'Access to read your project\'s migrations',
     ],

app/config/services.php

@@ -160,6 +160,19 @@ return [
         'optional' => true,
         'icon' => '/images/services/users.png',
     ],
+    'vcs' => [
+        'key' => 'vcs',
+        'name' => 'VCS',
+        'subtitle' => 'The VCS service allows you to interact with providers like GitHub, GitLab etc.',
+        'description' => '',
+        'controller' => 'api/vcs.php',
+        'sdk' => false,
+        'docs' => false,
+        'docsUrl' => '',
+        'tests' => false,
+        'optional' => true,
+        'icon' => '',
+    ],
     'functions' => [
         'key' => 'functions',
         'name' => 'Functions',
@@ -173,6 +186,19 @@ return [
         'optional' => true,
         'icon' => '/images/services/functions.png',
     ],
+    'proxy' => [
+        'key' => 'proxy',
+        'name' => 'Proxy',
+        'subtitle' => 'The Proxy Service allows you to configure actions for your domains beyond DNS configuration.',
+        'description' => '/docs/services/proxy.md',
+        'controller' => 'api/proxy.php',
+        'sdk' => true,
+        'docs' => true,
+        'docsUrl' => 'https://appwrite.io/docs/proxy',
+        'tests' => false,
+        'optional' => true,
+        'icon' => '/images/services/proxy.png',
+    ],
     'mock' => [
         'key' => 'mock',
         'name' => 'Mock',
@@ -215,7 +241,7 @@ return [
     'migrations' => [
         'key' => 'migrations',
         'name' => 'Migrations',
-        'subtitle' => 'The Migrations service allows you to migrate third-party data to your Appwrite server.',
+        'subtitle' => 'The Migrations service allows you to migrate third-party data to your Appwrite project.',
         'description' => '/docs/services/migrations.md',
         'controller' => 'api/migrations.php',
         'sdk' => true,

app/config/variables.php

@@ -61,6 +61,15 @@ return [
                 'question' => 'Enter your Appwrite hostname',
                 'filter' => ''
             ],
+            [
+                'name' => '_APP_DOMAIN_FUNCTIONS',
+                'description' => 'A domain to use for function preview URLs. Setting to empty turns off function preview URLs.',
+                'introduction' => '',
+                'default' => '',
+                'required' => false,
+                'question' => '',
+                'filter' => ''
+            ],
             [
                 'name' => '_APP_DOMAIN_TARGET',
                 'description' => 'A DNS A record hostname to serve as a CNAME target for your Appwrite custom domains. You can use the same value as used for the Appwrite \'_APP_DOMAIN\' variable. The default value is \'localhost\'.',
@@ -105,15 +114,6 @@ return [
                 'question' => '',
                 'filter' => ''
             ],
-            [
-                'name' => '_APP_CONSOLE_ROOT_SESSION',
-                'description' => 'Domain policy for the Appwrite console session cookie. By default, set to \'disabled\', meaning the session cookie will be set to the domain of the Appwrite console (e.g. cloud.appwrite.io). When set to \'enabled\', the session cookie will be set to the registerable domain of the Appwrite server (e.g. appwrite.io).',
-                'introduction' => '',
-                'default' => 'disabled',
-                'required' => false,
-                'question' => '',
-                'filter' => ''
-            ],
             [
                 'name' => '_APP_SYSTEM_EMAIL_NAME',
                 'description' => 'This is the sender name value that will appear on email messages sent to developers from the Appwrite console. The default value is: \'Appwrite\'. You can use url encoded strings for spaces and special chars.',
@@ -152,7 +152,7 @@ return [
             ],
             [
                 'name' => '_APP_USAGE_STATS',
-                'description' => 'This variable allows you to disable the collection and displaying of usage stats. This value is set to \'enabled\' by default, to disable the usage stats set the value to \'disabled\'. When disabled, it\'s recommended to turn off the Worker Usage container for better resource usage.',
+                'description' => 'This variable allows you to disable the collection and displaying of usage stats. This value is set to \'enabled\' by default, to disable the usage stats set the value to \'disabled\'. When disabled, it\'s recommended to turn off the Worker Usage container to reduce resource usage.',
                 'introduction' => '0.7.0',
                 'default' => 'enabled',
                 'required' => false,
@@ -161,7 +161,7 @@ return [
             ],
             [
                 'name' => '_APP_LOGGING_PROVIDER',
-                'description' => 'This variable allows you to enable logging errors to 3rd party providers. This value is empty by default, to enable the logger set the value to one of \'sentry\', \'raygun\', \'appSignal\', \'logOwl\'',
+                'description' => 'This variable allows you to enable logging errors to 3rd party providers. This value is empty by default, set the value to one of \'sentry\', \'raygun\', \'appSignal\', \'logOwl\' to enable the logger.',
                 'introduction' => '0.12.0',
                 'default' => '',
                 'required' => false,
@@ -213,15 +213,6 @@ return [
                 'question' => '',
                 'filter' => ''
             ],
-            [
-                'name' => '_APP_CONSOLE_INVITES',
-                'description' => 'This option allows you to disable the invitation of new users to the Appwrite console. When enabled, console users are allowed to invite new users to a project. By default this option is enabled.',
-                'introduction' => '1.2.0',
-                'default' => 'enabled',
-                'required' => false,
-                'question' => '',
-                'filter' => ''
-            ],
         ],
     ],
     [
@@ -324,33 +315,54 @@ return [
                 'question' => '',
                 'filter' => 'password'
             ],
+        ],
+    ],
+    [
+        'category' => 'InfluxDB',
+        'description' => 'Appwrite uses an InfluxDB server for managing time-series data and server stats. The InfluxDB env vars are used to allow Appwrite server to connect to the InfluxDB container.',
+        'variables' => [
             [
-                'name' => '_APP_CONNECTIONS_MAX',
-                'description' => 'MariaDB server maximum connections.',
-                'introduction' => 'TBD',
-                'default' => 251,
+                'name' => '_APP_INFLUXDB_HOST',
+                'description' => 'InfluxDB server host name address. Default value is: \'influxdb\'.',
+                'introduction' => '',
+                'default' => 'influxdb',
+                'required' => false,
+                'question' => '',
+                'filter' => ''
+            ],
+            [
+                'name' => '_APP_INFLUXDB_PORT',
+                'description' => 'InfluxDB server TCP port. Default value is: \'8086\'.',
+                'introduction' => '',
+                'default' => '8086',
+                'required' => false,
+                'question' => '',
+                'filter' => ''
+            ],
+        ],
+    ],
+    [
+        'category' => 'StatsD',
+        'description' => 'Appwrite uses a StatsD server for aggregating and sending stats data over a fast UDP connection. The StatsD env vars are used to allow Appwrite server to connect to the StatsD container.',
+        'variables' => [
+            [
+                'name' => '_APP_STATSD_HOST',
+                'description' => 'StatsD server host name address. Default value is: \'telegraf\'.',
+                'introduction' => '',
+                'default' => 'telegraf',
+                'required' => false,
+                'question' => '',
+                'filter' => ''
+            ],
+            [
+                'name' => '_APP_STATSD_PORT',
+                'description' => 'StatsD server TCP port. Default value is: \'8125\'.',
+                'introduction' => '',
+                'default' => '8125',
                 'required' => false,
                 'question' => '',
                 'filter' => ''
             ],
-//             [
-//                 'name' => '_APP_CONNECTIONS_DB_PROJECT',
-//                 'description' => 'A list of comma-separated key value pairs representing Project DBs where key is the database name and value is the DSN connection string.',
-//                 'introduction' => 'TBD',
-//                 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite',
-//                 'required' => true,
-//                 'question' => '',
-//                 'filter' => ''
-//             ],
-//             [
-//                 'name' => '_APP_CONNECTIONS_DB_CONSOLE',
-//                 'description' => 'A key value pair representing the Console DB where key is the database name and value is the DSN connection string.',
-//                 'introduction' => 'TBD',
-//                 'default' => 'db_fra1_01=mysql://user:password@mariadb:3306/appwrite',
-//                 'required' => true,
-//                 'question' => '',
-//                 'filter' => ''
-//             ]
         ],
     ],
     [
@@ -477,17 +489,9 @@ return [
                 'question' => '',
                 'filter' => ''
             ],
-            [
-                'name' => '_APP_CONNECTIONS_STORAGE',
-                'description' => 'A DSN representing the storage device to connect to. The DSN takes the following format <device>://<access_key>:<access_secret>@<host>:<port>/<bucket>?region=<region>. For example, for S3: \'s3://access_key:access_secret@host:port/bucket?region=us-east-1\'. To use the local filesystem, you can leave this variable empty. Available devices are local, s3, dospaces, linode, backblaze and wasabi.',
-                'introduction' => '1.1.0',
-                'default' => '',
-                'required' => false,
-                'question' => '',
-            ],
             [
                 'name' => '_APP_STORAGE_DEVICE',
-                'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
+                'description' => 'Select default storage device. The default value is \'local\'. List of supported adapters are \'local\', \'s3\', \'dospaces\', \'backblaze\', \'linode\' and \'wasabi\'.',
                 'introduction' => '0.13.0',
                 'default' => 'local',
                 'required' => false,
@@ -495,7 +499,7 @@ return [
             ],
             [
                 'name' => '_APP_STORAGE_S3_ACCESS_KEY',
-                'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
+                'description' => 'AWS S3 storage access key. Required when the storage adapter is set to S3. You can get your access key from your AWS console',
                 'introduction' => '0.13.0',
                 'default' => '',
                 'required' => false,
@@ -503,7 +507,7 @@ return [
             ],
             [
                 'name' => '_APP_STORAGE_S3_SECRET',
-                'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
+                'description' => 'AWS S3 storage secret key. Required when the storage adapter is set to S3. You can get your secret key from your AWS console.',
                 'introduction' => '0.13.0',
                 'default' => '',
                 'required' => false,
@@ -511,7 +515,7 @@ return [
             ],
             [
                 'name' => '_APP_STORAGE_S3_REGION',
-                'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
+                'description' => 'AWS S3 storage region. Required when storage adapter is set to S3. You can find your region info for your bucket from AWS console.',
                 'introduction' => '0.13.0',
                 'default' => 'us-east-1',
                 'required' => false,
@@ -519,7 +523,7 @@ return [
             ],
             [
                 'name' => '_APP_STORAGE_S3_BUCKET',
-                'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
+                'description' => 'AWS S3 storage bucket. Required when storage adapter is set to S3. You can create buckets in your AWS console.',
                 'introduction' => '0.13.0',
                 'default' => '',
                 'required' => false,
@@ -527,7 +531,7 @@ return [
             ],
             [
                 'name' => '_APP_STORAGE_DO_SPACES_ACCESS_KEY',
-                'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
+                'description' => 'DigitalOcean spaces access key. Required when the storage adapter is set to DOSpaces. You can get your access key from your DigitalOcean console.',
                 'introduction' => '0.13.0',
                 'default' => '',
                 'required' => false,
@@ -535,7 +539,7 @@ return [
             ],
             [
                 'name' => '_APP_STORAGE_DO_SPACES_SECRET',
-                'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
+                'description' => 'DigitalOcean spaces secret key. Required when the storage adapter is set to DOSpaces. You can get your secret key from your DigitalOcean console.',
                 'introduction' => '0.13.0',
                 'default' => '',
                 'required' => false,
@@ -543,7 +547,7 @@ return [
             ],
             [
                 'name' => '_APP_STORAGE_DO_SPACES_REGION',
-                'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
+                'description' => 'DigitalOcean spaces region. Required when storage adapter is set to DOSpaces. You can find your region info for your space from DigitalOcean console.',
                 'introduction' => '0.13.0',
                 'default' => 'us-east-1',
                 'required' => false,
@@ -551,7 +555,7 @@ return [
             ],
             [
                 'name' => '_APP_STORAGE_DO_SPACES_BUCKET',
-                'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
+                'description' => 'DigitalOcean spaces bucket. Required when storage adapter is set to DOSpaces. You can create spaces in your DigitalOcean console.',
                 'introduction' => '0.13.0',
                 'default' => '',
                 'required' => false,
@@ -559,7 +563,7 @@ return [
             ],
             [
                 'name' => '_APP_STORAGE_BACKBLAZE_ACCESS_KEY',
-                'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
+                'description' => 'Backblaze access key. Required when the storage adapter is set to Backblaze. Your Backblaze keyID will be your access key. You can get your keyID from your Backblaze console.',
                 'introduction' => '0.14.2',
                 'default' => '',
                 'required' => false,
@@ -567,7 +571,7 @@ return [
             ],
             [
                 'name' => '_APP_STORAGE_BACKBLAZE_SECRET',
-                'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
+                'description' => 'Backblaze secret key. Required when the storage adapter is set to Backblaze. Your Backblaze applicationKey will be your secret key. You can get your applicationKey from your Backblaze console.',
                 'introduction' => '0.14.2',
                 'default' => '',
                 'required' => false,
@@ -575,7 +579,7 @@ return [
             ],
             [
                 'name' => '_APP_STORAGE_BACKBLAZE_REGION',
-                'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
+                'description' => 'Backblaze region. Required when storage adapter is set to Backblaze. You can find your region info from your Backblaze console.',
                 'introduction' => '0.14.2',
                 'default' => 'us-west-004',
                 'required' => false,
@@ -583,7 +587,7 @@ return [
             ],
             [
                 'name' => '_APP_STORAGE_BACKBLAZE_BUCKET',
-                'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
+                'description' => 'Backblaze bucket. Required when storage adapter is set to Backblaze. You can create your bucket from your Backblaze console.',
                 'introduction' => '0.14.2',
                 'default' => '',
                 'required' => false,
@@ -591,7 +595,7 @@ return [
             ],
             [
                 'name' => '_APP_STORAGE_LINODE_ACCESS_KEY',
-                'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
+                'description' => 'Linode object storage access key. Required when the storage adapter is set to Linode. You can get your access key from your Linode console.',
                 'introduction' => '0.14.2',
                 'default' => '',
                 'required' => false,
@@ -599,7 +603,7 @@ return [
             ],
             [
                 'name' => '_APP_STORAGE_LINODE_SECRET',
-                'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
+                'description' => 'Linode object storage secret key. Required when the storage adapter is set to Linode. You can get your secret key from your Linode console.',
                 'introduction' => '0.14.2',
                 'default' => '',
                 'required' => false,
@@ -607,7 +611,7 @@ return [
             ],
             [
                 'name' => '_APP_STORAGE_LINODE_REGION',
-                'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
+                'description' => 'Linode object storage region. Required when storage adapter is set to Linode. You can find your region info from your Linode console.',
                 'introduction' => '0.14.2',
                 'default' => 'eu-central-1',
                 'required' => false,
@@ -615,7 +619,7 @@ return [
             ],
             [
                 'name' => '_APP_STORAGE_LINODE_BUCKET',
-                'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
+                'description' => 'Linode object storage bucket. Required when storage adapter is set to Linode. You can create buckets in your Linode console.',
                 'introduction' => '0.14.2',
                 'default' => '',
                 'required' => false,
@@ -623,7 +627,7 @@ return [
             ],
             [
                 'name' => '_APP_STORAGE_WASABI_ACCESS_KEY',
-                'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
+                'description' => 'Wasabi access key. Required when the storage adapter is set to Wasabi. You can get your access key from your Wasabi console.',
                 'introduction' => '0.14.2',
                 'default' => '',
                 'required' => false,
@@ -631,7 +635,7 @@ return [
             ],
             [
                 'name' => '_APP_STORAGE_WASABI_SECRET',
-                'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
+                'description' => 'Wasabi secret key. Required when the storage adapter is set to Wasabi. You can get your secret key from your Wasabi console.',
                 'introduction' => '0.14.2',
                 'default' => '',
                 'required' => false,
@@ -639,7 +643,7 @@ return [
             ],
             [
                 'name' => '_APP_STORAGE_WASABI_REGION',
-                'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
+                'description' => 'Wasabi region. Required when storage adapter is set to Wasabi. You can find your region info from your Wasabi console.',
                 'introduction' => '0.14.2',
                 'default' => 'eu-central-1',
                 'required' => false,
@@ -647,7 +651,7 @@ return [
             ],
             [
                 'name' => '_APP_STORAGE_WASABI_BUCKET',
-                'description' => 'Deprecated since 1.2.0. Use _APP_CONNECTIONS_STORAGE instead.',
+                'description' => 'Wasabi bucket. Required when storage adapter is set to Wasabi. You can create buckets in your Wasabi console.',
                 'introduction' => '0.14.2',
                 'default' => '',
                 'required' => false,
@@ -814,7 +818,7 @@ return [
             ],
             [
                 'name' => '_APP_FUNCTIONS_RUNTIMES_NETWORK',
-                'description' => 'The docker network used for communication between the executor and runtimes. Change this if you have altered the default network names.',
+                'description' => 'The docker network used for communication between the executor and runtimes.',
                 'introduction' => '1.2.0',
                 'default' => 'runtimes',
                 'required' => false,
@@ -850,6 +854,66 @@ return [
             ],
         ],
     ],
+    [
+        'category' => 'VCS (Version Control System)',
+        'description' => '',
+        'variables' => [
+            [
+                'name' => '_APP_VCS_GITHUB_APP_NAME',
+                'description' => 'Name of your GitHub app. This value should be set to your GitHub application\'s URL.',
+                'introduction' => '1.4.0',
+                'default' => '',
+                'required' => false,
+                'question' => '',
+                'filter' => ''
+            ],
+            [
+                'name' => '_APP_VCS_GITHUB_PRIVATE_KEY',
+                'description' => 'GitHub app RSA private key. You can generate private keys from GitHub application settings.',
+                'introduction' => '1.4.0',
+                'default' => '',
+                'required' => false,
+                'question' => '',
+                'filter' => ''
+            ],
+            [
+                'name' => '_APP_VCS_GITHUB_APP_ID',
+                'description' => 'GitHub application ID. You can find it in your GitHub application details.',
+                'introduction' => '1.4.0',
+                'default' => '',
+                'required' => false,
+                'question' => '',
+                'filter' => ''
+            ],
+            [
+                'name' => '_APP_VCS_GITHUB_CLIENT_ID',
+                'description' => 'GitHub client ID. You can find it in your GitHub application details.',
+                'introduction' => '1.4.0',
+                'default' => '',
+                'required' => false,
+                'question' => '',
+                'filter' => ''
+            ],
+            [
+                'name' => '_APP_VCS_GITHUB_CLIENT_SECRET',
+                'description' => 'GitHub client secret. You can generate secrets in your GitHub application settings.',
+                'introduction' => '1.4.0',
+                'default' => '',
+                'required' => false,
+                'question' => '',
+                'filter' => ''
+            ],
+            [
+                'name' => '_APP_VCS_GITHUB_WEBHOOK_SECRET',
+                'description' => 'GitHub webhook secret. You can configure it in your GitHub application settings under webhook section.',
+                'introduction' => '1.4.0',
+                'default' => '',
+                'required' => false,
+                'question' => '',
+                'filter' => ''
+            ],
+        ],
+    ],
     [
         'category' => 'Maintenance',
         'description' => '',
@@ -952,4 +1016,43 @@ return [
             ],
         ],
     ],
+    [
+        'category' => 'Migrations',
+        'description' => '',
+        'variables' => [
+            [
+                'name' => '_APP_MIGRATIONS_FIREBASE_CLIENT_ID',
+                'description' => 'Google OAuth client ID. You can find it in your GCP application settings.',
+                'introduction' => '1.4.0',
+                'default' => '',
+                'required' => false,
+                'question' => '',
+                'filter' => ''
+            ],
+            [
+                'name' => '_APP_MIGRATIONS_FIREBASE_CLIENT_SECRET',
+                'description' => 'Google OAuth client secret. You can generate secrets in your GCP application settings.',
+                'introduction' => '1.4.0',
+                'default' => '',
+                'required' => false,
+                'question' => '',
+                'filter' => ''
+            ]
+        ]
+    ],
+    [
+        'category' => 'Assistant',
+        'description' => '',
+        'variables' => [
+            [
+                'name' => '_APP_ASSISTANT_OPENAI_API_KEY',
+                'description' => 'OpenAI API key. You can find it in your OpenAI application settings.',
+                'introduction' => '1.4.0',
+                'default' => '',
+                'required' => false,
+                'question' => '',
+                'filter' => ''
+            ]
+        ]
+    ]
 ];

app/console

@@ -1 +1 @@
-Subproject commit 9174d8f8cb584744dd7a53f69d324f490ee82ee3
+Subproject commit 8ced2e82dd1cba2c2790c6562211b36f44198d89

app/controllers/api/account.php

@@ -58,6 +58,7 @@ App::post('/v1/account/invite')
     ->label('audits.event', 'user.create')
     ->label('audits.resource', 'user/{response.$id}')
     ->label('audits.userId', '{response.$id}')
+    ->label('usage.metric', 'users.{scope}.requests.create')
     ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'createWithInviteCode')
@@ -84,13 +85,16 @@ App::post('/v1/account/invite')
 
         $email = \strtolower($email);
 
-        $whitelistCodes = (!empty(App::getEnv('_APP_CONSOLE_WHITELIST_CODES', null))) ? \explode(',', App::getEnv('_APP_CONSOLE_WHITELIST_CODES', null)) : [];
+        $codes = App::getEnv('_APP_CONSOLE_WHITELIST_CODES');
+        $whitelistCodes = !empty($codes)
+            ? \explode(',', $codes)
+            : [];
 
         if (empty($whitelistCodes)) {
             throw new Exception(Exception::GENERAL_CODES_DISABLED);
         }
 
-        if (!empty($whitelistCodes) && !\in_array($code, $whitelistCodes)) {
+        if (!\in_array($code, $whitelistCodes)) {
             throw new Exception(Exception::USER_INVALID_CODE);
         }
 
@@ -127,9 +131,10 @@ App::post('/v1/account/invite')
                 'sessions' => null,
                 'tokens' => null,
                 'memberships' => null,
-                'search' => implode(' ', [$userId, $email, $name])
+                'search' => implode(' ', [$userId, $email, $name]),
+                'accessedAt' => DateTime::now(),
             ])));
-        } catch (Duplicate $th) {
+        } catch (Duplicate) {
             throw new Exception(Exception::USER_ALREADY_EXISTS);
         }
 
@@ -153,6 +158,7 @@ App::post('/v1/account')
     ->label('audits.event', 'user.create')
     ->label('audits.resource', 'user/{response.$id}')
     ->label('audits.userId', '{response.$id}')
+    ->label('usage.metric', 'users.{scope}.requests.create')
     ->label('sdk.auth', [])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'create')
@@ -238,10 +244,10 @@ App::post('/v1/account')
                 'tokens' => null,
                 'memberships' => null,
                 'search' => implode(' ', [$userId, $email, $name]),
-                'accessedAt' => DateTime::now(), // Add this here to make sure it's returned in the response
+                'accessedAt' => DateTime::now(),
             ]);
             Authorization::skip(fn() => $dbForProject->createDocument('users', $user));
-        } catch (Duplicate $th) {
+        } catch (Duplicate) {
             throw new Exception(Exception::USER_ALREADY_EXISTS);
         }
 
@@ -266,6 +272,8 @@ App::post('/v1/account/sessions/email')
     ->label('audits.event', 'session.create')
     ->label('audits.resource', 'user/{response.userId}')
     ->label('audits.userId', '{response.userId}')
+    ->label('usage.metric', 'sessions.{scope}.requests.create')
+    ->label('usage.params', ['provider:email'])
     ->label('sdk.auth', [])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'createEmailSession')
@@ -390,8 +398,8 @@ App::get('/v1/account/sessions/oauth2/:provider')
     ->label('abuse-limit', 50)
     ->label('abuse-key', 'ip:{ip}')
     ->param('provider', '', new WhiteList(\array_keys(Config::getParam('providers')), true), 'OAuth2 Provider. Currently, supported providers are: ' . \implode(', ', \array_keys(\array_filter(Config::getParam('providers'), fn($node) => (!$node['mock'])))) . '.')
-    ->param('success', '', fn($clients) => new Host($clients), 'URL to redirect back to your app after a successful login attempt.  Only URLs from hostnames in your project platform list are allowed. This requirement helps to prevent an [open redirect](https://cheatsheetseries.owasp.org/cheatsheets/Unvalidated_Redirects_and_Forwards_Cheat_Sheet.html) attack against your project API.', true, ['clients'])
-    ->param('failure', '', fn($clients) => new Host($clients), 'URL to redirect back to your app after a failed login attempt.  Only URLs from hostnames in your project platform list are allowed. This requirement helps to prevent an [open redirect](https://cheatsheetseries.owasp.org/cheatsheets/Unvalidated_Redirects_and_Forwards_Cheat_Sheet.html) attack against your project API.', true, ['clients'])
+    ->param('success', '', fn($clients) => new Host($clients), 'URL to redirect back to your app after a successful login attempt.  Only URLs from hostnames in your project\'s platform list are allowed. This requirement helps to prevent an [open redirect](https://cheatsheetseries.owasp.org/cheatsheets/Unvalidated_Redirects_and_Forwards_Cheat_Sheet.html) attack against your project API.', true, ['clients'])
+    ->param('failure', '', fn($clients) => new Host($clients), 'URL to redirect back to your app after a failed login attempt.  Only URLs from hostnames in your project\'s platform list are allowed. This requirement helps to prevent an [open redirect](https://cheatsheetseries.owasp.org/cheatsheets/Unvalidated_Redirects_and_Forwards_Cheat_Sheet.html) attack against your project API.', true, ['clients'])
     ->param('scopes', [], new ArrayList(new Text(APP_LIMIT_ARRAY_ELEMENT_SIZE), APP_LIMIT_ARRAY_PARAMS_SIZE), 'A list of custom OAuth2 scopes. Check each provider internal docs for a list of supported scopes. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' scopes are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long.', true)
     ->inject('request')
     ->inject('response')
@@ -449,7 +457,7 @@ App::get('/v1/account/sessions/oauth2/callback/:provider/:projectId')
     ->label('docs', false)
     ->param('projectId', '', new Text(1024), 'Project ID.')
     ->param('provider', '', new WhiteList(\array_keys(Config::getParam('providers')), true), 'OAuth2 provider.')
-    ->param('code', '', new Text(2048, 0), 'OAuth2 code.', true)
+    ->param('code', '', new Text(2048, 0), 'OAuth2 code. This is a temporary code that the will be later exchanged for an access token.', true)
     ->param('state', '', new Text(2048), 'Login state params.', true)
     ->param('error', '', new Text(2048, 0), 'Error code returned from the OAuth2 provider.', true)
     ->param('error_description', '', new Text(2048, 0), 'Human-readable text providing additional information about the error returned from the OAuth2 provider.', true)
@@ -482,7 +490,7 @@ App::post('/v1/account/sessions/oauth2/callback/:provider/:projectId')
     ->label('docs', false)
     ->param('projectId', '', new Text(1024), 'Project ID.')
     ->param('provider', '', new WhiteList(\array_keys(Config::getParam('providers')), true), 'OAuth2 provider.')
-    ->param('code', '', new Text(2048, 0), 'OAuth2 code.', true)
+    ->param('code', '', new Text(2048, 0), 'OAuth2 code. This is a temporary code that the will be later exchanged for an access token.', true)
     ->param('state', '', new Text(2048), 'Login state params.', true)
     ->param('error', '', new Text(2048, 0), 'Error code returned from the OAuth2 provider.', true)
     ->param('error_description', '', new Text(2048, 0), 'Human-readable text providing additional information about the error returned from the OAuth2 provider.', true)
@@ -518,8 +526,10 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect')
     ->label('abuse-limit', 50)
     ->label('abuse-key', 'ip:{ip}')
     ->label('docs', false)
+    ->label('usage.metric', 'sessions.{scope}.requests.create')
+    ->label('usage.params', ['provider:{request.provider}'])
     ->param('provider', '', new WhiteList(\array_keys(Config::getParam('providers')), true), 'OAuth2 provider.')
-    ->param('code', '', new Text(2048, 0), 'OAuth2 code.', true)
+    ->param('code', '', new Text(2048, 0), 'OAuth2 code. This is a temporary code that the will be later exchanged for an access token.', true)
     ->param('state', '', new Text(2048), 'OAuth2 state params.', true)
     ->param('error', '', new Text(2048, 0), 'Error code returned from the OAuth2 provider.', true)
     ->param('error_description', '', new Text(2048, 0), 'Human-readable text providing additional information about the error returned from the OAuth2 provider.', true)
@@ -740,10 +750,11 @@ App::get('/v1/account/sessions/oauth2/:provider/redirect')
                         'sessions' => null,
                         'tokens' => null,
                         'memberships' => null,
-                        'search' => implode(' ', [$userId, $email, $name])
+                        'search' => implode(' ', [$userId, $email, $name]),
+                        'accessedAt' => DateTime::now(),
                     ]);
                     Authorization::skip(fn() => $dbForProject->createDocument('users', $user));
-                } catch (Duplicate $th) {
+                } catch (Duplicate) {
                     $failureRedirect(Exception::USER_ALREADY_EXISTS);
                 }
             }
@@ -1040,7 +1051,8 @@ App::post('/v1/account/sessions/magic-url')
                 'sessions' => null,
                 'tokens' => null,
                 'memberships' => null,
-                'search' => implode(' ', [$userId, $email])
+                'search' => implode(' ', [$userId, $email]),
+                'accessedAt' => DateTime::now(),
             ]);
 
             Authorization::skip(fn () => $dbForProject->createDocument('users', $user));
@@ -1079,39 +1091,80 @@ App::post('/v1/account/sessions/magic-url')
         $url['query'] = Template::mergeQuery(((isset($url['query'])) ? $url['query'] : ''), ['userId' => $user->getId(), 'secret' => $loginSecret, 'expire' => $expire, 'project' => $project->getId()]);
         $url = Template::unParseURL($url);
 
-        $from = $project->isEmpty() || $project->getId() === 'console' ? '' : \sprintf($locale->getText('emails.sender'), $project->getAttribute('name'));
-        $body = Template::fromFile(__DIR__ . '/../../config/locale/templates/email-base.tpl');
+        $body = $locale->getText("emails.magicSession.body");
         $subject = $locale->getText("emails.magicSession.subject");
-
-        $smtpEnabled = $project->getAttribute('smtp', [])['enabled'] ?? false;
         $customTemplate = $project->getAttribute('templates', [])['email.magicSession-' . $locale->default] ?? [];
-        if ($smtpEnabled && !empty($customTemplate)) {
-            $body = $customTemplate['message'] ?? $body;
-            $subject = $customTemplate['subject'] ?? $subject;
-            $from = $customTemplate['senderName'] ?? $from;
+
+        $message = Template::fromFile(__DIR__ . '/../../config/locale/templates/email-inner-base.tpl');
+        $message->setParam('{{body}}', $body);
+        $body = $message->render();
+
+        $smtp = $project->getAttribute('smtp', []);
+        $smtpEnabled = $smtp['enabled'] ?? false;
+
+        $senderEmail = App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM);
+        $senderName = App::getEnv('_APP_SYSTEM_EMAIL_NAME', APP_NAME . ' Server');
+        $replyTo = "";
+
+        if ($smtpEnabled) {
+            if (!empty($smtp['senderEmail'])) {
+                $senderEmail = $smtp['senderEmail'];
+            }
+            if (!empty($smtp['senderName'])) {
+                $senderName = $smtp['senderName'];
+            }
+            if (!empty($smtp['replyTo'])) {
+                $replyTo = $smtp['replyTo'];
+            }
+
+            $mails
+                ->setSmtpHost($smtp['host'] ?? '')
+                ->setSmtpPort($smtp['port'] ?? '')
+                ->setSmtpUsername($smtp['username'] ?? '')
+                ->setSmtpPassword($smtp['password'] ?? '')
+                ->setSmtpSecure($smtp['secure'] ?? '');
         }
 
-        $body
-            ->setParam('{{subject}}', $subject)
-            ->setParam('{{hello}}', $locale->getText("emails.magicSession.hello"))
-            ->setParam('{{name}}', '')
-            ->setParam('{{body}}', $locale->getText("emails.magicSession.body"))
-            ->setParam('{{redirect}}', $url)
-            ->setParam('{{footer}}', $locale->getText("emails.magicSession.footer"))
-            ->setParam('{{thanks}}', $locale->getText("emails.magicSession.thanks"))
-            ->setParam('{{signature}}', $locale->getText("emails.magicSession.signature"))
-            ->setParam('{{project}}', $project->getAttribute('name'))
-            ->setParam('{{direction}}', $locale->getText('settings.direction'))
-            ->setParam('{{bg-body}}', '#f7f7f7')
-            ->setParam('{{bg-content}}', '#ffffff')
-            ->setParam('{{text-content}}', '#000000');
+        if (!empty($customTemplate)) {
+            if (!empty($customTemplate['senderEmail'])) {
+                $senderEmail = $customTemplate['senderEmail'];
+            }
+            if (!empty($customTemplate['senderName'])) {
+                $senderName = $customTemplate['senderName'];
+            }
+            if (!empty($customTemplate['replyTo'])) {
+                $replyTo = $customTemplate['replyTo'];
+            }
 
-        $body = $body->render();
+            $body = $customTemplate['message'] ?? '';
+            $subject = $customTemplate['subject'] ?? $subject;
+        }
+
+        $mails
+            ->setSmtpReplyTo($replyTo)
+            ->setSmtpSenderEmail($senderEmail)
+            ->setSmtpSenderName($senderName);
+
+        $emailVariables = [
+            'subject' => $subject,
+            'hello' => $locale->getText("emails.magicSession.hello"),
+            'name' => '',
+            'body' => $body,
+            'redirect' => $url,
+            'footer' => $locale->getText("emails.magicSession.footer"),
+            'thanks' => $locale->getText("emails.magicSession.thanks"),
+            'signature' => $locale->getText("emails.magicSession.signature"),
+            'project' => $project->getAttribute('name'),
+            'direction' => $locale->getText('settings.direction'),
+            'bg-body' => '#f7f7f7',
+            'bg-content' => '#ffffff',
+            'text-content' => '#000000',
+        ];
 
         $mails
             ->setSubject($subject)
             ->setBody($body)
-            ->setFrom($from)
+            ->setVariables($emailVariables)
             ->setRecipient($user->getAttribute('email'))
             ->trigger()
         ;
@@ -1140,6 +1193,8 @@ App::put('/v1/account/sessions/magic-url')
     ->label('audits.event', 'session.update')
     ->label('audits.resource', 'user/{response.userId}')
     ->label('audits.userId', '{response.userId}')
+    ->label('usage.metric', 'sessions.{scope}.requests.create')
+    ->label('usage.params', ['provider:magic-url'])
     ->label('sdk.auth', [])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'updateMagicURLSession')
@@ -1327,7 +1382,8 @@ App::post('/v1/account/sessions/phone')
                 'sessions' => null,
                 'tokens' => null,
                 'memberships' => null,
-                'search' => implode(' ', [$userId, $phone])
+                'search' => implode(' ', [$userId, $phone]),
+                'accessedAt' => DateTime::now(),
             ]);
 
             Authorization::skip(fn () => $dbForProject->createDocument('users', $user));
@@ -1514,6 +1570,8 @@ App::post('/v1/account/sessions/anonymous')
     ->label('audits.event', 'session.create')
     ->label('audits.resource', 'user/{response.userId}')
     ->label('audits.userId', '{response.userId}')
+    ->label('usage.metric', 'sessions.{scope}.requests.create')
+    ->label('usage.params', ['provider:anonymous'])
     ->label('sdk.auth', [])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'createAnonymousSession')
@@ -1576,6 +1634,7 @@ App::post('/v1/account/sessions/anonymous')
             'tokens' => null,
             'memberships' => null,
             'search' => $userId,
+            'accessedAt' => DateTime::now(),
         ]);
         Authorization::skip(fn() => $dbForProject->createDocument('users', $user));
 
@@ -1689,6 +1748,7 @@ App::get('/v1/account')
     ->desc('Get Account')
     ->groups(['api', 'account'])
     ->label('scope', 'account')
+    ->label('usage.metric', 'users.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'get')
@@ -1709,6 +1769,7 @@ App::get('/v1/account/prefs')
     ->desc('Get Account Preferences')
     ->groups(['api', 'account'])
     ->label('scope', 'account')
+    ->label('usage.metric', 'users.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'getPrefs')
@@ -1731,6 +1792,7 @@ App::get('/v1/account/sessions')
     ->desc('List Sessions')
     ->groups(['api', 'account'])
     ->label('scope', 'account')
+    ->label('usage.metric', 'users.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'listSessions')
@@ -1769,6 +1831,7 @@ App::get('/v1/account/logs')
     ->desc('List Logs')
     ->groups(['api', 'account'])
     ->label('scope', 'account')
+    ->label('usage.metric', 'users.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'listLogs')
@@ -1829,6 +1892,7 @@ App::get('/v1/account/sessions/:sessionId')
     ->desc('Get Session')
     ->groups(['api', 'account'])
     ->label('scope', 'account')
+    ->label('usage.metric', 'users.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'getSession')
@@ -1876,6 +1940,7 @@ App::patch('/v1/account/name')
     ->label('scope', 'account')
     ->label('audits.event', 'user.update')
     ->label('audits.resource', 'user/{response.$id}')
+    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'updateName')
@@ -1910,6 +1975,7 @@ App::patch('/v1/account/password')
     ->label('audits.event', 'user.update')
     ->label('audits.resource', 'user/{response.$id}')
     ->label('audits.userId', '{response.$id}')
+    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'updatePassword')
@@ -1975,6 +2041,7 @@ App::patch('/v1/account/email')
     ->label('scope', 'account')
     ->label('audits.event', 'user.update')
     ->label('audits.resource', 'user/{response.$id}')
+    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'updateEmail')
@@ -2044,6 +2111,7 @@ App::patch('/v1/account/phone')
     ->label('scope', 'account')
     ->label('audits.event', 'user.update')
     ->label('audits.resource', 'user/{response.$id}')
+    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'updatePhone')
@@ -2102,6 +2170,7 @@ App::patch('/v1/account/prefs')
     ->label('scope', 'account')
     ->label('audits.event', 'user.update')
     ->label('audits.resource', 'user/{response.$id}')
+    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'updatePrefs')
@@ -2135,6 +2204,7 @@ App::patch('/v1/account/status')
     ->label('scope', 'account')
     ->label('audits.event', 'user.update')
     ->label('audits.resource', 'user/{response.$id}')
+    ->label('usage.metric', 'users.{scope}.requests.delete')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'updateStatus')
@@ -2178,6 +2248,7 @@ App::delete('/v1/account/sessions/:sessionId')
     ->label('event', 'users.[userId].sessions.[sessionId].delete')
     ->label('audits.event', 'session.delete')
     ->label('audits.resource', 'user/{user.$id}')
+    ->label('usage.metric', 'sessions.{scope}.requests.delete')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'deleteSession')
@@ -2254,6 +2325,7 @@ App::patch('/v1/account/sessions/:sessionId')
     ->label('audits.event', 'session.update')
     ->label('audits.resource', 'user/{response.userId}')
     ->label('audits.userId', '{response.userId}')
+    ->label('usage.metric', 'sessions.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'updateSession')
@@ -2338,6 +2410,7 @@ App::delete('/v1/account/sessions')
     ->label('event', 'users.[userId].sessions.[sessionId].delete')
     ->label('audits.event', 'session.delete')
     ->label('audits.resource', 'user/{user.$id}')
+    ->label('usage.metric', 'sessions.{scope}.requests.delete')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'deleteSessions')
@@ -2399,6 +2472,7 @@ App::post('/v1/account/recovery')
     ->label('audits.event', 'recovery.create')
     ->label('audits.resource', 'user/{response.userId}')
     ->label('audits.userId', '{response.userId}')
+    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'createRecovery')
@@ -2474,41 +2548,82 @@ App::post('/v1/account/recovery')
         $url = Template::unParseURL($url);
 
         $projectName = $project->isEmpty() ? 'Console' : $project->getAttribute('name', '[APP-NAME]');
-        $from = $project->isEmpty() || $project->getId() === 'console' ? '' : \sprintf($locale->getText('emails.sender'), $projectName);
-        $body = Template::fromFile(__DIR__ . '/../../config/locale/templates/email-base.tpl');
+        $body = $locale->getText("emails.recovery.body");
         $subject = $locale->getText("emails.recovery.subject");
-
-        $smtpEnabled = $project->getAttribute('smtp', [])['enabled'] ?? false;
         $customTemplate = $project->getAttribute('templates', [])['email.recovery-' . $locale->default] ?? [];
-        if ($smtpEnabled && !empty($customTemplate)) {
-            $body = $customTemplate['message'] ?? $body;
-            $subject = $customTemplate['subject'] ?? $subject;
-            $from = $customTemplate['senderName'] ?? $from;
+
+        $message = Template::fromFile(__DIR__ . '/../../config/locale/templates/email-inner-base.tpl');
+        $message->setParam('{{body}}', $body);
+        $body = $message->render();
+
+        $smtp = $project->getAttribute('smtp', []);
+        $smtpEnabled = $smtp['enabled'] ?? false;
+
+        $senderEmail = App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM);
+        $senderName = App::getEnv('_APP_SYSTEM_EMAIL_NAME', APP_NAME . ' Server');
+        $replyTo = "";
+
+        if ($smtpEnabled) {
+            if (!empty($smtp['senderEmail'])) {
+                $senderEmail = $smtp['senderEmail'];
+            }
+            if (!empty($smtp['senderName'])) {
+                $senderName = $smtp['senderName'];
+            }
+            if (!empty($smtp['replyTo'])) {
+                $replyTo = $smtp['replyTo'];
+            }
+
+            $mails
+                ->setSmtpHost($smtp['host'] ?? '')
+                ->setSmtpPort($smtp['port'] ?? '')
+                ->setSmtpUsername($smtp['username'] ?? '')
+                ->setSmtpPassword($smtp['password'] ?? '')
+                ->setSmtpSecure($smtp['secure'] ?? '');
         }
 
-        $body
-            ->setParam('{{subject}}', $subject)
-            ->setParam('{{hello}}', $locale->getText("emails.recovery.hello"))
-            ->setParam('{{name}}', $profile->getAttribute('name'))
-            ->setParam('{{body}}', $locale->getText("emails.recovery.body"))
-            ->setParam('{{redirect}}', $url)
-            ->setParam('{{footer}}', $locale->getText("emails.recovery.footer"))
-            ->setParam('{{thanks}}', $locale->getText("emails.recovery.thanks"))
-            ->setParam('{{signature}}', $locale->getText("emails.recovery.signature"))
-            ->setParam('{{project}}', $projectName)
-            ->setParam('{{direction}}', $locale->getText('settings.direction'))
-            ->setParam('{{bg-body}}', '#f7f7f7')
-            ->setParam('{{bg-content}}', '#ffffff')
-            ->setParam('{{text-content}}', '#000000');
+        if (!empty($customTemplate)) {
+            if (!empty($customTemplate['senderEmail'])) {
+                $senderEmail = $customTemplate['senderEmail'];
+            }
+            if (!empty($customTemplate['senderName'])) {
+                $senderName = $customTemplate['senderName'];
+            }
+            if (!empty($customTemplate['replyTo'])) {
+                $replyTo = $customTemplate['replyTo'];
+            }
 
-        $body = $body->render();
+            $body = $customTemplate['message'] ?? '';
+            $subject = $customTemplate['subject'] ?? $subject;
+        }
+
+        $mails
+            ->setSmtpReplyTo($replyTo)
+            ->setSmtpSenderEmail($senderEmail)
+            ->setSmtpSenderName($senderName);
+
+        $emailVariables = [
+            'subject' => $subject,
+            'hello' => $locale->getText("emails.recovery.hello"),
+            'name' => $profile->getAttribute('name'),
+            'body' => $body,
+            'redirect' => $url,
+            'footer' => $locale->getText("emails.recovery.footer"),
+            'thanks' => $locale->getText("emails.recovery.thanks"),
+            'signature' => $locale->getText("emails.recovery.signature"),
+            'project' => $projectName,
+            'direction' => $locale->getText('settings.direction'),
+            'bg-body' => '#f7f7f7',
+            'bg-content' => '#ffffff',
+            'text-content' => '#000000',
+        ];
 
 
         $mails
             ->setRecipient($profile->getAttribute('email', ''))
             ->setName($profile->getAttribute('name'))
             ->setBody($body)
-            ->setFrom($from)
+            ->setVariables($emailVariables)
             ->setSubject($subject)
             ->trigger();
         ;
@@ -2539,6 +2654,7 @@ App::put('/v1/account/recovery')
     ->label('audits.event', 'recovery.update')
     ->label('audits.resource', 'user/{response.userId}')
     ->label('audits.userId', '{response.userId}')
+    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'updateRecovery')
@@ -2625,6 +2741,7 @@ App::post('/v1/account/verification')
     ->label('event', 'users.[userId].verification.[tokenId].create')
     ->label('audits.event', 'verification.create')
     ->label('audits.resource', 'user/{response.userId}')
+    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'createVerification')
@@ -2682,39 +2799,80 @@ App::post('/v1/account/verification')
         $url = Template::unParseURL($url);
 
         $projectName = $project->isEmpty() ? 'Console' : $project->getAttribute('name', '[APP-NAME]');
-        $from = $project->isEmpty() || $project->getId() === 'console' ? '' : \sprintf($locale->getText('emails.sender'), $projectName);
-        $body = Template::fromFile(__DIR__ . '/../../config/locale/templates/email-base.tpl');
+        $body = $locale->getText("emails.verification.body");
         $subject = $locale->getText("emails.verification.subject");
-
-        $smtpEnabled = $project->getAttribute('smtp', [])['enabled'] ?? false;
         $customTemplate = $project->getAttribute('templates', [])['email.verification-' . $locale->default] ?? [];
-        if ($smtpEnabled && !empty($customTemplate)) {
-            $body = $customTemplate['message'] ?? $body;
-            $subject = $customTemplate['subject'] ?? $subject;
-            $from = $customTemplate['senderName'] ?? $from;
+
+        $message = Template::fromFile(__DIR__ . '/../../config/locale/templates/email-inner-base.tpl');
+        $message->setParam('{{body}}', $body);
+        $body = $message->render();
+
+        $smtp = $project->getAttribute('smtp', []);
+        $smtpEnabled = $smtp['enabled'] ?? false;
+
+        $senderEmail = App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM);
+        $senderName = App::getEnv('_APP_SYSTEM_EMAIL_NAME', APP_NAME . ' Server');
+        $replyTo = "";
+
+        if ($smtpEnabled) {
+            if (!empty($smtp['senderEmail'])) {
+                $senderEmail = $smtp['senderEmail'];
+            }
+            if (!empty($smtp['senderName'])) {
+                $senderName = $smtp['senderName'];
+            }
+            if (!empty($smtp['replyTo'])) {
+                $replyTo = $smtp['replyTo'];
+            }
+
+            $mails
+                ->setSmtpHost($smtp['host'] ?? '')
+                ->setSmtpPort($smtp['port'] ?? '')
+                ->setSmtpUsername($smtp['username'] ?? '')
+                ->setSmtpPassword($smtp['password'] ?? '')
+                ->setSmtpSecure($smtp['secure'] ?? '');
         }
 
-        $body
-            ->setParam('{{subject}}', $subject)
-            ->setParam('{{hello}}', $locale->getText("emails.verification.hello"))
-            ->setParam('{{name}}', $user->getAttribute('name'))
-            ->setParam('{{body}}', $locale->getText("emails.verification.body"))
-            ->setParam('{{redirect}}', $url)
-            ->setParam('{{footer}}', $locale->getText("emails.verification.footer"))
-            ->setParam('{{thanks}}', $locale->getText("emails.verification.thanks"))
-            ->setParam('{{signature}}', $locale->getText("emails.verification.signature"))
-            ->setParam('{{project}}', $projectName)
-            ->setParam('{{direction}}', $locale->getText('settings.direction'))
-            ->setParam('{{bg-body}}', '#f7f7f7')
-            ->setParam('{{bg-content}}', '#ffffff')
-            ->setParam('{{text-content}}', '#000000');
+        if (!empty($customTemplate)) {
+            if (!empty($customTemplate['senderEmail'])) {
+                $senderEmail = $customTemplate['senderEmail'];
+            }
+            if (!empty($customTemplate['senderName'])) {
+                $senderName = $customTemplate['senderName'];
+            }
+            if (!empty($customTemplate['replyTo'])) {
+                $replyTo = $customTemplate['replyTo'];
+            }
 
-        $body = $body->render();
+            $body = $customTemplate['message'] ?? '';
+            $subject = $customTemplate['subject'] ?? $subject;
+        }
+
+        $mails
+            ->setSmtpReplyTo($replyTo)
+            ->setSmtpSenderEmail($senderEmail)
+            ->setSmtpSenderName($senderName);
+
+        $emailVariables = [
+            'subject' => $subject,
+            'hello' => $locale->getText("emails.verification.hello"),
+            'name' => $user->getAttribute('name'),
+            'body' => $body,
+            'redirect' => $url,
+            'footer' => $locale->getText("emails.verification.footer"),
+            'thanks' => $locale->getText("emails.verification.thanks"),
+            'signature' => $locale->getText("emails.verification.signature"),
+            'project' => $projectName,
+            'direction' => $locale->getText('settings.direction'),
+            'bg-body' => '#f7f7f7',
+            'bg-content' => '#ffffff',
+            'text-content' => '#000000',
+        ];
 
         $mails
             ->setSubject($subject)
             ->setBody($body)
-            ->setFrom($from)
+            ->setVariables($emailVariables)
             ->setRecipient($user->getAttribute('email'))
             ->setName($user->getAttribute('name') ?? '')
             ->trigger()
@@ -2744,6 +2902,7 @@ App::put('/v1/account/verification')
     ->label('event', 'users.[userId].verification.[tokenId].update')
     ->label('audits.event', 'verification.update')
     ->label('audits.resource', 'user/{response.userId}')
+    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'updateVerification')
@@ -2804,6 +2963,7 @@ App::post('/v1/account/verification/phone')
     ->label('event', 'users.[userId].verification.[tokenId].create')
     ->label('audits.event', 'verification.create')
     ->label('audits.resource', 'user/{response.userId}')
+    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'createPhoneVerification')
@@ -2899,6 +3059,7 @@ App::put('/v1/account/verification/phone')
     ->label('event', 'users.[userId].verification.[tokenId].update')
     ->label('audits.event', 'verification.update')
     ->label('audits.resource', 'user/{response.userId}')
+    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'account')
     ->label('sdk.method', 'updatePhoneVerification')

app/controllers/api/console.php

@@ -29,12 +29,27 @@ App::get('/v1/console/variables')
     ->label('sdk.response.model', Response::MODEL_CONSOLE_VARIABLES)
     ->inject('response')
     ->action(function (Response $response) {
+        $isDomainEnabled = !empty(App::getEnv('_APP_DOMAIN', ''))
+            && !empty(App::getEnv('_APP_DOMAIN_TARGET', ''))
+            && App::getEnv('_APP_DOMAIN', '') !== 'localhost'
+            && App::getEnv('_APP_DOMAIN_TARGET', '') !== 'localhost';
+
+        $isVcsEnabled = !empty(App::getEnv('_APP_VCS_GITHUB_APP_NAME', ''))
+            && !empty(App::getEnv('_APP_VCS_GITHUB_PRIVATE_KEY', ''))
+            && !empty(App::getEnv('_APP_VCS_GITHUB_APP_ID', ''))
+            && !empty(App::getEnv('_APP_VCS_GITHUB_CLIENT_ID', ''))
+            && !empty(App::getEnv('_APP_VCS_GITHUB_CLIENT_SECRET', ''));
+
+        $isAssistantEnabled = !empty(App::getEnv('_APP_ASSISTANT_OPENAI_API_KEY', ''));
 
         $variables = new Document([
             '_APP_DOMAIN_TARGET' => App::getEnv('_APP_DOMAIN_TARGET'),
             '_APP_STORAGE_LIMIT' => +App::getEnv('_APP_STORAGE_LIMIT'),
             '_APP_FUNCTIONS_SIZE_LIMIT' => +App::getEnv('_APP_FUNCTIONS_SIZE_LIMIT'),
             '_APP_USAGE_STATS' => App::getEnv('_APP_USAGE_STATS'),
+            '_APP_VCS_ENABLED' => $isVcsEnabled,
+            '_APP_DOMAIN_ENABLED' => $isDomainEnabled,
+            '_APP_ASSISTANT_ENABLED' => $isAssistantEnabled
         ]);
 
         $response->dynamic($variables, Response::MODEL_CONSOLE_VARIABLES);
@@ -53,12 +68,12 @@ App::post('/v1/console/assistant')
     ->label('sdk.response.type', Response::CONTENT_TYPE_TEXT)
     ->label('abuse-limit', 15)
     ->label('abuse-key', 'userId:{userId}')
-    ->param('query', '', new Text(2000), 'Query')
+    ->param('prompt', '', new Text(2000), 'Prompt. A string containing questions asked to the AI assistant.')
     ->inject('response')
-    ->action(function (string $query, Response $response) {
+    ->action(function (string $prompt, Response $response) {
         $ch = curl_init('http://appwrite-assistant:3003/');
         $responseHeaders = [];
-        $query = json_encode(['prompt' => $query]);
+        $query = json_encode(['prompt' => $prompt]);
         $headers = ['accept: text/event-stream'];
         $handleEvent = function ($ch, $data) use ($response) {
             $response->chunk($data);

app/controllers/api/databases.php

@@ -377,6 +377,7 @@ App::post('/v1/databases')
     ->label('scope', 'databases.write')
     ->label('audits.event', 'database.create')
     ->label('audits.resource', 'database/{response.$id}')
+    ->label('usage.metric', 'databases.{scope}.requests.create')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'create')
@@ -386,7 +387,7 @@ App::post('/v1/databases')
     ->label('sdk.response.model', Response::MODEL_DATABASE) // Model for database needs to be created
     ->param('databaseId', '', new CustomId(), 'Unique Id. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.')
     ->param('name', '', new Text(128), 'Database name. Max length: 128 chars.')
-    ->param('enabled', true, new Boolean(), 'Is database enabled?', true)
+    ->param('enabled', true, new Boolean(), 'Is the database enabled? When set to \'disabled\', users cannot access the database but Server SDKs with an API key can still read and write to the database. No data is lost when this is toggled.', true)
     ->inject('response')
     ->inject('dbForProject')
     ->inject('events')
@@ -450,6 +451,7 @@ App::get('/v1/databases')
     ->desc('List Databases')
     ->groups(['api', 'database'])
     ->label('scope', 'databases.read')
+    ->label('usage.metric', 'databases.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'list')
@@ -497,6 +499,7 @@ App::get('/v1/databases/:databaseId')
     ->desc('Get Database')
     ->groups(['api', 'database'])
     ->label('scope', 'databases.read')
+    ->label('usage.metric', 'databases.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'get')
@@ -611,6 +614,7 @@ App::put('/v1/databases/:databaseId')
     ->label('event', 'databases.[databaseId].update')
     ->label('audits.event', 'database.update')
     ->label('audits.resource', 'database/{response.$id}')
+    ->label('usage.metric', 'databases.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'update')
@@ -620,7 +624,7 @@ App::put('/v1/databases/:databaseId')
     ->label('sdk.response.model', Response::MODEL_DATABASE)
     ->param('databaseId', '', new UID(), 'Database ID.')
     ->param('name', null, new Text(128), 'Database name. Max length: 128 chars.')
-    ->param('enabled', true, new Boolean(), 'Is database enabled?', true)
+    ->param('enabled', true, new Boolean(), 'Is database enabled? When set to \'disabled\', users cannot access the database but Server SDKs with an API key can still read and write to the database. No data is lost when this is toggled.', true)
     ->inject('response')
     ->inject('dbForProject')
     ->inject('events')
@@ -655,6 +659,7 @@ App::delete('/v1/databases/:databaseId')
     ->label('event', 'databases.[databaseId].delete')
     ->label('audits.event', 'database.delete')
     ->label('audits.resource', 'database/{request.databaseId}')
+    ->label('usage.metric', 'databases.{scope}.requests.delete')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'delete')
@@ -693,13 +698,14 @@ App::delete('/v1/databases/:databaseId')
     });
 
 App::post('/v1/databases/:databaseId/collections')
-    ->alias('/v1/database/collections', ['databaseId' => 'default'])
     ->desc('Create Collection')
     ->groups(['api', 'database'])
     ->label('event', 'databases.[databaseId].collections.[collectionId].create')
     ->label('scope', 'collections.write')
     ->label('audits.event', 'collection.create')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{response.$id}')
+    ->label('usage.metric', 'collections.{scope}.requests.create')
+    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'createCollection')
@@ -712,7 +718,7 @@ App::post('/v1/databases/:databaseId/collections')
     ->param('name', '', new Text(128), 'Collection name. Max length: 128 chars.')
     ->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE), 'An array of permissions strings. By default, no user is granted with any permissions. [Learn more about permissions](/docs/permissions).', true)
     ->param('documentSecurity', false, new Boolean(true), 'Enables configuring permissions for individual documents. A user needs one of document or collection level permissions to access a document. [Learn more about permissions](/docs/permissions).', true)
-    ->param('enabled', true, new Boolean(), 'Is collection enabled?', true)
+    ->param('enabled', true, new Boolean(), 'Is collection enabled? When set to \'disabled\', users cannot access the collection but Server SDKs with and API key can still read and write to the collection. No data is lost when this is toggled.', true)
     ->inject('response')
     ->inject('dbForProject')
     ->inject('mode')
@@ -765,6 +771,8 @@ App::get('/v1/databases/:databaseId/collections')
     ->desc('List Collections')
     ->groups(['api', 'database'])
     ->label('scope', 'collections.read')
+    ->label('usage.metric', 'collections.{scope}.requests.read')
+    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'listCollections')
@@ -822,6 +830,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId')
     ->desc('Get Collection')
     ->groups(['api', 'database'])
     ->label('scope', 'collections.read')
+    ->label('usage.metric', 'collections.{scope}.requests.read')
+    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'getCollection')
@@ -856,6 +866,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/logs')
     ->desc('List Collection Logs')
     ->groups(['api', 'database'])
     ->label('scope', 'collections.read')
+    ->label('usage.metric', 'collections.{scope}.requests.read')
+    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'listCollectionLogs')
@@ -953,6 +965,8 @@ App::put('/v1/databases/:databaseId/collections/:collectionId')
     ->label('event', 'databases.[databaseId].collections.[collectionId].update')
     ->label('audits.event', 'collection.update')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
+    ->label('usage.metric', 'collections.{scope}.requests.update')
+    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'updateCollection')
@@ -965,7 +979,7 @@ App::put('/v1/databases/:databaseId/collections/:collectionId')
     ->param('name', null, new Text(128), 'Collection name. Max length: 128 chars.')
     ->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE), 'An array of permission strings. By default, the current permissions are inherited. [Learn more about permissions](/docs/permissions).', true)
     ->param('documentSecurity', false, new Boolean(true), 'Enables configuring permissions for individual documents. A user needs one of document or collection level permissions to access a document. [Learn more about permissions](/docs/permissions).', true)
-    ->param('enabled', true, new Boolean(), 'Is collection enabled?', true)
+    ->param('enabled', true, new Boolean(), 'Is collection enabled? When set to \'disabled\', users cannot access the collection but Server SDKs with and API key can still read and write to the collection. No data is lost when this is toggled.', true)
     ->inject('response')
     ->inject('dbForProject')
     ->inject('mode')
@@ -1021,6 +1035,8 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId')
     ->label('event', 'databases.[databaseId].collections.[collectionId].delete')
     ->label('audits.event', 'collection.delete')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
+    ->label('usage.metric', 'collections.{scope}.requests.delete')
+    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'deleteCollection')
@@ -1075,6 +1091,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/string
     ->label('scope', 'collections.write')
     ->label('audits.event', 'attribute.create')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
+    ->label('usage.metric', 'collections.{scope}.requests.update')
+    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'createStringAttribute')
@@ -1131,6 +1149,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/email'
     ->label('scope', 'collections.write')
     ->label('audits.event', 'attribute.create')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
+    ->label('usage.metric', 'collections.{scope}.requests.update')
+    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.method', 'createEmailAttribute')
@@ -1173,6 +1193,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/enum')
     ->label('scope', 'collections.write')
     ->label('audits.event', 'attribute.create')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
+    ->label('usage.metric', 'collections.{scope}.requests.update')
+    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.method', 'createEnumAttribute')
@@ -1231,6 +1253,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/ip')
     ->label('scope', 'collections.write')
     ->label('audits.event', 'attribute.create')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
+    ->label('usage.metric', 'collections.{scope}.requests.update')
+    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.method', 'createIpAttribute')
@@ -1273,6 +1297,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/url')
     ->label('scope', 'collections.write')
     ->label('audits.event', 'attribute.create')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
+    ->label('usage.metric', 'collections.{scope}.requests.update')
+    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.method', 'createUrlAttribute')
@@ -1315,6 +1341,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/intege
     ->label('scope', 'collections.write')
     ->label('audits.event', 'attribute.create')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
+    ->label('usage.metric', 'collections.{scope}.requests.update')
+    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.method', 'createIntegerAttribute')
@@ -1386,6 +1414,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/float'
     ->label('scope', 'collections.write')
     ->label('audits.event', 'attribute.create')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
+    ->label('usage.metric', 'collections.{scope}.requests.update')
+    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.method', 'createFloatAttribute')
@@ -1460,6 +1490,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/boolea
     ->label('scope', 'collections.write')
     ->label('audits.event', 'attribute.create')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
+    ->label('usage.metric', 'collections.{scope}.requests.update')
+    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.method', 'createBooleanAttribute')
@@ -1501,6 +1533,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/dateti
     ->label('scope', 'collections.write')
     ->label('audits.event', 'attribute.create')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
+    ->label('usage.metric', 'collections.{scope}.requests.update')
+    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.method', 'createDatetimeAttribute')
@@ -1545,6 +1579,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/attributes/relati
     ->label('scope', 'collections.write')
     ->label('audits.event', 'attribute.create')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
+    ->label('usage.metric', 'collections.{scope}.requests.update')
+    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.method', 'createRelationshipAttribute')
@@ -1622,6 +1658,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/attributes')
     ->desc('List Attributes')
     ->groups(['api', 'database'])
     ->label('scope', 'collections.read')
+    ->label('usage.metric', 'collections.{scope}.requests.read')
+    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'listAttributes')
@@ -1695,6 +1733,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/attributes/:key')
     ->desc('Get Attribute')
     ->groups(['api', 'database'])
     ->label('scope', 'collections.read')
+    ->label('usage.metric', 'collections.{scope}.requests.read')
+    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'getAttribute')
@@ -1772,6 +1812,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/strin
     ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
     ->label('audits.event', 'attribute.update')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
+    ->label('usage.metric', 'collections.{scope}.requests.update')
+    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'updateStringAttribute')
@@ -1811,6 +1853,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/email
     ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
     ->label('audits.event', 'attribute.update')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
+    ->label('usage.metric', 'collections.{scope}.requests.update')
+    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'updateEmailAttribute')
@@ -1850,6 +1894,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/enum/
     ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
     ->label('audits.event', 'attribute.update')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
+    ->label('usage.metric', 'collections.{scope}.requests.update')
+    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'updateEnumAttribute')
@@ -1891,6 +1937,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/ip/:k
     ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
     ->label('audits.event', 'attribute.update')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
+    ->label('usage.metric', 'collections.{scope}.requests.update')
+    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'updateIpAttribute')
@@ -1930,6 +1978,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/url/:
     ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
     ->label('audits.event', 'attribute.update')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
+    ->label('usage.metric', 'collections.{scope}.requests.update')
+    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'updateUrlAttribute')
@@ -1969,6 +2019,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/integ
     ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
     ->label('audits.event', 'attribute.update')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
+    ->label('usage.metric', 'collections.{scope}.requests.update')
+    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'updateIntegerAttribute')
@@ -2018,6 +2070,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/float
     ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
     ->label('audits.event', 'attribute.update')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
+    ->label('usage.metric', 'collections.{scope}.requests.update')
+    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'updateFloatAttribute')
@@ -2067,6 +2121,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/boole
     ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
     ->label('audits.event', 'attribute.update')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
+    ->label('usage.metric', 'collections.{scope}.requests.update')
+    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'updateBooleanAttribute')
@@ -2105,6 +2161,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/datet
     ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
     ->label('audits.event', 'attribute.update')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
+    ->label('usage.metric', 'documents.{scope}.requests.update')
+    ->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'updateDatetimeAttribute')
@@ -2143,6 +2201,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/attributes/:key/
     ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].update')
     ->label('audits.event', 'attribute.update')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
+    ->label('usage.metric', 'documents.{scope}.requests.update')
+    ->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'updateRelationshipAttribute')
@@ -2197,6 +2257,8 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/attributes/:key
     ->label('event', 'databases.[databaseId].collections.[collectionId].attributes.[attributeId].delete')
     ->label('audits.event', 'attribute.delete')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
+    ->label('usage.metric', 'collections.{scope}.requests.update')
+    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'deleteAttribute')
@@ -2306,6 +2368,8 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/indexes')
     ->label('scope', 'collections.write')
     ->label('audits.event', 'index.create')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
+    ->label('usage.metric', 'collections.{scope}.requests.update')
+    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'createIndex')
@@ -2461,6 +2525,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/indexes')
     ->desc('List Indexes')
     ->groups(['api', 'database'])
     ->label('scope', 'collections.read')
+    ->label('usage.metric', 'collections.{scope}.requests.read')
+    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'listIndexes')
@@ -2524,6 +2590,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/indexes/:key')
     ->desc('Get Index')
     ->groups(['api', 'database'])
     ->label('scope', 'collections.read')
+    ->label('usage.metric', 'collections.{scope}.requests.read')
+    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'getIndex')
@@ -2566,6 +2634,8 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/indexes/:key')
     ->label('event', 'databases.[databaseId].collections.[collectionId].indexes.[indexId].delete')
     ->label('audits.event', 'index.delete')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
+    ->label('usage.metric', 'collections.{scope}.requests.update')
+    ->label('usage.params', ['databaseId:{request.databaseId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'deleteIndex')
@@ -2629,7 +2699,9 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents')
     ->label('event', 'databases.[databaseId].collections.[collectionId].documents.[documentId].create')
     ->label('scope', 'documents.write')
     ->label('audits.event', 'document.create')
-    ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}/document/{response.$id}')
+    ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}')
+    ->label('usage.metric', 'documents.{scope}.requests.create')
+    ->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
     ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
     ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2)
     ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
@@ -2865,6 +2937,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents')
     ->desc('List Documents')
     ->groups(['api', 'database'])
     ->label('scope', 'documents.read')
+    ->label('usage.metric', 'documents.{scope}.requests.read')
+    ->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'listDocuments')
@@ -2990,6 +3064,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen
     ->desc('Get Document')
     ->groups(['api', 'database'])
     ->label('scope', 'documents.read')
+    ->label('usage.metric', 'documents.{scope}.requests.read')
+    ->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'getDocument')
@@ -3083,6 +3159,8 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/documents/:documen
     ->desc('List Document Logs')
     ->groups(['api', 'database'])
     ->label('scope', 'documents.read')
+    ->label('usage.metric', 'documents.{scope}.requests.read')
+    ->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
     ->label('sdk.namespace', 'databases')
     ->label('sdk.method', 'listDocumentLogs')
@@ -3185,6 +3263,8 @@ App::patch('/v1/databases/:databaseId/collections/:collectionId/documents/:docum
     ->label('scope', 'documents.write')
     ->label('audits.event', 'document.update')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}/document/{response.$id}')
+    ->label('usage.metric', 'documents.{scope}.requests.update')
+    ->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
     ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
     ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT * 2)
     ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
@@ -3412,6 +3492,8 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu
     ->label('event', 'databases.[databaseId].collections.[collectionId].documents.[documentId].delete')
     ->label('audits.event', 'document.delete')
     ->label('audits.resource', 'database/{request.databaseId}/collection/{request.collectionId}/document/{request.documentId}')
+    ->label('usage.metric', 'documents.{scope}.requests.delete')
+    ->label('usage.params', ['databaseId:{request.databaseId}', 'collectionId:{request.collectionId}'])
     ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
     ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT)
     ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
@@ -3520,7 +3602,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu
 
 App::get('/v1/databases/usage')
     ->desc('Get usage stats for the database')
-    ->groups(['api', 'database', 'usage'])
+    ->groups(['api', 'database'])
     ->label('scope', 'collections.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
     ->label('sdk.namespace', 'databases')
@@ -3533,62 +3615,112 @@ App::get('/v1/databases/usage')
     ->inject('dbForProject')
     ->action(function (string $range, Response $response, Database $dbForProject) {
 
-        $periods = Config::getParam('usage', []);
-        $stats = $usage = [];
-        $days = $periods[$range];
-        $metrics = [
-            METRIC_DATABASES,
-            METRIC_COLLECTIONS,
-            METRIC_DOCUMENTS,
-        ];
-
-        Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
-            foreach ($metrics as $metric) {
-                $limit = $days['limit'];
-                $period = $days['period'];
-                $results = $dbForProject->find('stats', [
-                    Query::equal('period', [$period]),
-                    Query::equal('metric', [$metric]),
-                    Query::limit($limit),
-                    Query::orderDesc('time'),
-                ]);
-                $stats[$metric] = [];
-                foreach ($results as $result) {
-                    $stats[$metric][$result->getAttribute('time')] = [
-                        'value' => $result->getAttribute('value'),
-                    ];
-                }
-            }
-        });
-
-        $format = match ($days['period']) {
-            '1h' => 'Y-m-d\TH:00:00.000P',
-            '1d' => 'Y-m-d\T00:00:00.000P',
-        };
-
-    foreach ($metrics as $metric) {
-        $usage[$metric] = [];
-        $leap = time() - ($days['limit'] * $days['factor']);
-        while ($leap < time()) {
-            $leap += $days['factor'];
-            $formatDate = date($format, $leap);
-            $usage[$metric][] = [
-                'value' => $stats[$metric][$formatDate]['value'] ?? 0,
-                'date' => $formatDate,
+        $usage = [];
+        if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
+            $periods = [
+                '24h' => [
+                    'period' => '1h',
+                    'limit' => 24,
+                ],
+                '7d' => [
+                    'period' => '1d',
+                    'limit' => 7,
+                ],
+                '30d' => [
+                    'period' => '1d',
+                    'limit' => 30,
+                ],
+                '90d' => [
+                    'period' => '1d',
+                    'limit' => 90,
+                ],
             ];
+
+            $metrics = [
+                'databases.$all.count.total',
+                'documents.$all.count.total',
+                'collections.$all.count.total',
+                'databases.$all.requests.create',
+                'databases.$all.requests.read',
+                'databases.$all.requests.update',
+                'databases.$all.requests.delete',
+                'collections.$all.requests.create',
+                'collections.$all.requests.read',
+                'collections.$all.requests.update',
+                'collections.$all.requests.delete',
+                'documents.$all.requests.create',
+                'documents.$all.requests.read',
+                'documents.$all.requests.update',
+                'documents.$all.requests.delete'
+            ];
+
+            $stats = [];
+
+            Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
+                foreach ($metrics as $metric) {
+                    $limit = $periods[$range]['limit'];
+                    $period = $periods[$range]['period'];
+
+                    $requestDocs = $dbForProject->find('stats', [
+                        Query::equal('period', [$period]),
+                        Query::equal('metric', [$metric]),
+                        Query::limit($limit),
+                        Query::orderDesc('time'),
+                    ]);
+
+                    $stats[$metric] = [];
+                    foreach ($requestDocs as $requestDoc) {
+                        $stats[$metric][] = [
+                            'value' => $requestDoc->getAttribute('value'),
+                            'date' => $requestDoc->getAttribute('time'),
+                        ];
+                    }
+
+                    // backfill metrics with empty values for graphs
+                    $backfill = $limit - \count($requestDocs);
+                    while ($backfill > 0) {
+                        $last = $limit - $backfill - 1; // array index of last added metric
+                        $diff = match ($period) { // convert period to seconds for unix timestamp math
+                            '1h' => 3600,
+                            '1d' => 86400,
+                        };
+                        $stats[$metric][] = [
+                            'value' => 0,
+                            'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
+                        ];
+                        $backfill--;
+                    }
+                    // Added 3'rd level to Index [period, metric, time] because of order by.
+                    $stats[$metric] = array_reverse($stats[$metric]);
+                }
+            });
+
+            $usage = new Document([
+                'range' => $range,
+                'databasesCount' => $stats['databases.$all.count.total'] ?? [],
+                'documentsCount' => $stats['documents.$all.count.total'] ?? [],
+                'collectionsCount' => $stats['collections.$all.count.total'] ?? [],
+                'documentsCreate' =>  $stats['documents.$all.requests.create'] ?? [],
+                'documentsRead' =>  $stats['documents.$all.requests.read'] ?? [],
+                'documentsUpdate' => $stats['documents.$all.requests.update'] ?? [],
+                'documentsDelete' => $stats['documents.$all.requests.delete'] ?? [],
+                'collectionsCreate' => $stats['collections.$all.requests.create'] ?? [],
+                'collectionsRead' =>  $stats['collections.$all.requests.read'] ?? [],
+                'collectionsUpdate' => $stats['collections.$all.requests.update'] ?? [],
+                'collectionsDelete' => $stats['collections.$all.requests.delete'] ?? [],
+                'databasesCreate' => $stats['databases.$all.requests.create'] ?? [],
+                'databasesRead' =>  $stats['databases.$all.requests.read'] ?? [],
+                'databasesUpdate' => $stats['databases.$all.requests.update'] ?? [],
+                'databasesDelete' => $stats['databases.$all.requests.delete'] ?? [],
+            ]);
         }
-    }
-        $response->dynamic(new Document([
-            'range' => $range,
-            'databasesTotal'   => $usage[$metrics[0]],
-            'collectionsTotal' => $usage[$metrics[1]],
-            'documentsTotal'   => $usage[$metrics[2]],
-        ]), Response::MODEL_USAGE_DATABASES);
+
+        $response->dynamic($usage, Response::MODEL_USAGE_DATABASES);
     });
 
 App::get('/v1/databases/:databaseId/usage')
     ->desc('Get usage stats for the database')
-    ->groups(['api', 'database', 'usage'])
+    ->groups(['api', 'database'])
     ->label('scope', 'collections.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
     ->label('sdk.namespace', 'databases')
@@ -3602,68 +3734,103 @@ App::get('/v1/databases/:databaseId/usage')
     ->inject('dbForProject')
     ->action(function (string $databaseId, string $range, Response $response, Database $dbForProject) {
 
-        $database =  $dbForProject->getDocument('databases', $databaseId);
-
-        if ($database->isEmpty()) {
-            throw new Exception(Exception::DATABASE_NOT_FOUND);
-        }
-
-        $periods = Config::getParam('usage', []);
-        $stats = $usage = [];
-        $days = $periods[$range];
-        $metrics = [
-            str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS),
-            str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS),
-        ];
-
-        Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
-            foreach ($metrics as $metric) {
-                $limit = $days['limit'];
-                $period = $days['period'];
-                $results = $dbForProject->find('stats', [
-                    Query::equal('period', [$period]),
-                    Query::equal('metric', [$metric]),
-                    Query::limit($limit),
-                    Query::orderDesc('time'),
-                ]);
-                $stats[$metric] = [];
-                foreach ($results as $result) {
-                    $stats[$metric][$result->getAttribute('time')] = [
-                        'value' => $result->getAttribute('value'),
-                    ];
-                }
-            }
-        });
-
-        $format = match ($days['period']) {
-            '1h' => 'Y-m-d\TH:00:00.000P',
-            '1d' => 'Y-m-d\T00:00:00.000P',
-        };
-
-    foreach ($metrics as $metric) {
-        $usage[$metric] = [];
-        $leap = time() - ($days['limit'] * $days['factor']);
-        while ($leap < time()) {
-            $leap += $days['factor'];
-            $formatDate = date($format, $leap);
-            $usage[$metric][] = [
-                'value' => $stats[$metric][$formatDate]['value'] ?? 0,
-                'date' => $formatDate,
+        $usage = [];
+        if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
+            $periods = [
+                '24h' => [
+                    'period' => '1h',
+                    'limit' => 24,
+                ],
+                '7d' => [
+                    'period' => '1d',
+                    'limit' => 7,
+                ],
+                '30d' => [
+                    'period' => '1d',
+                    'limit' => 30,
+                ],
+                '90d' => [
+                    'period' => '1d',
+                    'limit' => 90,
+                ],
             ];
-        }
-    }
 
-        $response->dynamic(new Document([
-            'range' => $range,
-            'collectionsTotal'   => $usage[$metrics[0]],
-            'documentsTotal'   => $usage[$metrics[1]],
-        ]), Response::MODEL_USAGE_DATABASE);
+            $metrics = [
+                'collections.' . $databaseId . '.count.total',
+                'collections.' . $databaseId . '.requests.create',
+                'collections.' . $databaseId . '.requests.read',
+                'collections.' . $databaseId . '.requests.update',
+                'collections.' . $databaseId . '.requests.delete',
+                'documents.' . $databaseId . '.count.total',
+                'documents.' . $databaseId . '.requests.create',
+                'documents.' . $databaseId . '.requests.read',
+                'documents.' . $databaseId . '.requests.update',
+                'documents.' . $databaseId . '.requests.delete'
+            ];
+
+            $stats = [];
+
+            Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
+                foreach ($metrics as $metric) {
+                    $limit = $periods[$range]['limit'];
+                    $period = $periods[$range]['period'];
+
+                    $requestDocs = $dbForProject->find('stats', [
+                        Query::equal('period', [$period]),
+                        Query::equal('metric', [$metric]),
+                        Query::limit($limit),
+                        Query::orderDesc('time'),
+                    ]);
+
+                    $stats[$metric] = [];
+                    foreach ($requestDocs as $requestDoc) {
+                        $stats[$metric][] = [
+                            'value' => $requestDoc->getAttribute('value'),
+                            'date' => $requestDoc->getAttribute('time'),
+                        ];
+                    }
+
+                    // backfill metrics with empty values for graphs
+                    $backfill = $limit - \count($requestDocs);
+                    while ($backfill > 0) {
+                        $last = $limit - $backfill - 1; // array index of last added metric
+                        $diff = match ($period) { // convert period to seconds for unix timestamp math
+                            '1h' => 3600,
+                            '1d' => 86400,
+                        };
+                        $stats[$metric][] = [
+                            'value' => 0,
+                            'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
+                        ];
+                        $backfill--;
+                    }
+                    // TODO@kodumbeats explore performance if query is ordered by time ASC
+                    $stats[$metric] = array_reverse($stats[$metric]);
+                }
+            });
+
+            $usage = new Document([
+                'range' => $range,
+                'collectionsCount' => $stats["collections.{$databaseId}.count.total"] ?? [],
+                'collectionsCreate' => $stats["collections.{$databaseId}.requests.create"] ?? [],
+                'collectionsRead' =>  $stats["collections.{$databaseId}.requests.read"] ?? [],
+                'collectionsUpdate' => $stats["collections.{$databaseId}.requests.update"] ?? [],
+                'collectionsDelete' => $stats["collections.{$databaseId}.requests.delete"] ?? [],
+                'documentsCount' => $stats["documents.{$databaseId}.count.total"] ?? [],
+                'documentsCreate' =>  $stats["documents.{$databaseId}.requests.create"] ?? [],
+                'documentsRead' =>  $stats["documents.{$databaseId}.requests.read"] ?? [],
+                'documentsUpdate' => $stats["documents.{$databaseId}.requests.update"] ?? [],
+                'documentsDelete' => $stats["documents.{$databaseId}.requests.delete"] ?? [],
+            ]);
+        }
+
+        $response->dynamic($usage, Response::MODEL_USAGE_DATABASE);
     });
 
 App::get('/v1/databases/:databaseId/collections/:collectionId/usage')
     ->alias('/v1/database/:collectionId/usage', ['databaseId' => 'default'])
     ->desc('Get usage stats for a collection')
-    ->groups(['api', 'database', 'usage'])
+    ->groups(['api', 'database'])
     ->label('scope', 'collections.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
     ->label('sdk.namespace', 'databases')
@@ -3686,52 +3853,84 @@ App::get('/v1/databases/:databaseId/collections/:collectionId/usage')
             throw new Exception(Exception::COLLECTION_NOT_FOUND);
         }
 
-        $periods = Config::getParam('usage', []);
-        $stats = $usage = [];
-        $days = $periods[$range];
-        $metrics = [
-            str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collectionDocument->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS),
-        ];
-
-        Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
-            foreach ($metrics as $metric) {
-                $limit = $days['limit'];
-                $period = $days['period'];
-                $results = $dbForProject->find('stats', [
-                    Query::equal('period', [$period]),
-                    Query::equal('metric', [$metric]),
-                    Query::limit($limit),
-                    Query::orderDesc('time'),
-                ]);
-                $stats[$metric] = [];
-                foreach ($results as $result) {
-                    $stats[$metric][$result->getAttribute('time')] = [
-                        'value' => $result->getAttribute('value'),
-                    ];
-                }
-            }
-        });
-
-        $format = match ($days['period']) {
-            '1h' => 'Y-m-d\TH:00:00.000P',
-            '1d' => 'Y-m-d\T00:00:00.000P',
-        };
-
-    foreach ($metrics as $metric) {
-        $usage[$metric] = [];
-        $leap = time() - ($days['limit'] * $days['factor']);
-        while ($leap < time()) {
-            $leap += $days['factor'];
-            $formatDate = date($format, $leap);
-            $usage[$metric][] = [
-                'value' => $stats[$metric][$formatDate]['value'] ?? 0,
-                'date' => $formatDate,
+        $usage = [];
+        if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
+            $periods = [
+                '24h' => [
+                    'period' => '1h',
+                    'limit' => 24,
+                ],
+                '7d' => [
+                    'period' => '1d',
+                    'limit' => 7,
+                ],
+                '30d' => [
+                    'period' => '1d',
+                    'limit' => 30,
+                ],
+                '90d' => [
+                    'period' => '1d',
+                    'limit' => 90,
+                ],
             ];
-        }
-    }
 
-        $response->dynamic(new Document([
-            'range' => $range,
-            'documentsTotal'   => $usage[$metrics[0]],
-        ]), Response::MODEL_USAGE_COLLECTION);
+            $metrics = [
+                "documents.{$databaseId}/{$collectionId}.count.total",
+                "documents.{$databaseId}/{$collectionId}.requests.create",
+                "documents.{$databaseId}/{$collectionId}.requests.read",
+                "documents.{$databaseId}/{$collectionId}.requests.update",
+                "documents.{$databaseId}/{$collectionId}.requests.delete",
+            ];
+
+            $stats = [];
+
+            Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
+                foreach ($metrics as $metric) {
+                    $limit = $periods[$range]['limit'];
+                    $period = $periods[$range]['period'];
+
+                    $requestDocs = $dbForProject->find('stats', [
+                        Query::equal('period', [$period]),
+                        Query::equal('metric', [$metric]),
+                        Query::limit($limit),
+                        Query::orderDesc('time'),
+                    ]);
+
+                    $stats[$metric] = [];
+                    foreach ($requestDocs as $requestDoc) {
+                        $stats[$metric][] = [
+                            'value' => $requestDoc->getAttribute('value'),
+                            'date' => $requestDoc->getAttribute('time'),
+                        ];
+                    }
+
+                    // backfill metrics with empty values for graphs
+                    $backfill = $limit - \count($requestDocs);
+                    while ($backfill > 0) {
+                        $last = $limit - $backfill - 1; // array index of last added metric
+                        $diff = match ($period) { // convert period to seconds for unix timestamp math
+                            '1h' => 3600,
+                            '1d' => 86400,
+                        };
+                        $stats[$metric][] = [
+                            'value' => 0,
+                            'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
+                        ];
+                        $backfill--;
+                    }
+                    $stats[$metric] = array_reverse($stats[$metric]);
+                }
+            });
+
+            $usage = new Document([
+                'range' => $range,
+                'documentsCount' => $stats["documents.{$databaseId}/{$collectionId}.count.total"] ?? [],
+                'documentsCreate' => $stats["documents.{$databaseId}/{$collectionId}.requests.create"] ?? [],
+                'documentsRead' => $stats["documents.{$databaseId}/{$collectionId}.requests.read"] ?? [],
+                'documentsUpdate' =>  $stats["documents.{$databaseId}/{$collectionId}.requests.update"] ?? [],
+                'documentsDelete' =>  $stats["documents.{$databaseId}/{$collectionId}.requests.delete" ?? []]
+            ]);
+        }
+
+        $response->dynamic($usage, Response::MODEL_USAGE_COLLECTION);
     });

app/controllers/api/migrations.php

@@ -34,7 +34,7 @@ App::post('/v1/migrations/appwrite')
     ->groups(['api', 'migrations'])
     ->desc('Migrate Appwrite Data')
     ->label('scope', 'migrations.write')
-    ->label('event', 'migrations.create')
+    ->label('event', 'migrations.[migrationId].create')
     ->label('audits.event', 'migration.create')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'migrations')
@@ -88,7 +88,7 @@ App::post('/v1/migrations/firebase/oauth')
     ->groups(['api', 'migrations'])
     ->desc('Migrate Firebase Data (OAuth)')
     ->label('scope', 'migrations.write')
-    ->label('event', 'migrations.create')
+    ->label('event', 'migrations.[migrationId].create')
     ->label('audits.event', 'migration.create')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'migrations')
@@ -146,12 +146,13 @@ App::post('/v1/migrations/firebase/oauth')
             $dbForConsole->updateDocument('identities', $identity->getId(), $identity);
         }
 
-        if ($identity->getAttribute('secret')) {
-            $serviceAccount = $identity->getAttribute('secret');
+        if ($identity->getAttribute('secrets')) {
+            $serviceAccount = $identity->getAttribute('secrets');
         } else {
+            $firebase->cleanupServiceAccounts($accessToken, $projectId);
             $serviceAccount = $firebase->createServiceAccount($accessToken, $projectId);
             $identity = $identity
-                ->setAttribute('secret', $serviceAccount);
+                ->setAttribute('secrets', json_encode($serviceAccount));
 
             $dbForConsole->updateDocument('identities', $identity->getId(), $identity);
         }
@@ -189,7 +190,7 @@ App::post('/v1/migrations/firebase')
     ->groups(['api', 'migrations'])
     ->desc('Migrate Firebase Data (Service Account)')
     ->label('scope', 'migrations.write')
-    ->label('event', 'migrations.create')
+    ->label('event', 'migrations.[migrationId].create')
     ->label('audits.event', 'migration.create')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'migrations')
@@ -239,7 +240,7 @@ App::post('/v1/migrations/supabase')
     ->groups(['api', 'migrations'])
     ->desc('Migrate Supabase Data')
     ->label('scope', 'migrations.write')
-    ->label('event', 'migrations.create')
+    ->label('event', 'migrations.[migrationId].create')
     ->label('audits.event', 'migration.create')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'migrations')
@@ -299,7 +300,7 @@ App::post('/v1/migrations/nhost')
     ->groups(['api', 'migrations'])
     ->desc('Migrate NHost Data')
     ->label('scope', 'migrations.write')
-    ->label('event', 'migrations.create')
+    ->label('event', 'migrations.[migrationId].create')
     ->label('audits.event', 'migration.create')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'migrations')
@@ -309,7 +310,7 @@ App::post('/v1/migrations/nhost')
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_MIGRATION)
     ->param('resources', [], new ArrayList(new WhiteList(NHost::getSupportedResources())), 'List of resources to migrate')
-    ->param('subdomain', '', new URL(), 'Source\'s Subdomain')
+    ->param('subdomain', '', new Text(512), 'Source\'s Subdomain')
     ->param('region', '', new Text(512), 'Source\'s Region')
     ->param('adminSecret', '', new Text(512), 'Source\'s Admin Secret')
     ->param('database', '', new Text(512), 'Source\'s Database Name')
@@ -454,8 +455,8 @@ App::get('/v1/migrations/appwrite/report')
             $response
                 ->setStatusCode(Response::STATUS_CODE_OK)
                 ->dynamic(new Document($appwrite->report($resources)), Response::MODEL_MIGRATION_REPORT);
-        } catch (\Exception $e) {
-            throw new Exception(Exception::GENERAL_SERVER_ERROR, $e->getMessage());
+        } catch (\Throwable $e) {
+            throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Source Error: ' . $e->getMessage());
         }
     });
 
@@ -481,7 +482,7 @@ App::get('/v1/migrations/firebase/report')
                 ->setStatusCode(Response::STATUS_CODE_OK)
                 ->dynamic(new Document($firebase->report($resources)), Response::MODEL_MIGRATION_REPORT);
         } catch (\Exception $e) {
-            throw new Exception(Exception::GENERAL_SERVER_ERROR, $e->getMessage());
+            throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Source Error: ' . $e->getMessage());
         }
     });
 
@@ -503,67 +504,77 @@ App::get('/v1/migrations/firebase/report/oauth')
     ->inject('user')
     ->inject('dbForConsole')
     ->action(function (array $resources, string $projectId, Response $response, Request $request, Document $user, Database $dbForConsole) {
-        try {
-            $firebase = new OAuth2Firebase(
-                App::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_ID', ''),
-                App::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_SECRET', ''),
-                $request->getProtocol() . '://' . $request->getHostname() . '/v1/migrations/firebase/redirect'
-            );
+        $firebase = new OAuth2Firebase(
+            App::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_ID', ''),
+            App::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_SECRET', ''),
+            $request->getProtocol() . '://' . $request->getHostname() . '/v1/migrations/firebase/redirect'
+        );
 
-            $identity = $dbForConsole->findOne('identities', [
-                Query::equal('provider', ['firebase']),
-                Query::equal('userInternalId', [$user->getInternalId()]),
-            ]);
-            if ($identity === false || $identity->isEmpty()) {
-                throw new Exception(Exception::USER_IDENTITY_NOT_FOUND);
-            }
+        $identity = $dbForConsole->findOne('identities', [
+            Query::equal('provider', ['firebase']),
+            Query::equal('userInternalId', [$user->getInternalId()]),
+        ]);
 
-            $accessToken = $identity->getAttribute('providerAccessToken');
-            $refreshToken = $identity->getAttribute('providerRefreshToken');
-            $accessTokenExpiry = $identity->getAttribute('providerAccessTokenExpiry');
-
-            $isExpired = new \DateTime($accessTokenExpiry) < new \DateTime('now');
-            if ($isExpired) {
-                $firebase->refreshTokens($refreshToken);
-
-                $accessToken = $firebase->getAccessToken('');
-                $refreshToken = $firebase->getRefreshToken('');
-
-                $verificationId = $firebase->getUserID($accessToken);
-
-                if (empty($verificationId)) {
-                    throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'Another request is currently refreshing OAuth token. Please try again.');
-                }
-
-                $identity = $identity
-                    ->setAttribute('providerAccessToken', $accessToken)
-                    ->setAttribute('providerRefreshToken', $refreshToken)
-                    ->setAttribute('providerAccessTokenExpiry', DateTime::addSeconds(new \DateTime(), (int)$firebase->getAccessTokenExpiry('')));
-
-                $dbForConsole->updateDocument('identities', $identity->getId(), $identity);
-            }
-
-            // Get Service Account
-            if ($identity->getAttribute('secret')) {
-                $serviceAccount = $identity->getAttribute('secret');
-            } else {
-                $serviceAccount = $firebase->createServiceAccount($accessToken, $projectId);
-                $identity = $identity
-                    ->setAttribute('secret', $serviceAccount);
-
-                $dbForConsole->updateDocument('identities', $identity->getId(), $identity);
-            }
-
-            $firebase = new Firebase(array_merge($serviceAccount, ['project_id' => $projectId]));
-
-            $report = $firebase->report($resources);
-
-            $response
-                ->setStatusCode(Response::STATUS_CODE_OK)
-                ->dynamic(new Document($report), Response::MODEL_MIGRATION_REPORT);
-        } catch (\Exception $e) {
-            throw new Exception(Exception::GENERAL_SERVER_ERROR, $e->getMessage());
+        if ($identity === false || $identity->isEmpty()) {
+            throw new Exception(Exception::USER_IDENTITY_NOT_FOUND);
         }
+
+        $accessToken = $identity->getAttribute('providerAccessToken');
+        $refreshToken = $identity->getAttribute('providerRefreshToken');
+        $accessTokenExpiry = $identity->getAttribute('providerAccessTokenExpiry');
+
+        if (empty($accessToken) || empty($refreshToken) || empty($accessTokenExpiry)) {
+            throw new Exception(Exception::USER_IDENTITY_NOT_FOUND);
+        }
+
+        if (App::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_ID', '') === '' || App::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_SECRET', '') === '') {
+            throw new Exception(Exception::USER_IDENTITY_NOT_FOUND);
+        }
+
+        $isExpired = new \DateTime($accessTokenExpiry) < new \DateTime('now');
+        if ($isExpired) {
+            $firebase->refreshTokens($refreshToken);
+
+            $accessToken = $firebase->getAccessToken('');
+            $refreshToken = $firebase->getRefreshToken('');
+
+            $verificationId = $firebase->getUserID($accessToken);
+
+            if (empty($verificationId)) {
+                throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'Another request is currently refreshing OAuth token. Please try again.');
+            }
+
+            $identity = $identity
+                ->setAttribute('providerAccessToken', $accessToken)
+                ->setAttribute('providerRefreshToken', $refreshToken)
+                ->setAttribute('providerAccessTokenExpiry', DateTime::addSeconds(new \DateTime(), (int)$firebase->getAccessTokenExpiry('')));
+
+            $dbForConsole->updateDocument('identities', $identity->getId(), $identity);
+        }
+
+        // Get Service Account
+        if ($identity->getAttribute('secrets')) {
+            $serviceAccount = $identity->getAttribute('secrets');
+        } else {
+            $firebase->cleanupServiceAccounts($accessToken, $projectId);
+            $serviceAccount = $firebase->createServiceAccount($accessToken, $projectId);
+            $identity = $identity
+                ->setAttribute('secrets', json_encode($serviceAccount));
+
+            $dbForConsole->updateDocument('identities', $identity->getId(), $identity);
+        }
+
+        $firebase = new Firebase($serviceAccount);
+
+        try {
+            $report = $firebase->report($resources);
+        } catch (\Exception $e) {
+            throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Source Error: ' . $e->getMessage());
+        }
+
+        $response
+            ->setStatusCode(Response::STATUS_CODE_OK)
+            ->dynamic(new Document($report), Response::MODEL_MIGRATION_REPORT);
     });
 
 App::get('/v1/migrations/firebase/connect')
@@ -614,7 +625,7 @@ App::get('/v1/migrations/firebase/redirect')
     ->groups(['api', 'migrations'])
     ->label('scope', 'public')
     ->label('error', __DIR__ . '/../../views/general/error.phtml')
-    ->param('code', '', new Text(2048), 'OAuth2 code.', true)
+    ->param('code', '', new Text(2048), 'OAuth2 code. This is a temporary code that the will be later exchanged for an access token.', true)
     ->inject('user')
     ->inject('project')
     ->inject('request')
@@ -747,6 +758,7 @@ App::get('/v1/migrations/firebase/projects')
             Query::equal('provider', ['firebase']),
             Query::equal('userInternalId', [$user->getInternalId()]),
         ]);
+
         if ($identity === false || $identity->isEmpty()) {
             throw new Exception(Exception::USER_IDENTITY_NOT_FOUND);
         }
@@ -756,46 +768,50 @@ App::get('/v1/migrations/firebase/projects')
         $accessTokenExpiry = $identity->getAttribute('providerAccessTokenExpiry');
 
         if (empty($accessToken) || empty($refreshToken) || empty($accessTokenExpiry)) {
-            throw new Exception(Exception::GENERAL_ACCESS_FORBIDDEN, 'Not authenticated with Firebase');
+            throw new Exception(Exception::USER_IDENTITY_NOT_FOUND);
         }
 
         if (App::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_ID', '') === '' || App::getEnv('_APP_MIGRATIONS_FIREBASE_CLIENT_SECRET', '') === '') {
-            throw new Exception(Exception::GENERAL_ACCESS_FORBIDDEN, 'Missing Google OAuth credentials');
+            throw new Exception(Exception::USER_IDENTITY_NOT_FOUND);
         }
 
-        $isExpired = new \DateTime($accessTokenExpiry) < new \DateTime('now');
-        if ($isExpired) {
-            try {
-                $firebase->refreshTokens($refreshToken);
-            } catch (\Exception $e) {
-                throw new Exception(Exception::GENERAL_ACCESS_FORBIDDEN, 'Failed to refresh Firebase access token');
+        try {
+            $isExpired = new \DateTime($accessTokenExpiry) < new \DateTime('now');
+            if ($isExpired) {
+                try {
+                    $firebase->refreshTokens($refreshToken);
+                } catch (\Exception $e) {
+                    throw new Exception(Exception::USER_IDENTITY_NOT_FOUND);
+                }
+
+                $accessToken = $firebase->getAccessToken('');
+                $refreshToken = $firebase->getRefreshToken('');
+
+                $verificationId = $firebase->getUserID($accessToken);
+
+                if (empty($verificationId)) {
+                    throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'Another request is currently refreshing OAuth token. Please try again.');
+                }
+
+                $identity = $identity
+                    ->setAttribute('providerAccessToken', $accessToken)
+                    ->setAttribute('providerRefreshToken', $refreshToken)
+                    ->setAttribute('providerAccessTokenExpiry', DateTime::addSeconds(new \DateTime(), (int)$firebase->getAccessTokenExpiry('')));
+
+                $dbForConsole->updateDocument('identities', $identity->getId(), $identity);
             }
 
-            $accessToken = $firebase->getAccessToken('');
-            $refreshToken = $firebase->getRefreshToken('');
+            $projects = $firebase->getProjects($accessToken);
 
-            $verificationId = $firebase->getUserID($accessToken);
-
-            if (empty($verificationId)) {
-                throw new Exception(Exception::GENERAL_RATE_LIMIT_EXCEEDED, 'Another request is currently refreshing OAuth token. Please try again.');
+            $output = [];
+            foreach ($projects as $project) {
+                $output[] = [
+                    'displayName' => $project['displayName'],
+                    'projectId' => $project['projectId'],
+                ];
             }
-
-            $identity = $identity
-                ->setAttribute('providerAccessToken', $accessToken)
-                ->setAttribute('providerRefreshToken', $refreshToken)
-                ->setAttribute('providerAccessTokenExpiry', DateTime::addSeconds(new \DateTime(), (int)$firebase->getAccessTokenExpiry('')));
-
-            $dbForConsole->updateDocument('identities', $identity->getId(), $identity);
-        }
-
-        $projects = $firebase->getProjects($accessToken);
-
-        $output = [];
-        foreach ($projects as $project) {
-            $output[] = [
-                'displayName' => $project['displayName'],
-                'projectId' => $project['projectId'],
-            ];
+        } catch (\Exception $e) {
+            throw new Exception(Exception::USER_IDENTITY_NOT_FOUND);
         }
 
         $response->dynamic(new Document([
@@ -843,12 +859,12 @@ App::get('/v1/migrations/supabase/report')
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_MIGRATION_REPORT)
     ->param('resources', [], new ArrayList(new WhiteList(Supabase::getSupportedResources(), true)), 'List of resources to migrate')
-    ->param('endpoint', '', new URL(), 'Source\'s Supabase Endpoint')
-    ->param('apiKey', '', new Text(512), 'Source\'s API Key')
-    ->param('databaseHost', '', new Text(512), 'Source\'s Database Host')
-    ->param('username', '', new Text(512), 'Source\'s Database Username')
-    ->param('password', '', new Text(512), 'Source\'s Database Password')
-    ->param('port', 5432, new Integer(true), 'Source\'s Database Port', true)
+    ->param('endpoint', '', new URL(), 'Source\'s Supabase Endpoint.')
+    ->param('apiKey', '', new Text(512), 'Source\'s API Key.')
+    ->param('databaseHost', '', new Text(512), 'Source\'s Database Host.')
+    ->param('username', '', new Text(512), 'Source\'s Database Username.')
+    ->param('password', '', new Text(512), 'Source\'s Database Password.')
+    ->param('port', 5432, new Integer(true), 'Source\'s Database Port.', true)
     ->inject('response')
     ->inject('dbForProject')
     ->action(function (array $resources, string $endpoint, string $apiKey, string $databaseHost, string $username, string $password, int $port, Response $response) {
@@ -859,7 +875,7 @@ App::get('/v1/migrations/supabase/report')
                 ->setStatusCode(Response::STATUS_CODE_OK)
                 ->dynamic(new Document($supabase->report($resources)), Response::MODEL_MIGRATION_REPORT);
         } catch (\Exception $e) {
-            throw new Exception(Exception::GENERAL_SERVER_ERROR, $e->getMessage());
+            throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Source Error: ' . $e->getMessage());
         }
     });
 
@@ -874,14 +890,14 @@ App::get('/v1/migrations/nhost/report')
     ->label('sdk.response.code', Response::STATUS_CODE_OK)
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_MIGRATION_REPORT)
-    ->param('resources', [], new ArrayList(new WhiteList(NHost::getSupportedResources())), 'List of resources to migrate')
-    ->param('subdomain', '', new URL(), 'Source\'s Subdomain')
-    ->param('region', '', new Text(512), 'Source\'s Region')
-    ->param('adminSecret', '', new Text(512), 'Source\'s Admin Secret')
-    ->param('database', '', new Text(512), 'Source\'s Database Name')
-    ->param('username', '', new Text(512), 'Source\'s Database Username')
-    ->param('password', '', new Text(512), 'Source\'s Database Password')
-    ->param('port', 5432, new Integer(true), 'Source\'s Database Port', true)
+    ->param('resources', [], new ArrayList(new WhiteList(NHost::getSupportedResources())), 'List of resources to migrate.')
+    ->param('subdomain', '', new Text(512), 'Source\'s Subdomain.')
+    ->param('region', '', new Text(512), 'Source\'s Region.')
+    ->param('adminSecret', '', new Text(512), 'Source\'s Admin Secret.')
+    ->param('database', '', new Text(512), 'Source\'s Database Name.')
+    ->param('username', '', new Text(512), 'Source\'s Database Username.')
+    ->param('password', '', new Text(512), 'Source\'s Database Password.')
+    ->param('port', 5432, new Integer(true), 'Source\'s Database Port.', true)
     ->inject('response')
     ->action(function (array $resources, string $subdomain, string $region, string $adminSecret, string $database, string $username, string $password, int $port, Response $response) {
         try {
@@ -891,7 +907,7 @@ App::get('/v1/migrations/nhost/report')
                 ->setStatusCode(Response::STATUS_CODE_OK)
                 ->dynamic(new Document($nhost->report($resources)), Response::MODEL_MIGRATION_REPORT);
         } catch (\Exception $e) {
-            throw new Exception(Exception::GENERAL_SERVER_ERROR, $e->getMessage());
+            throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Source Error: ' . $e->getMessage());
         }
     });
 
@@ -957,9 +973,8 @@ App::delete('/v1/migrations/:migrationId')
     ->param('migrationId', '', new UID(), 'Migration ID.')
     ->inject('response')
     ->inject('dbForProject')
-    ->inject('deletes')
     ->inject('events')
-    ->action(function (string $migrationId, Response $response, Database $dbForProject, Delete $deletes, Event $events) {
+    ->action(function (string $migrationId, Response $response, Database $dbForProject, Event $events) {
         $migration = $dbForProject->getDocument('migrations', $migrationId);
 
         if ($migration->isEmpty()) {
@@ -967,7 +982,7 @@ App::delete('/v1/migrations/:migrationId')
         }
 
         if (!$dbForProject->deleteDocument('migrations', $migration->getId())) {
-            throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove migration from DB', 500);
+            throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove migration from DB');
         }
 
         $events->setParam('migrationId', $migration->getId());

app/controllers/api/project.php

@@ -1,18 +1,24 @@
 <?php
 
+use Appwrite\Extend\Exception;
 use Appwrite\Utopia\Response;
 use Utopia\App;
-use Utopia\Config\Config;
 use Utopia\Database\Database;
-use Utopia\Database\DateTime;
 use Utopia\Database\Document;
+use Utopia\Database\Exception\Duplicate as DuplicateException;
+use Utopia\Database\Helpers\ID;
+use Utopia\Database\Helpers\Permission;
+use Utopia\Database\Helpers\Role;
 use Utopia\Database\Query;
 use Utopia\Database\Validator\Authorization;
+use Utopia\Database\Validator\UID;
+use Utopia\Validator\Text;
 use Utopia\Validator\WhiteList;
+use Utopia\Database\DateTime;
 
 App::get('/v1/project/usage')
     ->desc('Get usage stats for a project')
-    ->groups(['api', 'usage'])
+    ->groups(['api'])
     ->label('scope', 'projects.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
     ->label('sdk.namespace', 'project')
@@ -24,71 +30,282 @@ App::get('/v1/project/usage')
     ->inject('response')
     ->inject('dbForProject')
     ->action(function (string $range, Response $response, Database $dbForProject) {
-
-        $periods = Config::getParam('usage', []);
-        $stats = $usage = [];
-        $days = $periods[$range];
-        $metrics = [
-            METRIC_NETWORK_REQUESTS,
-            METRIC_NETWORK_INBOUND,
-            METRIC_NETWORK_OUTBOUND,
-            METRIC_EXECUTIONS,
-            METRIC_DOCUMENTS,
-            METRIC_DATABASES,
-            METRIC_USERS,
-            METRIC_BUCKETS,
-            METRIC_FILES_STORAGE
-        ];
-
-        Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
-            foreach ($metrics as $metric) {
-                $limit = $days['limit'];
-                $period = $days['period'];
-                $results = $dbForProject->find('stats', [
-                    Query::equal('period', [$period]),
-                    Query::equal('metric', [$metric]),
-                    Query::limit($limit),
-                    Query::orderDesc('time'),
-                ]);
-
-                $stats[$metric] = [];
-                foreach ($results as $result) {
-                    $stats[$metric][$result->getAttribute('time')] = [
-                        'value' => $result->getAttribute('value'),
-                    ];
-                }
-            }
-        });
-
-
-        $format = match ($days['period']) {
-            '1h' => 'Y-m-d\TH:00:00.000P',
-            '1d' => 'Y-m-d\T00:00:00.000P',
-        };
-
-    foreach ($metrics as $metric) {
-        $usage[$metric] = [];
-        $leap = time() - ($days['limit'] * $days['factor']);
-        while ($leap < time()) {
-            $leap += $days['factor'];
-            $formatDate = date($format, $leap);
-            $usage[$metric][] = [
-                'value' => $stats[$metric][$formatDate]['value'] ?? 0,
-                'date' => $formatDate,
+        $usage = [];
+        if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
+            $periods = [
+                '24h' => [
+                    'period' => '1h',
+                    'limit' => 24,
+                ],
+                '7d' => [
+                    'period' => '1d',
+                    'limit' => 7,
+                ],
+                '30d' => [
+                    'period' => '1d',
+                    'limit' => 30,
+                ],
+                '90d' => [
+                    'period' => '1d',
+                    'limit' => 90,
+                ],
             ];
-        }
-    }
 
+            $metrics = [
+                'project.$all.network.requests',
+                'project.$all.network.bandwidth',
+                'project.$all.storage.size',
+                'users.$all.count.total',
+                'databases.$all.count.total',
+                'documents.$all.count.total',
+                'executions.$all.compute.total',
+                'buckets.$all.count.total'
+            ];
+
+            $stats = [];
+
+            Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
+                foreach ($metrics as $metric) {
+                    $limit = $periods[$range]['limit'];
+                    $period = $periods[$range]['period'];
+
+                    $requestDocs = $dbForProject->find('stats', [
+                        Query::equal('period', [$period]),
+                        Query::equal('metric', [$metric]),
+                        Query::limit($limit),
+                        Query::orderDesc('time'),
+                    ]);
+
+                    $stats[$metric] = [];
+                    foreach ($requestDocs as $requestDoc) {
+                        $stats[$metric][] = [
+                            'value' => $requestDoc->getAttribute('value'),
+                            'date' => $requestDoc->getAttribute('time'),
+                        ];
+                    }
+
+                    // backfill metrics with empty values for graphs
+                    $backfill = $limit - \count($requestDocs);
+                    while ($backfill > 0) {
+                        $last = $limit - $backfill - 1; // array index of last added metric
+                        $diff = match ($period) { // convert period to seconds for unix timestamp math
+                            '1h' => 3600,
+                            '1d' => 86400,
+                        };
+                        $stats[$metric][] = [
+                            'value' => 0,
+                            'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
+                        ];
+                        $backfill--;
+                    }
+                    $stats[$metric] = array_reverse($stats[$metric]);
+                }
+            });
+
+            $usage = new Document([
+                'range' => $range,
+                'requests' => $stats[$metrics[0]] ?? [],
+                'network' => $stats[$metrics[1]] ?? [],
+                'storage' => $stats[$metrics[2]] ?? [],
+                'users' => $stats[$metrics[3]] ?? [],
+                'databases' => $stats[$metrics[4]] ?? [],
+                'documents' => $stats[$metrics[5]] ?? [],
+                'executions' => $stats[$metrics[6]] ?? [],
+                'buckets' => $stats[$metrics[7]] ?? [],
+            ]);
+        }
+
+        $response->dynamic($usage, Response::MODEL_USAGE_PROJECT);
+    });
+
+// Variables
+
+App::post('/v1/project/variables')
+    ->desc('Create Variable')
+    ->groups(['api'])
+    ->label('scope', 'projects.write')
+    ->label('audits.event', 'variable.create')
+    ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
+    ->label('sdk.namespace', 'project')
+    ->label('sdk.method', 'createVariable')
+    ->label('sdk.description', '/docs/references/project/create-variable.md')
+    ->label('sdk.response.code', Response::STATUS_CODE_CREATED)
+    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
+    ->label('sdk.response.model', Response::MODEL_VARIABLE)
+    ->param('key', null, new Text(Database::LENGTH_KEY), 'Variable key. Max length: ' . Database::LENGTH_KEY  . ' chars.', false)
+    ->param('value', null, new Text(8192, 0), 'Variable value. Max length: 8192 chars.', false)
+    ->inject('project')
+    ->inject('response')
+    ->inject('dbForProject')
+    ->inject('dbForConsole')
+    ->action(function (string $key, string $value, Document $project, Response $response, Database $dbForProject, Database $dbForConsole) {
+        $variableId = ID::unique();
+
+        $variable = new Document([
+            '$id' => $variableId,
+            '$permissions' => [
+                Permission::read(Role::any()),
+                Permission::update(Role::any()),
+                Permission::delete(Role::any()),
+            ],
+            'resourceInternalId' => '',
+            'resourceId' => '',
+            'resourceType' => 'project',
+            'key' => $key,
+            'value' => $value,
+            'search' => implode(' ', [$variableId, $key, 'project']),
+        ]);
+
+        try {
+            $variable = $dbForProject->createDocument('variables', $variable);
+        } catch (DuplicateException $th) {
+            throw new Exception(Exception::VARIABLE_ALREADY_EXISTS);
+        }
+        $dbForConsole->deleteCachedDocument('projects', $project->getId());
+
+        $functions = $dbForProject->find('functions', [
+            Query::limit(APP_LIMIT_SUBQUERY)
+        ]);
+
+        foreach ($functions as $function) {
+            $dbForProject->updateDocument('functions', $function->getId(), $function->setAttribute('live', false));
+        }
+
+        $dbForProject->deleteCachedDocument('projects', $project->getId());
+
+        $response
+            ->setStatusCode(Response::STATUS_CODE_CREATED)
+            ->dynamic($variable, Response::MODEL_VARIABLE);
+    });
+
+App::get('/v1/project/variables')
+    ->desc('List Variables')
+    ->groups(['api'])
+    ->label('scope', 'projects.read')
+    ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
+    ->label('sdk.namespace', 'project')
+    ->label('sdk.method', 'listVariables')
+    ->label('sdk.description', '/docs/references/project/list-variables.md')
+    ->label('sdk.response.code', Response::STATUS_CODE_OK)
+    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
+    ->label('sdk.response.model', Response::MODEL_VARIABLE_LIST)
+    ->inject('response')
+    ->inject('dbForProject')
+    ->action(function (Response $response, Database $dbForProject) {
+        $variables = $dbForProject->find('variables', [
+            Query::equal('resourceType', ['project']),
+            Query::limit(APP_LIMIT_SUBQUERY)
+        ]);
 
         $response->dynamic(new Document([
-            'range' => $range,
-            'requestsTotal' => ($usage[$metrics[0]]),
-            'network' => ($usage[$metrics[1]] + $usage[$metrics[2]]),
-            'executionsTotal' => $usage[$metrics[3]],
-            'documentsTotal' => $usage[$metrics[4]],
-            'databasesTotal' => $usage[$metrics[5]],
-            'usersTotal' => $usage[$metrics[6]],
-            'bucketsTotal' => $usage[$metrics[7]],
-            'filesStorage' => $usage[$metrics[8]],
-        ]), Response::MODEL_USAGE_PROJECT);
+            'variables' => $variables,
+            'total' => \count($variables),
+        ]), Response::MODEL_VARIABLE_LIST);
+    });
+
+App::get('/v1/project/variables/:variableId')
+    ->desc('Get Variable')
+    ->groups(['api'])
+    ->label('scope', 'projects.read')
+    ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
+    ->label('sdk.namespace', 'project')
+    ->label('sdk.method', 'getVariable')
+    ->label('sdk.description', '/docs/references/project/get-variable.md')
+    ->label('sdk.response.code', Response::STATUS_CODE_OK)
+    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
+    ->label('sdk.response.model', Response::MODEL_VARIABLE)
+    ->param('variableId', '', new UID(), 'Variable unique ID.', false)
+    ->inject('response')
+    ->inject('project')
+    ->inject('dbForProject')
+    ->action(function (string $variableId, Response $response, Document $project, Database $dbForProject) {
+        $variable = $dbForProject->getDocument('variables', $variableId);
+        if ($variable === false || $variable->isEmpty() || $variable->getAttribute('resourceType') !== 'project') {
+            throw new Exception(Exception::VARIABLE_NOT_FOUND);
+        }
+
+        $response->dynamic($variable, Response::MODEL_VARIABLE);
+    });
+
+App::put('/v1/project/variables/:variableId')
+    ->desc('Update Variable')
+    ->groups(['api'])
+    ->label('scope', 'projects.write')
+    ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
+    ->label('sdk.namespace', 'project')
+    ->label('sdk.method', 'updateVariable')
+    ->label('sdk.description', '/docs/references/project/update-variable.md')
+    ->label('sdk.response.code', Response::STATUS_CODE_OK)
+    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
+    ->label('sdk.response.model', Response::MODEL_VARIABLE)
+    ->param('variableId', '', new UID(), 'Variable unique ID.', false)
+    ->param('key', null, new Text(255), 'Variable key. Max length: 255 chars.', false)
+    ->param('value', null, new Text(8192, 0), 'Variable value. Max length: 8192 chars.', true)
+    ->inject('project')
+    ->inject('response')
+    ->inject('dbForProject')
+    ->inject('dbForConsole')
+    ->action(function (string $variableId, string $key, ?string $value, Document $project, Response $response, Database $dbForProject, Database $dbForConsole) {
+        $variable = $dbForProject->getDocument('variables', $variableId);
+        if ($variable === false || $variable->isEmpty() || $variable->getAttribute('resourceType') !== 'project') {
+            throw new Exception(Exception::VARIABLE_NOT_FOUND);
+        }
+
+        $variable
+            ->setAttribute('key', $key)
+            ->setAttribute('value', $value ?? $variable->getAttribute('value'))
+            ->setAttribute('search', implode(' ', [$variableId, $key, 'project']));
+
+        try {
+            $dbForProject->updateDocument('variables', $variable->getId(), $variable);
+        } catch (DuplicateException $th) {
+            throw new Exception(Exception::VARIABLE_ALREADY_EXISTS);
+        }
+        $dbForConsole->deleteCachedDocument('projects', $project->getId());
+
+        $functions = $dbForProject->find('functions', [
+            Query::limit(APP_LIMIT_SUBQUERY)
+        ]);
+
+        foreach ($functions as $function) {
+            $dbForProject->updateDocument('functions', $function->getId(), $function->setAttribute('live', false));
+        }
+
+        $dbForProject->deleteCachedDocument('projects', $project->getId());
+
+        $response->dynamic($variable, Response::MODEL_VARIABLE);
+    });
+
+App::delete('/v1/project/variables/:variableId')
+    ->desc('Delete Variable')
+    ->groups(['api'])
+    ->label('scope', 'projects.write')
+    ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
+    ->label('sdk.namespace', 'project')
+    ->label('sdk.method', 'deleteVariable')
+    ->label('sdk.description', '/docs/references/project/delete-variable.md')
+    ->label('sdk.response.code', Response::STATUS_CODE_NOCONTENT)
+    ->label('sdk.response.model', Response::MODEL_NONE)
+    ->param('variableId', '', new UID(), 'Variable unique ID.', false)
+    ->inject('project')
+    ->inject('response')
+    ->inject('dbForProject')
+    ->action(function (string $variableId, Document $project, Response $response, Database $dbForProject) {
+        $variable = $dbForProject->getDocument('variables', $variableId);
+        if ($variable === false || $variable->isEmpty() || $variable->getAttribute('resourceType') !== 'project') {
+            throw new Exception(Exception::VARIABLE_NOT_FOUND);
+        }
+
+        $functions = $dbForProject->find('functions', [
+            Query::limit(APP_LIMIT_SUBQUERY)
+        ]);
+
+        foreach ($functions as $function) {
+            $dbForProject->updateDocument('functions', $function->getId(), $function->setAttribute('live', false));
+        }
+
+        $dbForProject->deleteDocument('variables', $variable->getId());
+        $dbForProject->deleteCachedDocument('projects', $project->getId());
+
+        $response->noContent();
     });

app/controllers/api/projects.php

@@ -1,19 +1,20 @@
 <?php
 
 use Appwrite\Auth\Auth;
-use Appwrite\Auth\Validator\Password;
-use Appwrite\Event\Certificate;
 use Appwrite\Event\Delete;
 use Appwrite\Event\Validator\Event;
-use Appwrite\Network\Validator\CNAME;
-use Utopia\Validator\Domain as DomainValidator;
+use Appwrite\Extend\Exception;
+use Appwrite\Network\Validator\Email;
 use Appwrite\Network\Validator\Origin;
-use Utopia\Validator\URL;
+use Appwrite\Template\Template;
 use Appwrite\Utopia\Database\Validator\ProjectId;
+use Appwrite\Utopia\Database\Validator\Queries\Projects;
 use Appwrite\Utopia\Response;
+use PHPMailer\PHPMailer\PHPMailer;
 use Utopia\Abuse\Adapters\TimeLimit;
 use Utopia\App;
 use Utopia\Audit\Audit;
+use Utopia\Cache\Cache;
 use Utopia\Config\Config;
 use Utopia\Database\Database;
 use Utopia\Database\DateTime;
@@ -21,26 +22,22 @@ use Utopia\Database\Document;
 use Utopia\Database\Exception\Duplicate;
 use Utopia\Database\Helpers\ID;
 use Utopia\Database\Helpers\Permission;
-use Utopia\Database\Query;
 use Utopia\Database\Helpers\Role;
+use Utopia\Database\Query;
+use Utopia\Database\Validator\Authorization;
 use Utopia\Database\Validator\Datetime as DatetimeValidator;
 use Utopia\Database\Validator\UID;
-use Utopia\Domains\Domain;
-use Appwrite\Extend\Exception;
-use Appwrite\Network\Validator\Email;
-use Appwrite\Utopia\Database\Validator\Queries\Projects;
-use Utopia\Cache\Cache;
+use Utopia\Locale\Locale;
 use Utopia\Pools\Group;
+use Utopia\Registry\Registry;
 use Utopia\Validator\ArrayList;
 use Utopia\Validator\Boolean;
 use Utopia\Validator\Hostname;
 use Utopia\Validator\Integer;
 use Utopia\Validator\Range;
 use Utopia\Validator\Text;
+use Utopia\Validator\URL;
 use Utopia\Validator\WhiteList;
-use Appwrite\Template\Template;
-use Utopia\Locale\Locale;
-use PHPMailer\PHPMailer\PHPMailer;
 
 App::init()
     ->groups(['projects'])
@@ -64,7 +61,7 @@ App::post('/v1/projects')
     ->param('projectId', '', new ProjectId(), 'Unique Id. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, and hyphen. Can\'t start with a special char. Max length is 36 chars.')
     ->param('name', null, new Text(128), 'Project name. Max length: 128 chars.')
     ->param('teamId', '', new UID(), 'Team unique ID.')
-    ->param('region', App::getEnv('_APP_REGION', 'default'), new Whitelist(array_keys(array_filter(Config::getParam('regions'), fn($config) => !$config['disabled']))), 'Project Region.', true)
+    ->param('region', App::getEnv('_APP_REGION', 'default'), new Whitelist(array_keys(array_filter(Config::getParam('regions'), fn ($config) => !$config['disabled']))), 'Project Region.', true)
     ->param('description', '', new Text(256), 'Project description. Max length: 256 chars.', true)
     ->param('logo', '', new Text(1024), 'Project logo.', true)
     ->param('url', '', new URL(), 'Project URL.', true)
@@ -99,6 +96,7 @@ App::post('/v1/projects')
         $backups['database_db_fra1_03'] = ['from' => '10:30', 'to' => '11:15'];
         $backups['database_db_fra1_04'] = ['from' => '13:30', 'to' => '14:15'];
         $backups['database_db_fra1_05'] = ['from' => '4:30', 'to' => '5:15'];
+        $backups['database_db_fra1_06'] = ['from' => '16:30', 'to' => '17:15'];
 
         $databases = Config::getParam('pools-database', []);
 
@@ -122,7 +120,7 @@ App::post('/v1/projects')
             }
         }
 
-        if ($index = array_search('database_db_fra1_05', $databases)) {
+        if ($index = array_search('database_db_fra1_06', $databases)) {
             $database = $databases[$index];
         } else {
             $database = $databases[array_rand($databases)];
@@ -161,7 +159,6 @@ App::post('/v1/projects')
                 'authProviders' => [],
                 'webhooks' => null,
                 'keys' => null,
-                'domains' => null,
                 'auths' => $auths,
                 'search' => implode(' ', [$projectId, $name]),
                 'database' => $database
@@ -293,6 +290,120 @@ App::get('/v1/projects/:projectId')
         $response->dynamic($project, Response::MODEL_PROJECT);
     });
 
+App::get('/v1/projects/:projectId/usage')
+    ->desc('Get usage stats for a project')
+    ->groups(['api', 'projects', 'usage'])
+    ->label('scope', 'projects.read')
+    ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
+    ->label('sdk.namespace', 'projects')
+    ->label('sdk.method', 'getUsage')
+    ->label('sdk.response.code', Response::STATUS_CODE_OK)
+    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
+    ->label('sdk.response.model', Response::MODEL_USAGE_PROJECT)
+    ->param('projectId', '', new UID(), 'Project unique ID.')
+    ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true)
+    ->inject('response')
+    ->inject('dbForConsole')
+    ->inject('dbForProject')
+    ->inject('register')
+    ->action(function (string $projectId, string $range, Response $response, Database $dbForConsole, Database $dbForProject, Registry $register) {
+
+        $project = $dbForConsole->getDocument('projects', $projectId);
+
+        if ($project->isEmpty()) {
+            throw new Exception(Exception::PROJECT_NOT_FOUND);
+        }
+
+        $usage = [];
+        if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
+            $periods = [
+                '24h' => [
+                    'period' => '1h',
+                    'limit' => 24,
+                ],
+                '7d' => [
+                    'period' => '1d',
+                    'limit' => 7,
+                ],
+                '30d' => [
+                    'period' => '1d',
+                    'limit' => 30,
+                ],
+                '90d' => [
+                    'period' => '1d',
+                    'limit' => 90,
+                ],
+            ];
+
+            $dbForProject->setNamespace("_{$project->getInternalId()}");
+
+            $metrics = [
+                'project.$all.network.requests',
+                'project.$all.network.bandwidth',
+                'project.$all.storage.size',
+                'users.$all.count.total',
+                'databases.$all.count.total',
+                'documents.$all.count.total',
+                'executions.$all.compute.total',
+                'buckets.$all.count.total'
+            ];
+
+            $stats = [];
+
+            Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
+                foreach ($metrics as $metric) {
+                    $limit = $periods[$range]['limit'];
+                    $period = $periods[$range]['period'];
+
+                    $requestDocs = $dbForProject->find('stats', [
+                        Query::equal('period', [$period]),
+                        Query::equal('metric', [$metric]),
+                        Query::limit($limit),
+                        Query::orderDesc('time'),
+                    ]);
+
+                    $stats[$metric] = [];
+                    foreach ($requestDocs as $requestDoc) {
+                        $stats[$metric][] = [
+                            'value' => $requestDoc->getAttribute('value'),
+                            'date' => $requestDoc->getAttribute('time'),
+                        ];
+                    }
+
+                    // backfill metrics with empty values for graphs
+                    $backfill = $limit - \count($requestDocs);
+                    while ($backfill > 0) {
+                        $last = $limit - $backfill - 1; // array index of last added metric
+                        $diff = match ($period) { // convert period to seconds for unix timestamp math
+                            '1h' => 3600,
+                            '1d' => 86400,
+                        };
+                        $stats[$metric][] = [
+                            'value' => 0,
+                            'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
+                        ];
+                        $backfill--;
+                    }
+                    $stats[$metric] = array_reverse($stats[$metric]);
+                }
+            });
+
+            $usage = new Document([
+                'range' => $range,
+                'requests' => $stats[$metrics[0]] ?? [],
+                'network' => $stats[$metrics[1]] ?? [],
+                'storage' => $stats[$metrics[2]] ?? [],
+                'users' => $stats[$metrics[3]] ?? [],
+                'databases' => $stats[$metrics[4]] ?? [],
+                'documents' => $stats[$metrics[5]] ?? [],
+                'executions' => $stats[$metrics[6]] ?? [],
+                'buckets' => $stats[$metrics[7]] ?? [],
+            ]);
+        }
+
+        $response->dynamic($usage, Response::MODEL_USAGE_PROJECT);
+    });
+
 App::patch('/v1/projects/:projectId')
     ->desc('Update Project')
     ->groups(['api', 'projects'])
@@ -325,17 +436,17 @@ App::patch('/v1/projects/:projectId')
         }
 
         $project = $dbForConsole->updateDocument('projects', $project->getId(), $project
-                ->setAttribute('name', $name)
-                ->setAttribute('description', $description)
-                ->setAttribute('logo', $logo)
-                ->setAttribute('url', $url)
-                ->setAttribute('legalName', $legalName)
-                ->setAttribute('legalCountry', $legalCountry)
-                ->setAttribute('legalState', $legalState)
-                ->setAttribute('legalCity', $legalCity)
-                ->setAttribute('legalAddress', $legalAddress)
-                ->setAttribute('legalTaxId', $legalTaxId)
-                ->setAttribute('search', implode(' ', [$projectId, $name])));
+            ->setAttribute('name', $name)
+            ->setAttribute('description', $description)
+            ->setAttribute('logo', $logo)
+            ->setAttribute('url', $url)
+            ->setAttribute('legalName', $legalName)
+            ->setAttribute('legalCountry', $legalCountry)
+            ->setAttribute('legalState', $legalState)
+            ->setAttribute('legalCity', $legalCity)
+            ->setAttribute('legalAddress', $legalAddress)
+            ->setAttribute('legalTaxId', $legalTaxId)
+            ->setAttribute('search', implode(' ', [$projectId, $name])));
 
         $response->dynamic($project, Response::MODEL_PROJECT);
     });
@@ -368,14 +479,14 @@ App::patch('/v1/projects/:projectId/team')
         }
 
         $project = $dbForConsole->updateDocument('projects', $project->getId(), $project
-                ->setAttribute('teamId', $teamId)
-                ->setAttribute('$permissions', [
-                    Permission::read(Role::team(ID::custom($teamId))),
-                    Permission::update(Role::team(ID::custom($teamId), 'owner')),
-                    Permission::update(Role::team(ID::custom($teamId), 'developer')),
-                    Permission::delete(Role::team(ID::custom($teamId), 'owner')),
-                    Permission::delete(Role::team(ID::custom($teamId), 'developer')),
-                ]));
+            ->setAttribute('teamId', $teamId)
+            ->setAttribute('$permissions', [
+                Permission::read(Role::team(ID::custom($teamId))),
+                Permission::update(Role::team(ID::custom($teamId), 'owner')),
+                Permission::update(Role::team(ID::custom($teamId), 'developer')),
+                Permission::delete(Role::team(ID::custom($teamId), 'owner')),
+                Permission::delete(Role::team(ID::custom($teamId), 'developer')),
+            ]));
 
         $response->dynamic($project, Response::MODEL_PROJECT);
     });
@@ -391,7 +502,7 @@ App::patch('/v1/projects/:projectId/service')
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_PROJECT)
     ->param('projectId', '', new UID(), 'Project unique ID.')
-    ->param('service', '', new WhiteList(array_keys(array_filter(Config::getParam('services'), fn($element) => $element['optional'])), true), 'Service name.')
+    ->param('service', '', new WhiteList(array_keys(array_filter(Config::getParam('services'), fn ($element) => $element['optional'])), true), 'Service name.')
     ->param('status', null, new Boolean(), 'Service status.')
     ->inject('response')
     ->inject('dbForConsole')
@@ -433,7 +544,7 @@ App::patch('/v1/projects/:projectId/service/all')
             throw new Exception(Exception::PROJECT_NOT_FOUND);
         }
 
-        $allServices = array_keys(array_filter(Config::getParam('services'), fn($element) => $element['optional']));
+        $allServices = array_keys(array_filter(Config::getParam('services'), fn ($element) => $element['optional']));
 
         $services = [];
         foreach ($allServices as $service) {
@@ -732,8 +843,7 @@ App::delete('/v1/projects/:projectId')
 
         $deletes
             ->setType(DELETE_TYPE_DOCUMENT)
-            ->setDocument($project)
-        ;
+            ->setDocument($project);
 
         if (!$dbForConsole->deleteDocument('projects', $projectId)) {
             throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove project from DB');
@@ -911,8 +1021,7 @@ App::put('/v1/projects/:projectId/webhooks/:webhookId')
             ->setAttribute('url', $url)
             ->setAttribute('security', $security)
             ->setAttribute('httpUser', $httpUser)
-            ->setAttribute('httpPass', $httpPass)
-        ;
+            ->setAttribute('httpPass', $httpPass);
 
         $dbForConsole->updateDocument('webhooks', $webhook->getId(), $webhook);
         $dbForConsole->deleteCachedDocument('projects', $project->getId());
@@ -1151,8 +1260,7 @@ App::put('/v1/projects/:projectId/keys/:keyId')
         $key
             ->setAttribute('name', $name)
             ->setAttribute('scopes', $scopes)
-            ->setAttribute('expire', $expire)
-        ;
+            ->setAttribute('expire', $expire);
 
         $dbForConsole->updateDocument('keys', $key->getId(), $key);
 
@@ -1354,8 +1462,7 @@ App::put('/v1/projects/:projectId/platforms/:platformId')
             ->setAttribute('name', $name)
             ->setAttribute('key', $key)
             ->setAttribute('store', $store)
-            ->setAttribute('hostname', $hostname)
-        ;
+            ->setAttribute('hostname', $hostname);
 
         $dbForConsole->updateDocument('platforms', $platform->getId(), $platform);
 
@@ -1401,248 +1508,6 @@ App::delete('/v1/projects/:projectId/platforms/:platformId')
         $response->noContent();
     });
 
-// Domains
-
-App::post('/v1/projects/:projectId/domains')
-    ->desc('Create Domain')
-    ->groups(['api', 'projects'])
-    ->label('scope', 'projects.write')
-    ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
-    ->label('sdk.namespace', 'projects')
-    ->label('sdk.method', 'createDomain')
-    ->label('sdk.response.code', Response::STATUS_CODE_CREATED)
-    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
-    ->label('sdk.response.model', Response::MODEL_DOMAIN)
-    ->param('projectId', '', new UID(), 'Project unique ID.')
-    ->param('domain', null, new DomainValidator(), 'Domain name.')
-    ->inject('response')
-    ->inject('dbForConsole')
-    ->action(function (string $projectId, string $domain, Response $response, Database $dbForConsole) {
-
-        $project = $dbForConsole->getDocument('projects', $projectId);
-
-        if ($project->isEmpty()) {
-            throw new Exception(Exception::PROJECT_NOT_FOUND);
-        }
-
-        if ($domain === App::getEnv('_APP_DOMAIN', '') || $domain === App::getEnv('_APP_DOMAIN_TARGET', '')) {
-            throw new Exception(Exception::DOMAIN_FORBIDDEN);
-        }
-
-        $document = $dbForConsole->findOne('domains', [
-            Query::equal('domain', [$domain])
-        ]);
-
-        if ($document && !$document->isEmpty()) {
-            throw new Exception(Exception::DOMAIN_ALREADY_EXISTS);
-        }
-
-        $target = new Domain(App::getEnv('_APP_DOMAIN_TARGET', ''));
-
-        if (!$target->isKnown() || $target->isTest()) {
-            throw new Exception(Exception::DOMAIN_TARGET_INVALID, 'Unreachable CNAME target (' . $target->get() . '). Please check the _APP_DOMAIN_TARGET environment variable of your Appwrite server.');
-        }
-
-        $domain = new Domain($domain);
-
-        $domain = new Document([
-            '$id' => ID::unique(),
-            '$permissions' => [
-                Permission::read(Role::any()),
-                Permission::update(Role::any()),
-                Permission::delete(Role::any()),
-            ],
-            'projectInternalId' => $project->getInternalId(),
-            'projectId' => $project->getId(),
-            'updated' => DateTime::now(),
-            'domain' => $domain->get(),
-            'tld' => $domain->getSuffix(),
-            'registerable' => $domain->getRegisterable(),
-            'verification' => false,
-            'certificateId' => null,
-        ]);
-
-        $domain = $dbForConsole->createDocument('domains', $domain);
-
-        $dbForConsole->deleteCachedDocument('projects', $project->getId());
-
-        $response
-            ->setStatusCode(Response::STATUS_CODE_CREATED)
-            ->dynamic($domain, Response::MODEL_DOMAIN);
-    });
-
-App::get('/v1/projects/:projectId/domains')
-    ->desc('List Domains')
-    ->groups(['api', 'projects'])
-    ->label('scope', 'projects.read')
-    ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
-    ->label('sdk.namespace', 'projects')
-    ->label('sdk.method', 'listDomains')
-    ->label('sdk.response.code', Response::STATUS_CODE_OK)
-    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
-    ->label('sdk.response.model', Response::MODEL_DOMAIN_LIST)
-    ->param('projectId', '', new UID(), 'Project unique ID.')
-    ->inject('response')
-    ->inject('dbForConsole')
-    ->action(function (string $projectId, Response $response, Database $dbForConsole) {
-
-        $project = $dbForConsole->getDocument('projects', $projectId);
-
-        if ($project->isEmpty()) {
-            throw new Exception(Exception::PROJECT_NOT_FOUND);
-        }
-
-        $domains = $dbForConsole->find('domains', [
-            Query::equal('projectInternalId', [$project->getInternalId()]),
-            Query::limit(5000),
-        ]);
-
-        $response->dynamic(new Document([
-            'domains' => $domains,
-            'total' => count($domains),
-        ]), Response::MODEL_DOMAIN_LIST);
-    });
-
-App::get('/v1/projects/:projectId/domains/:domainId')
-    ->desc('Get Domain')
-    ->groups(['api', 'projects'])
-    ->label('scope', 'projects.read')
-    ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
-    ->label('sdk.namespace', 'projects')
-    ->label('sdk.method', 'getDomain')
-    ->label('sdk.response.code', Response::STATUS_CODE_OK)
-    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
-    ->label('sdk.response.model', Response::MODEL_DOMAIN)
-    ->param('projectId', '', new UID(), 'Project unique ID.')
-    ->param('domainId', '', new UID(), 'Domain unique ID.')
-    ->inject('response')
-    ->inject('dbForConsole')
-    ->action(function (string $projectId, string $domainId, Response $response, Database $dbForConsole) {
-
-        $project = $dbForConsole->getDocument('projects', $projectId);
-
-        if ($project->isEmpty()) {
-            throw new Exception(Exception::PROJECT_NOT_FOUND);
-        }
-
-        $domain = $dbForConsole->findOne('domains', [
-            Query::equal('$id', [$domainId]),
-            Query::equal('projectInternalId', [$project->getInternalId()]),
-        ]);
-
-        if ($domain === false || $domain->isEmpty()) {
-            throw new Exception(Exception::DOMAIN_NOT_FOUND);
-        }
-
-        $response->dynamic($domain, Response::MODEL_DOMAIN);
-    });
-
-App::patch('/v1/projects/:projectId/domains/:domainId/verification')
-    ->desc('Update Domain Verification Status')
-    ->groups(['api', 'projects'])
-    ->label('scope', 'projects.write')
-    ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
-    ->label('sdk.namespace', 'projects')
-    ->label('sdk.method', 'updateDomainVerification')
-    ->label('sdk.response.code', Response::STATUS_CODE_OK)
-    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
-    ->label('sdk.response.model', Response::MODEL_DOMAIN)
-    ->param('projectId', '', new UID(), 'Project unique ID.')
-    ->param('domainId', '', new UID(), 'Domain unique ID.')
-    ->inject('response')
-    ->inject('dbForConsole')
-    ->action(function (string $projectId, string $domainId, Response $response, Database $dbForConsole) {
-
-        $project = $dbForConsole->getDocument('projects', $projectId);
-
-        if ($project->isEmpty()) {
-            throw new Exception(Exception::PROJECT_NOT_FOUND);
-        }
-
-        $domain = $dbForConsole->findOne('domains', [
-            Query::equal('$id', [$domainId]),
-            Query::equal('projectInternalId', [$project->getInternalId()]),
-        ]);
-
-        if ($domain === false || $domain->isEmpty()) {
-            throw new Exception(Exception::DOMAIN_NOT_FOUND);
-        }
-
-        $target = new Domain(App::getEnv('_APP_DOMAIN_TARGET', ''));
-
-        if (!$target->isKnown() || $target->isTest()) {
-            throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Unreachable CNAME target (' . $target->get() . '), please use a domain with a public suffix.');
-        }
-
-        if ($domain->getAttribute('verification') === true) {
-            return $response->dynamic($domain, Response::MODEL_DOMAIN);
-        }
-
-        $validator = new CNAME($target->get()); // Verify Domain with DNS records
-
-        if (!$validator->isValid($domain->getAttribute('domain', ''))) {
-            throw new Exception(Exception::DOMAIN_VERIFICATION_FAILED);
-        }
-
-
-        $dbForConsole->updateDocument('domains', $domain->getId(), $domain->setAttribute('verification', true));
-        $dbForConsole->deleteCachedDocument('projects', $project->getId());
-
-        // Issue a TLS certificate when domain is verified
-        $event = new Certificate();
-        $event
-            ->setDomain($domain)
-            ->trigger();
-
-        $response->dynamic($domain, Response::MODEL_DOMAIN);
-    });
-
-App::delete('/v1/projects/:projectId/domains/:domainId')
-    ->desc('Delete Domain')
-    ->groups(['api', 'projects'])
-    ->label('scope', 'projects.write')
-    ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
-    ->label('sdk.namespace', 'projects')
-    ->label('sdk.method', 'deleteDomain')
-    ->label('sdk.response.code', Response::STATUS_CODE_NOCONTENT)
-    ->label('sdk.response.model', Response::MODEL_NONE)
-    ->param('projectId', '', new UID(), 'Project unique ID.')
-    ->param('domainId', '', new UID(), 'Domain unique ID.')
-    ->inject('response')
-    ->inject('dbForConsole')
-    ->inject('deletes')
-    ->action(function (string $projectId, string $domainId, Response $response, Database $dbForConsole, Delete $deletes) {
-
-        $project = $dbForConsole->getDocument('projects', $projectId);
-
-        if ($project->isEmpty()) {
-            throw new Exception(Exception::PROJECT_NOT_FOUND);
-        }
-
-        $domain = $dbForConsole->findOne('domains', [
-            Query::equal('$id', [$domainId]),
-            Query::equal('projectInternalId', [$project->getInternalId()]),
-        ]);
-
-        if ($domain === false || $domain->isEmpty()) {
-            throw new Exception(Exception::DOMAIN_NOT_FOUND);
-        }
-
-        if ($domain->getAttribute('domain') === App::getEnv('_APP_DOMAIN', '') || $domain->getAttribute('domain') === App::getEnv('_APP_DOMAIN_TARGET', '')) {
-            throw new Exception(Exception::DOMAIN_FORBIDDEN);
-        }
-
-        $dbForConsole->deleteDocument('domains', $domain->getId());
-
-        $dbForConsole->deleteCachedDocument('projects', $project->getId());
-
-        $deletes
-            ->setType(DELETE_TYPE_CERTIFICATES)
-            ->setDocument($domain);
-
-        $response->noContent();
-    });
-
 // CUSTOM SMTP and Templates
 App::patch('/v1/projects/:projectId/smtp')
     ->desc('Update SMTP configuration')
@@ -1656,15 +1521,17 @@ App::patch('/v1/projects/:projectId/smtp')
     ->label('sdk.response.model', Response::MODEL_PROJECT)
     ->param('projectId', '', new UID(), 'Project unique ID.')
     ->param('enabled', false, new Boolean(), 'Enable custom SMTP service')
-    ->param('sender', '', new Email(), 'SMTP sender email')
-    ->param('host', '', new HostName(), 'SMTP server host name')
-    ->param('port', null, new Integer(), 'SMTP server port')
-    ->param('username', null, new Text(0), 'SMTP server username')
-    ->param('password', null, new Text(0), 'SMTP server password')
+    ->param('senderName', '', new Text(255, 0), 'Name of the email sender', true)
+    ->param('senderEmail', '', new Email(), 'Email of the sender', true)
+    ->param('replyTo', '', new Email(), 'Reply to email', true)
+    ->param('host', '', new HostName(), 'SMTP server host name', true)
+    ->param('port', 587, new Integer(), 'SMTP server port', true)
+    ->param('username', '', new Text(0), 'SMTP server username', true)
+    ->param('password', '', new Text(0), 'SMTP server password', true)
     ->param('secure', '', new WhiteList(['tls'], true), 'Does SMTP server use secure connection', true)
     ->inject('response')
     ->inject('dbForConsole')
-    ->action(function (string $projectId, bool $enabled, string $sender, string $host, int $port, string $username, string $password, string $secure, Response $response, Database $dbForConsole) {
+    ->action(function (string $projectId, bool $enabled, string $senderName, string $senderEmail, string $replyTo, string $host, int $port, string $username, string $password, string $secure, Response $response, Database $dbForConsole) {
 
         $project = $dbForConsole->getDocument('projects', $projectId);
 
@@ -1672,30 +1539,64 @@ App::patch('/v1/projects/:projectId/smtp')
             throw new Exception(Exception::PROJECT_NOT_FOUND);
         }
 
-        // validate SMTP settings
-        $mail = new PHPMailer(true);
-        $mail->isSMTP();
-        $mail->Username = $username;
-        $mail->Password = $password;
-        $mail->Host = $host;
-        $mail->Port = $port;
-        $mail->SMTPSecure = $secure;
-        $mail->SMTPAutoTLS = false;
-        $valid = $mail->SmtpConnect();
-
-        if (!$valid) {
-            throw new Exception(Exception::GENERAL_SMTP_DISABLED);
+        // Ensure required params for when enabling SMTP
+        if ($enabled) {
+            if (empty($senderName)) {
+                throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'Sender name is required when enabling SMTP.');
+            } elseif (empty($senderEmail)) {
+                throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'Sender email is required when enabling SMTP.');
+            } elseif (empty($host)) {
+                throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'Host is required when enabling SMTP.');
+            } elseif (empty($port)) {
+                throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'Port is required when enabling SMTP.');
+            } elseif (empty($username)) {
+                throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'Username is required when enabling SMTP.');
+            } elseif (empty($password)) {
+                throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'Password is required when enabling SMTP.');
+            }
         }
 
-        $smtp = [
-            'enabled' => $enabled,
-            'sender' => $sender,
-            'host' => $host,
-            'port' => $port,
-            'username' => $username,
-            'password' => $password,
-            'secure' => $secure,
-        ];
+        // validate SMTP settings
+        if ($enabled) {
+            $mail = new PHPMailer(true);
+            $mail->isSMTP();
+            $mail->Username = $username;
+            $mail->Password = $password;
+            $mail->Host = $host;
+            $mail->Port = $port;
+            $mail->SMTPSecure = $secure;
+            $mail->SMTPAutoTLS = false;
+            $mail->Timeout = 5;
+
+            try {
+                $valid = $mail->SmtpConnect();
+
+                if (!$valid) {
+                    throw new Exception('Connection is not valid.');
+                }
+            } catch (Throwable $error) {
+                throw new Exception(Exception::PROJECT_SMTP_CONFIG_INVALID, 'Could not connect to SMTP server: ' . $error->getMessage());
+            }
+        }
+
+        // Save SMTP settings
+        if ($enabled) {
+            $smtp = [
+                'enabled' => $enabled,
+                'senderName' => $senderName,
+                'senderEmail' => $senderEmail,
+                'replyTo' => $replyTo,
+                'host' => $host,
+                'port' => $port,
+                'username' => $username,
+                'password' => $password,
+                'secure' => $secure,
+            ];
+        } else {
+            $smtp = [
+                'enabled' => false
+            ];
+        }
 
         $project = $dbForConsole->updateDocument('projects', $project->getId(), $project->setAttribute('smtp', $smtp));
 
@@ -1714,11 +1615,13 @@ App::get('/v1/projects/:projectId/templates/sms/:type/:locale')
     ->label('sdk.response.model', Response::MODEL_SMS_TEMPLATE)
     ->param('projectId', '', new UID(), 'Project unique ID.')
     ->param('type', '', new WhiteList(Config::getParam('locale-templates')['sms'] ?? []), 'Template type')
-    ->param('locale', '', fn($localeCodes) => new WhiteList($localeCodes), 'Template locale', false, ['localeCodes'])
+    ->param('locale', '', fn ($localeCodes) => new WhiteList($localeCodes), 'Template locale', false, ['localeCodes'])
     ->inject('response')
     ->inject('dbForConsole')
     ->action(function (string $projectId, string $type, string $locale, Response $response, Database $dbForConsole) {
 
+        throw new Exception(Exception::GENERAL_NOT_IMPLEMENTED);
+
         $project = $dbForConsole->getDocument('projects', $projectId);
 
         if ($project->isEmpty()) {
@@ -1730,7 +1633,7 @@ App::get('/v1/projects/:projectId/templates/sms/:type/:locale')
 
         if (is_null($template)) {
             $template = [
-            'message' => Template::fromFile(__DIR__ . '/../../config/locale/templates/sms-base.tpl')->render(),
+                'message' => Template::fromFile(__DIR__ . '/../../config/locale/templates/sms-base.tpl')->render(),
             ];
         }
 
@@ -1752,7 +1655,7 @@ App::get('/v1/projects/:projectId/templates/email/:type/:locale')
     ->label('sdk.response.model', Response::MODEL_EMAIL_TEMPLATE)
     ->param('projectId', '', new UID(), 'Project unique ID.')
     ->param('type', '', new WhiteList(Config::getParam('locale-templates')['email'] ?? []), 'Template type')
-    ->param('locale', '', fn($localeCodes) => new WhiteList($localeCodes), 'Template locale', false, ['localeCodes'])
+    ->param('locale', '', fn ($localeCodes) => new WhiteList($localeCodes), 'Template locale', false, ['localeCodes'])
     ->inject('response')
     ->inject('dbForConsole')
     ->action(function (string $projectId, string $type, string $locale, Response $response, Database $dbForConsole) {
@@ -1768,27 +1671,22 @@ App::get('/v1/projects/:projectId/templates/email/:type/:locale')
 
         $localeObj = new Locale($locale);
         if (is_null($template)) {
-            $message = Template::fromFile(__DIR__ . '/../../config/locale/templates/email-base.tpl');
-            $message = $message
+            $message = Template::fromFile(__DIR__ . '/../../config/locale/templates/email-inner-base.tpl');
+            $message
                 ->setParam('{{hello}}', $localeObj->getText("emails.{$type}.hello"))
                 ->setParam('{{name}}', '')
-                ->setParam('{{body}}', $localeObj->getText("emails.{$type}.body"))
                 ->setParam('{{footer}}', $localeObj->getText("emails.{$type}.footer"))
+                ->setParam('{{body}}', $localeObj->getText('emails.' . $type . '.body'))
                 ->setParam('{{thanks}}', $localeObj->getText("emails.{$type}.thanks"))
                 ->setParam('{{signature}}', $localeObj->getText("emails.{$type}.signature"))
-                ->setParam('{{direction}}', $localeObj->getText('settings.direction'))
-                ->setParam('{{bg-body}}', '#f7f7f7')
-                ->setParam('{{bg-content}}', '#ffffff')
-                ->setParam('{{text-content}}', '#000000')
-                ->render();
+                ->setParam('{{direction}}', $localeObj->getText('settings.direction'));
+            $message = $message->render();
 
-            $from = $project->isEmpty() || $project->getId() === 'console' ? '' : \sprintf($localeObj->getText('emails.sender'), $project->getAttribute('name'));
-            $from = empty($from) ? \urldecode(App::getEnv('_APP_SYSTEM_EMAIL_NAME', APP_NAME . ' Server')) : $from;
             $template = [
                 'message' => $message,
                 'subject' => $localeObj->getText('emails.' . $type . '.subject'),
-                'senderEmail' => App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', ''),
-                'senderName' => $from
+                'senderEmail' => '',
+                'senderName' => ''
             ];
         }
 
@@ -1810,12 +1708,14 @@ App::patch('/v1/projects/:projectId/templates/sms/:type/:locale')
     ->label('sdk.response.model', Response::MODEL_SMS_TEMPLATE)
     ->param('projectId', '', new UID(), 'Project unique ID.')
     ->param('type', '', new WhiteList(Config::getParam('locale-templates')['sms'] ?? []), 'Template type')
-    ->param('locale', '', fn($localeCodes) => new WhiteList($localeCodes), 'Template locale', false, ['localeCodes'])
+    ->param('locale', '', fn ($localeCodes) => new WhiteList($localeCodes), 'Template locale', false, ['localeCodes'])
     ->param('message', '', new Text(0), 'Template message')
     ->inject('response')
     ->inject('dbForConsole')
     ->action(function (string $projectId, string $type, string $locale, string $message, Response $response, Database $dbForConsole) {
 
+        throw new Exception(Exception::GENERAL_NOT_IMPLEMENTED);
+
         $project = $dbForConsole->getDocument('projects', $projectId);
 
         if ($project->isEmpty()) {
@@ -1848,15 +1748,15 @@ App::patch('/v1/projects/:projectId/templates/email/:type/:locale')
     ->label('sdk.response.model', Response::MODEL_PROJECT)
     ->param('projectId', '', new UID(), 'Project unique ID.')
     ->param('type', '', new WhiteList(Config::getParam('locale-templates')['email'] ?? []), 'Template type')
-    ->param('locale', '', fn($localeCodes) => new WhiteList($localeCodes), 'Template locale', false, ['localeCodes'])
-    ->param('senderName', '', new Text(255), 'Name of the email sender')
-    ->param('senderEmail', '', new Email(), 'Email of the sender')
+    ->param('locale', '', fn ($localeCodes) => new WhiteList($localeCodes), 'Template locale', false, ['localeCodes'])
     ->param('subject', '', new Text(255), 'Email Subject')
     ->param('message', '', new Text(0), 'Template message')
+    ->param('senderName', '', new Text(255, 0), 'Name of the email sender', true)
+    ->param('senderEmail', '', new Email(), 'Email of the sender', true)
     ->param('replyTo', '', new Email(), 'Reply to email', true)
     ->inject('response')
     ->inject('dbForConsole')
-    ->action(function (string $projectId, string $type, string $locale, string $senderName, string $senderEmail, string $subject, string $message, string $replyTo, Response $response, Database $dbForConsole) {
+    ->action(function (string $projectId, string $type, string $locale, string $subject, string $message, string $senderName, string $senderEmail, string $replyTo, Response $response, Database $dbForConsole) {
 
         $project = $dbForConsole->getDocument('projects', $projectId);
 
@@ -1898,11 +1798,13 @@ App::delete('/v1/projects/:projectId/templates/sms/:type/:locale')
     ->label('sdk.response.model', Response::MODEL_SMS_TEMPLATE)
     ->param('projectId', '', new UID(), 'Project unique ID.')
     ->param('type', '', new WhiteList(Config::getParam('locale-templates')['sms'] ?? []), 'Template type')
-    ->param('locale', '', fn($localeCodes) => new WhiteList($localeCodes), 'Template locale', false, ['localeCodes'])
+    ->param('locale', '', fn ($localeCodes) => new WhiteList($localeCodes), 'Template locale', false, ['localeCodes'])
     ->inject('response')
     ->inject('dbForConsole')
     ->action(function (string $projectId, string $type, string $locale, Response $response, Database $dbForConsole) {
 
+        throw new Exception(Exception::GENERAL_NOT_IMPLEMENTED);
+
         $project = $dbForConsole->getDocument('projects', $projectId);
 
         if ($project->isEmpty()) {
@@ -1939,7 +1841,7 @@ App::delete('/v1/projects/:projectId/templates/email/:type/:locale')
     ->label('sdk.response.model', Response::MODEL_EMAIL_TEMPLATE)
     ->param('projectId', '', new UID(), 'Project unique ID.')
     ->param('type', '', new WhiteList(Config::getParam('locale-templates')['email'] ?? []), 'Template type')
-    ->param('locale', '', fn($localeCodes) => new WhiteList($localeCodes), 'Template locale', false, ['localeCodes'])
+    ->param('locale', '', fn ($localeCodes) => new WhiteList($localeCodes), 'Template locale', false, ['localeCodes'])
     ->inject('response')
     ->inject('dbForConsole')
     ->action(function (string $projectId, string $type, string $locale, Response $response, Database $dbForConsole) {

app/controllers/api/proxy.php

@@ -0,0 +1,317 @@
+<?php
+
+use Appwrite\Event\Certificate;
+use Appwrite\Event\Delete;
+use Appwrite\Event\Event;
+use Appwrite\Extend\Exception;
+use Appwrite\Network\Validator\CNAME;
+use Appwrite\Utopia\Database\Validator\Queries\Rules;
+use Appwrite\Utopia\Response;
+use Utopia\App;
+use Utopia\Database\Database;
+use Utopia\Database\Document;
+use Utopia\Database\Helpers\ID;
+use Utopia\Database\Query;
+use Utopia\Database\Validator\UID;
+use Utopia\Domains\Domain;
+use Utopia\Validator\Domain as ValidatorDomain;
+use Utopia\Validator\Text;
+use Utopia\Validator\WhiteList;
+
+App::post('/v1/proxy/rules')
+    ->groups(['api', 'proxy'])
+    ->desc('Create Rule')
+    ->label('scope', 'rules.write')
+    ->label('event', 'rules.[ruleId].create')
+    ->label('audits.event', 'rule.create')
+    ->label('audits.resource', 'rule/{response.$id}')
+    ->label('sdk.auth', [APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_ADMIN])
+    ->label('sdk.namespace', 'proxy')
+    ->label('sdk.method', 'createRule')
+    ->label('sdk.description', '/docs/references/proxy/create-rule.md')
+    ->label('sdk.response.code', Response::STATUS_CODE_CREATED)
+    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
+    ->label('sdk.response.model', Response::MODEL_PROXY_RULE)
+    ->param('domain', null, new ValidatorDomain(), 'Domain name.')
+    ->param('resourceType', null, new WhiteList(['api', 'function']), 'Action definition for the rule. Possible values are "api", "function"')
+    ->param('resourceId', '', new UID(), 'ID of resource for the action type. If resourceType is "api", leave empty. If resourceType is "function", provide ID of the function.', true)
+    ->inject('response')
+    ->inject('project')
+    ->inject('events')
+    ->inject('dbForConsole')
+    ->inject('dbForProject')
+    ->action(function (string $domain, string $resourceType, string $resourceId, Response $response, Document $project, Event $events, Database $dbForConsole, Database $dbForProject) {
+        $mainDomain = App::getEnv('_APP_DOMAIN', '');
+        if ($domain === $mainDomain) {
+            throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'You cannot assign your main domain to specific resource. Please use subdomain or a different domain.');
+        }
+        if ($domain === 'localhost' || $domain === APP_HOSTNAME_INTERNAL) {
+            throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'This domain name is not allowed. Please pick another one.');
+        }
+
+        $document = $dbForConsole->findOne('rules', [
+            Query::equal('domain', [$domain]),
+        ]);
+
+        if ($document && !$document->isEmpty()) {
+            if ($document->getAttribute('projectId') === $project->getId()) {
+                $resourceType = $document->getAttribute('resourceType');
+                $resourceId = $document->getAttribute('resourceId');
+                $message = "Domain already assigned to '{$resourceType}' service";
+                if (!empty($resourceId)) {
+                    $message .= " with ID '{$resourceId}'";
+                }
+
+                $message .= '.';
+            } else {
+                $message = 'Domain already assigned to different project.';
+            }
+
+            throw new Exception(Exception::RULE_ALREADY_EXISTS, $message);
+        }
+
+        $resourceInternalId = '';
+
+        if ($resourceType == 'function') {
+            if (empty($resourceId)) {
+                throw new Exception(Exception::FUNCTION_NOT_FOUND);
+            }
+
+            $function = $dbForProject->getDocument('functions', $resourceId);
+
+            if ($function->isEmpty()) {
+                throw new Exception(Exception::RULE_RESOURCE_NOT_FOUND);
+            }
+
+            $resourceInternalId = $function->getInternalId();
+        }
+
+        $domain = new Domain($domain);
+
+        $ruleId = ID::unique();
+        $rule = new Document([
+            '$id' => $ruleId,
+            'projectId' => $project->getId(),
+            'projectInternalId' => $project->getInternalId(),
+            'domain' => $domain->get(),
+            'resourceType' => $resourceType,
+            'resourceId' => $resourceId,
+            'resourceInternalId' => $resourceInternalId,
+            'certificateId' => '',
+        ]);
+
+        $status = 'created';
+        $functionsDomain = App::getEnv('_APP_DOMAIN_FUNCTIONS');
+        if (!empty($functionsDomain) && \str_ends_with($domain->get(), $functionsDomain)) {
+            $status = 'verified';
+        }
+
+        if ($status === 'created') {
+            $target = new Domain(App::getEnv('_APP_DOMAIN_TARGET', ''));
+            $validator = new CNAME($target->get()); // Verify Domain with DNS records
+
+            if ($validator->isValid($domain->get())) {
+                $status = 'verifying';
+
+                $event = new Certificate();
+                $event
+                    ->setDomain(new Document([
+                        'domain' => $rule->getAttribute('domain')
+                    ]))
+                    ->trigger();
+            }
+        }
+
+        $rule->setAttribute('status', $status);
+        $rule = $dbForConsole->createDocument('rules', $rule);
+
+        $events->setParam('ruleId', $rule->getId());
+
+        $rule->setAttribute('logs', '');
+
+        $response
+            ->setStatusCode(Response::STATUS_CODE_CREATED)
+            ->dynamic($rule, Response::MODEL_PROXY_RULE);
+    });
+
+
+App::get('/v1/proxy/rules')
+    ->groups(['api', 'proxy'])
+    ->desc('List Rules')
+    ->label('scope', 'rules.read')
+    ->label('sdk.auth', [APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_ADMIN])
+    ->label('sdk.namespace', 'proxy')
+    ->label('sdk.method', 'listRules')
+    ->label('sdk.description', '/docs/references/proxy/list-rules.md')
+    ->label('sdk.response.code', Response::STATUS_CODE_OK)
+    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
+    ->label('sdk.response.model', Response::MODEL_PROXY_RULE_LIST)
+    ->param('queries', [], new Rules(), 'Array of query strings generated using the Query class provided by the SDK. [Learn more about queries](https://appwrite.io/docs/databases#querying-documents). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' queries are allowed, each ' . APP_LIMIT_ARRAY_ELEMENT_SIZE . ' characters long. You may filter on the following attributes: ' . implode(', ', Rules::ALLOWED_ATTRIBUTES), true)
+    ->param('search', '', new Text(256), 'Search term to filter your list results. Max length: 256 chars.', true)
+    ->inject('response')
+    ->inject('project')
+    ->inject('dbForConsole')
+    ->action(function (array $queries, string $search, Response $response, Document $project, Database $dbForConsole) {
+        $queries = Query::parseQueries($queries);
+
+        if (!empty($search)) {
+            $queries[] = Query::search('search', $search);
+        }
+
+        $queries[] = Query::equal('projectInternalId', [$project->getInternalId()]);
+
+        // Get cursor document if there was a cursor query
+        $cursor = Query::getByType($queries, [Query::TYPE_CURSORAFTER, Query::TYPE_CURSORBEFORE]);
+        $cursor = reset($cursor);
+        if ($cursor) {
+            /** @var Query $cursor */
+            $ruleId = $cursor->getValue();
+            $cursorDocument = $dbForConsole->getDocument('rules', $ruleId);
+
+            if ($cursorDocument->isEmpty()) {
+                throw new Exception(Exception::GENERAL_CURSOR_NOT_FOUND, "Rule '{$ruleId}' for the 'cursor' value not found.");
+            }
+
+            $cursor->setValue($cursorDocument);
+        }
+
+        $filterQueries = Query::groupByType($queries)['filters'];
+
+        $rules = $dbForConsole->find('rules', $queries);
+        foreach ($rules as $rule) {
+            $certificate = $dbForConsole->getDocument('certificates', $rule->getAttribute('certificateId', ''));
+            $rule->setAttribute('logs', $certificate->getAttribute('logs', ''));
+            $rule->setAttribute('renewAt', $certificate->getAttribute('renewDate', ''));
+        }
+
+        $response->dynamic(new Document([
+            'rules' => $rules,
+            'total' => $dbForConsole->count('rules', $filterQueries, APP_LIMIT_COUNT),
+        ]), Response::MODEL_PROXY_RULE_LIST);
+    });
+
+App::get('/v1/proxy/rules/:ruleId')
+    ->groups(['api', 'proxy'])
+    ->desc('Get Rule')
+    ->label('scope', 'rules.read')
+    ->label('sdk.auth', [APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_ADMIN])
+    ->label('sdk.namespace', 'proxy')
+    ->label('sdk.method', 'getRule')
+    ->label('sdk.description', '/docs/references/proxy/get-rule.md')
+    ->label('sdk.response.code', Response::STATUS_CODE_OK)
+    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
+    ->label('sdk.response.model', Response::MODEL_PROXY_RULE)
+    ->param('ruleId', '', new UID(), 'Rule ID.')
+    ->inject('response')
+    ->inject('project')
+    ->inject('dbForConsole')
+    ->action(function (string $ruleId, Response $response, Document $project, Database $dbForConsole) {
+        $rule = $dbForConsole->getDocument('rules', $ruleId);
+
+        if ($rule->isEmpty() || $rule->getAttribute('projectInternalId') !== $project->getInternalId()) {
+            throw new Exception(Exception::RULE_NOT_FOUND);
+        }
+
+        $certificate = $dbForConsole->getDocument('certificates', $rule->getAttribute('certificateId', ''));
+        $rule->setAttribute('logs', $certificate->getAttribute('logs', ''));
+        $rule->setAttribute('renewAt', $certificate->getAttribute('renewDate', ''));
+
+        $response->dynamic($rule, Response::MODEL_PROXY_RULE);
+    });
+
+App::delete('/v1/proxy/rules/:ruleId')
+    ->groups(['api', 'proxy'])
+    ->desc('Delete Rule')
+    ->label('scope', 'rules.write')
+    ->label('event', 'rules.[ruleId].delete')
+    ->label('audits.event', 'rules.delete')
+    ->label('audits.resource', 'rule/{request.ruleId}')
+    ->label('sdk.auth', [APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_ADMIN])
+    ->label('sdk.namespace', 'proxy')
+    ->label('sdk.method', 'deleteRule')
+    ->label('sdk.description', '/docs/references/proxy/delete-rule.md')
+    ->label('sdk.response.code', Response::STATUS_CODE_NOCONTENT)
+    ->label('sdk.response.model', Response::MODEL_NONE)
+    ->param('ruleId', '', new UID(), 'Rule ID.')
+    ->inject('response')
+    ->inject('project')
+    ->inject('dbForConsole')
+    ->inject('deletes')
+    ->inject('events')
+    ->action(function (string $ruleId, Response $response, Document $project, Database $dbForConsole, Delete $deletes, Event $events) {
+        $rule = $dbForConsole->getDocument('rules', $ruleId);
+
+        if ($rule->isEmpty() || $rule->getAttribute('projectInternalId') !== $project->getInternalId()) {
+            throw new Exception(Exception::RULE_NOT_FOUND);
+        }
+
+        $dbForConsole->deleteDocument('rules', $rule->getId());
+
+        $deletes
+            ->setType(DELETE_TYPE_DOCUMENT)
+            ->setDocument($rule);
+
+        $events->setParam('ruleId', $rule->getId());
+
+        $response->noContent();
+    });
+
+App::patch('/v1/proxy/rules/:ruleId/verification')
+    ->desc('Update Rule Verification Status')
+    ->groups(['api', 'proxy'])
+    ->label('scope', 'rules.write')
+    ->label('event', 'rules.[ruleId].update')
+    ->label('audits.event', 'rule.update')
+    ->label('audits.resource', 'rule/{response.$id}')
+    ->label('sdk.auth', [APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_ADMIN])
+    ->label('sdk.namespace', 'proxy')
+    ->label('sdk.method', 'updateRuleVerification')
+    ->label('sdk.response.code', Response::STATUS_CODE_OK)
+    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
+    ->label('sdk.response.model', Response::MODEL_PROXY_RULE)
+    ->param('ruleId', '', new UID(), 'Rule ID.')
+    ->inject('response')
+    ->inject('events')
+    ->inject('project')
+    ->inject('dbForConsole')
+    ->action(function (string $ruleId, Response $response, Event $events, Document $project, Database $dbForConsole) {
+        $rule = $dbForConsole->getDocument('rules', $ruleId);
+
+        if ($rule->isEmpty() || $rule->getAttribute('projectInternalId') !== $project->getInternalId()) {
+            throw new Exception(Exception::RULE_NOT_FOUND);
+        }
+
+        $target = new Domain(App::getEnv('_APP_DOMAIN_TARGET', ''));
+
+        if (!$target->isKnown() || $target->isTest()) {
+            throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Domain target must be configured as environment variable.');
+        }
+
+        if ($rule->getAttribute('verification') === true) {
+            return $response->dynamic($rule, Response::MODEL_PROXY_RULE);
+        }
+
+        $validator = new CNAME($target->get()); // Verify Domain with DNS records
+        $domain = new Domain($rule->getAttribute('domain', ''));
+
+        if (!$validator->isValid($domain->get())) {
+            throw new Exception(Exception::RULE_VERIFICATION_FAILED);
+        }
+
+        $dbForConsole->updateDocument('rules', $rule->getId(), $rule->setAttribute('status', 'verifying'));
+
+        // Issue a TLS certificate when domain is verified
+        $event = new Certificate();
+        $event
+            ->setDomain(new Document([
+                'domain' => $rule->getAttribute('domain')
+            ]))
+            ->trigger();
+
+        $events->setParam('ruleId', $rule->getId());
+
+        $certificate = $dbForConsole->getDocument('certificates', $rule->getAttribute('certificateId', ''));
+        $rule->setAttribute('logs', $certificate->getAttribute('logs', ''));
+
+        $response->dynamic($rule, Response::MODEL_PROXY_RULE);
+    });

app/controllers/api/storage.php

@@ -52,6 +52,7 @@ App::post('/v1/storage/buckets')
     ->label('event', 'buckets.[bucketId].create')
     ->label('audits.event', 'bucket.create')
     ->label('audits.resource', 'bucket/{response.$id}')
+    ->label('usage.metric', 'buckets.{scope}.requests.create')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'storage')
     ->label('sdk.method', 'createBucket')
@@ -63,8 +64,8 @@ App::post('/v1/storage/buckets')
     ->param('name', '', new Text(128), 'Bucket name')
     ->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE), 'An array of permission strings. By default, no user is granted with any permissions. [Learn more about permissions](/docs/permissions).', true)
     ->param('fileSecurity', false, new Boolean(true), 'Enables configuring permissions for individual file. A user needs one of file or bucket level permissions to access a file. [Learn more about permissions](/docs/permissions).', true)
-    ->param('enabled', true, new Boolean(true), 'Is bucket enabled?', true)
-    ->param('maximumFileSize', (int) App::getEnv('_APP_STORAGE_LIMIT', 0), new Range(1, (int) App::getEnv('_APP_STORAGE_LIMIT', 0)), 'Maximum file size allowed in bytes. Maximum allowed value is ' . Storage::human(App::getEnv('_APP_STORAGE_LIMIT', 0), 0) . '. For self-hosted setups you can change the max limit by changing the `_APP_STORAGE_LIMIT` environment variable. [Learn more about storage environment variables](/docs/environment-variables#storage)', true)
+    ->param('enabled', true, new Boolean(true), 'Is bucket enabled? When set to \'disabled\', users cannot access the files in this bucket but Server SDKs with and API key can still access the bucket. No files are lost when this is toggled.', true)
+    ->param('maximumFileSize', (int) App::getEnv('_APP_STORAGE_LIMIT', 0), new Range(1, (int) App::getEnv('_APP_STORAGE_LIMIT', 0)), 'Maximum file size allowed in bytes. Maximum allowed value is ' . Storage::human(App::getEnv('_APP_STORAGE_LIMIT', 0), 0) . '.', true)
     ->param('allowedFileExtensions', [], new ArrayList(new Text(64), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Allowed file extensions. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' extensions are allowed, each 64 characters long.', true)
     ->param('compression', COMPRESSION_TYPE_NONE, new WhiteList([COMPRESSION_TYPE_NONE, COMPRESSION_TYPE_GZIP, COMPRESSION_TYPE_ZSTD]), 'Compression algorithm choosen for compression. Can be one of ' . COMPRESSION_TYPE_NONE . ',  [' . COMPRESSION_TYPE_GZIP . '](https://en.wikipedia.org/wiki/Gzip), or [' . COMPRESSION_TYPE_ZSTD . '](https://en.wikipedia.org/wiki/Zstd), For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' compression is skipped even if it\'s enabled', true)
     ->param('encryption', true, new Boolean(true), 'Is encryption enabled? For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' encryption is skipped even if it\'s enabled', true)
@@ -147,6 +148,7 @@ App::get('/v1/storage/buckets')
     ->desc('List buckets')
     ->groups(['api', 'storage'])
     ->label('scope', 'buckets.read')
+    ->label('usage.metric', 'buckets.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'storage')
     ->label('sdk.method', 'listBuckets')
@@ -195,6 +197,7 @@ App::get('/v1/storage/buckets/:bucketId')
     ->desc('Get Bucket')
     ->groups(['api', 'storage'])
     ->label('scope', 'buckets.read')
+    ->label('usage.metric', 'buckets.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'storage')
     ->label('sdk.method', 'getBucket')
@@ -223,6 +226,7 @@ App::put('/v1/storage/buckets/:bucketId')
     ->label('event', 'buckets.[bucketId].update')
     ->label('audits.event', 'bucket.update')
     ->label('audits.resource', 'bucket/{response.$id}')
+    ->label('usage.metric', 'buckets.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'storage')
     ->label('sdk.method', 'updateBucket')
@@ -234,8 +238,8 @@ App::put('/v1/storage/buckets/:bucketId')
     ->param('name', null, new Text(128), 'Bucket name', false)
     ->param('permissions', null, new Permissions(APP_LIMIT_ARRAY_PARAMS_SIZE), 'An array of permission strings. By default, the current permissions are inherited. [Learn more about permissions](/docs/permissions).', true)
     ->param('fileSecurity', false, new Boolean(true), 'Enables configuring permissions for individual file. A user needs one of file or bucket level permissions to access a file. [Learn more about permissions](/docs/permissions).', true)
-    ->param('enabled', true, new Boolean(true), 'Is bucket enabled?', true)
-    ->param('maximumFileSize', null, new Range(1, (int) App::getEnv('_APP_STORAGE_LIMIT', 0)), 'Maximum file size allowed in bytes. Maximum allowed value is ' . Storage::human((int)App::getEnv('_APP_STORAGE_LIMIT', 0), 0) . '. For self hosted version you can change the limit by changing _APP_STORAGE_LIMIT environment variable. [Learn more about storage environment variables](/docs/environment-variables#storage)', true)
+    ->param('enabled', true, new Boolean(true), 'Is bucket enabled? When set to \'disabled\', users cannot access the files in this bucket but Server SDKs with and API key can still access the bucket. No files are lost when this is toggled.', true)
+    ->param('maximumFileSize', null, new Range(1, (int) App::getEnv('_APP_STORAGE_LIMIT', 0)), 'Maximum file size allowed in bytes. Maximum allowed value is ' . Storage::human((int)App::getEnv('_APP_STORAGE_LIMIT', 0), 0) . '.', true)
     ->param('allowedFileExtensions', [], new ArrayList(new Text(64), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Allowed file extensions. Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' extensions are allowed, each 64 characters long.', true)
     ->param('compression', COMPRESSION_TYPE_NONE, new WhiteList([COMPRESSION_TYPE_NONE, COMPRESSION_TYPE_GZIP, COMPRESSION_TYPE_ZSTD]), 'Compression algorithm choosen for compression. Can be one of ' . COMPRESSION_TYPE_NONE . ', [' . COMPRESSION_TYPE_GZIP . '](https://en.wikipedia.org/wiki/Gzip), or [' . COMPRESSION_TYPE_ZSTD . '](https://en.wikipedia.org/wiki/Zstd), For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' compression is skipped even if it\'s enabled', true)
     ->param('encryption', true, new Boolean(true), 'Is encryption enabled? For file size above ' . Storage::human(APP_STORAGE_READ_BUFFER, 0) . ' encryption is skipped even if it\'s enabled', true)
@@ -290,6 +294,7 @@ App::delete('/v1/storage/buckets/:bucketId')
     ->label('audits.event', 'bucket.delete')
     ->label('event', 'buckets.[bucketId].delete')
     ->label('audits.resource', 'bucket/{request.bucketId}')
+    ->label('usage.metric', 'buckets.{scope}.requests.delete')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'storage')
     ->label('sdk.method', 'deleteBucket')
@@ -332,6 +337,8 @@ App::post('/v1/storage/buckets/:bucketId/files')
     ->label('audits.event', 'file.create')
     ->label('event', 'buckets.[bucketId].files.[fileId].create')
     ->label('audits.resource', 'file/{response.$id}')
+    ->label('usage.metric', 'files.{scope}.requests.create')
+    ->label('usage.params', ['bucketId:{request.bucketId}'])
     ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
     ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT)
     ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
@@ -681,6 +688,8 @@ App::get('/v1/storage/buckets/:bucketId/files')
     ->groups(['api', 'storage'])
     ->label('scope', 'files.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
+    ->label('usage.metric', 'files.{scope}.requests.read')
+    ->label('usage.params', ['bucketId:{request.bucketId}'])
     ->label('sdk.namespace', 'storage')
     ->label('sdk.method', 'listFiles')
     ->label('sdk.description', '/docs/references/storage/list-files.md')
@@ -760,6 +769,8 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId')
     ->groups(['api', 'storage'])
     ->label('scope', 'files.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
+    ->label('usage.metric', 'files.{scope}.requests.read')
+    ->label('usage.params', ['bucketId:{request.bucketId}'])
     ->label('sdk.namespace', 'storage')
     ->label('sdk.method', 'getFile')
     ->label('sdk.description', '/docs/references/storage/get-file.md')
@@ -809,6 +820,8 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/preview')
     ->label('cache', true)
     ->label('cache.resourceType', 'bucket/{request.bucketId}')
     ->label('cache.resource', 'file/{request.fileId}')
+    ->label('usage.metric', 'files.{scope}.requests.read')
+    ->label('usage.params', ['bucketId:{request.bucketId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'storage')
     ->label('sdk.method', 'getFilePreview')
@@ -973,6 +986,8 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/download')
     ->desc('Get File for Download')
     ->groups(['api', 'storage'])
     ->label('scope', 'files.read')
+    ->label('usage.metric', 'files.{scope}.requests.read')
+    ->label('usage.params', ['bucketId:{request.bucketId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'storage')
     ->label('sdk.method', 'getFileDownload')
@@ -1114,6 +1129,8 @@ App::get('/v1/storage/buckets/:bucketId/files/:fileId/view')
     ->desc('Get File for View')
     ->groups(['api', 'storage'])
     ->label('scope', 'files.read')
+    ->label('usage.metric', 'files.{scope}.requests.read')
+    ->label('usage.params', ['bucketId:{request.bucketId}'])
     ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
     ->label('sdk.namespace', 'storage')
     ->label('sdk.method', 'getFileView')
@@ -1268,6 +1285,8 @@ App::put('/v1/storage/buckets/:bucketId/files/:fileId')
     ->label('event', 'buckets.[bucketId].files.[fileId].update')
     ->label('audits.event', 'file.update')
     ->label('audits.resource', 'file/{response.$id}')
+    ->label('usage.metric', 'files.{scope}.requests.update')
+    ->label('usage.params', ['bucketId:{request.bucketId}'])
     ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
     ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT)
     ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
@@ -1369,13 +1388,14 @@ App::put('/v1/storage/buckets/:bucketId/files/:fileId')
     });
 
 App::delete('/v1/storage/buckets/:bucketId/files/:fileId')
-    ->alias('/v1/storage/files/:fileId', ['bucketId' => 'default'])
     ->desc('Delete File')
     ->groups(['api', 'storage'])
     ->label('scope', 'files.write')
     ->label('event', 'buckets.[bucketId].files.[fileId].delete')
     ->label('audits.event', 'file.delete')
     ->label('audits.resource', 'file/{request.fileId}')
+    ->label('usage.metric', 'files.{scope}.requests.delete')
+    ->label('usage.params', ['bucketId:{request.bucketId}'])
     ->label('abuse-key', 'ip:{ip},method:{method},url:{url},userId:{userId}')
     ->label('abuse-limit', APP_LIMIT_WRITE_RATE_DEFAULT)
     ->label('abuse-time', APP_LIMIT_WRITE_RATE_PERIOD_DEFAULT)
@@ -1480,62 +1500,103 @@ App::get('/v1/storage/usage')
     ->inject('dbForProject')
     ->action(function (string $range, Response $response, Database $dbForProject) {
 
-        $periods = Config::getParam('usage', []);
-        $stats = $usage = [];
-        $days = $periods[$range];
-        $metrics = [
-            METRIC_BUCKETS,
-            METRIC_FILES,
-            METRIC_FILES_STORAGE,
-        ];
+        $usage = [];
+        if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') {
+            $periods = [
+                '24h' => [
+                    'period' => '1h',
+                    'limit' => 24,
+                ],
+                '7d' => [
+                    'period' => '1d',
+                    'limit' => 7,
+                ],
+                '30d' => [
+                    'period' => '1d',
+                    'limit' => 30,
+                ],
+                '90d' => [
+                    'period' => '1d',
+                    'limit' => 90,
+                ],
+            ];
 
-        Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
-            foreach ($metrics as $metric) {
-                $limit = $days['limit'];
-                $period = $days['period'];
-                $results = $dbForProject->find('stats', [
-                    Query::equal('period', [$period]),
-                    Query::equal('metric', [$metric]),
-                    Query::limit($limit),
-                    Query::orderDesc('time'),
-                ]);
-                $stats[$metric] = [];
-                foreach ($results as $result) {
-                    $stats[$metric][$result->getAttribute('time')] = [
-                        'value' => $result->getAttribute('value'),
-                    ];
+            $metrics = [
+                'project.$all.storage.size',
+                'buckets.$all.count.total',
+                'buckets.$all.requests.create',
+                'buckets.$all.requests.read',
+                'buckets.$all.requests.update',
+                'buckets.$all.requests.delete',
+                'files.$all.storage.size',
+                'files.$all.count.total',
+                'files.$all.requests.create',
+                'files.$all.requests.read',
+                'files.$all.requests.update',
+                'files.$all.requests.delete',
+            ];
+
+            $stats = [];
+
+            Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
+                foreach ($metrics as $metric) {
+                    $limit = $periods[$range]['limit'];
+                    $period = $periods[$range]['period'];
+
+                    $requestDocs = $dbForProject->find('stats', [
+                        Query::equal('period', [$period]),
+                        Query::equal('metric', [$metric]),
+                        Query::limit($limit),
+                        Query::orderDesc('time'),
+                    ]);
+
+                    $stats[$metric] = [];
+                    foreach ($requestDocs as $requestDoc) {
+                        $stats[$metric][] = [
+                            'value' => $requestDoc->getAttribute('value'),
+                            'date' => $requestDoc->getAttribute('time'),
+                        ];
+                    }
+
+                    // backfill metrics with empty values for graphs
+                    $backfill = $limit - \count($requestDocs);
+                    while ($backfill > 0) {
+                        $last = $limit - $backfill - 1; // array index of last added metric
+                        $diff = match ($period) { // convert period to seconds for unix timestamp math
+                            '1h' => 3600,
+                            '1d' => 86400,
+                        };
+                        $stats[$metric][] = [
+                            'value' => 0,
+                            'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
+                        ];
+                        $backfill--;
+                    }
+                    $stats[$metric] = array_reverse($stats[$metric]);
                 }
-            }
-        });
+            });
 
-        $format = (match ($days['period']) {
-            '1h' => 'Y-m-d\TH:00:00.000P',
-            '1d' => 'Y-m-d\T00:00:00.000P',
-        });
-
-        foreach ($metrics as $metric) {
-            $usage[$metric] = [];
-            $leap = time() - ($days['limit'] * $days['factor']);
-            while ($leap < time()) {
-                $leap += $days['factor'];
-                $formatDate = date($format, $leap);
-                $usage[$metric][] = [
-                    'value' => $stats[$metric][$formatDate]['value'] ?? 0,
-                    'date' => $formatDate,
-                ];
-            }
+            $usage = new Document([
+                'range' => $range,
+                'bucketsCount' => $stats['buckets.$all.count.total'],
+                'bucketsCreate' => $stats['buckets.$all.requests.create'],
+                'bucketsRead' => $stats['buckets.$all.requests.read'],
+                'bucketsUpdate' => $stats['buckets.$all.requests.update'],
+                'bucketsDelete' => $stats['buckets.$all.requests.delete'],
+                'storage' => $stats['project.$all.storage.size'],
+                'filesCount' => $stats['files.$all.count.total'],
+                'filesCreate' => $stats['files.$all.requests.create'],
+                'filesRead' => $stats['files.$all.requests.read'],
+                'filesUpdate' => $stats['files.$all.requests.update'],
+                'filesDelete' => $stats['files.$all.requests.delete'],
+            ]);
         }
 
-        $response->dynamic(new Document([
-            'range' => $range,
-            'bucketsTotal' => $usage[$metrics[0]],
-            'filesTotal' => $usage[$metrics[1]],
-            'filesStorage' => $usage[$metrics[2]],
-        ]), Response::MODEL_USAGE_STORAGE);
+        $response->dynamic($usage, Response::MODEL_USAGE_STORAGE);
     });
 
 App::get('/v1/storage/:bucketId/usage')
-    ->desc('Get usage stats for storage bucket')
+    ->desc('Get usage stats for a storage bucket')
     ->groups(['api', 'storage', 'usage'])
     ->label('scope', 'files.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_ADMIN])
@@ -1556,55 +1617,86 @@ App::get('/v1/storage/:bucketId/usage')
             throw new Exception(Exception::STORAGE_BUCKET_NOT_FOUND);
         }
 
-        $periods = Config::getParam('usage', []);
-        $stats = $usage = [];
-        $days = $periods[$range];
-        $metrics = [
-            str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES),
-            str_replace('{bucketInternalId}', $bucket->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE),
-        ];
+        $usage = [];
+        if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') {
+            $periods = [
+                '24h' => [
+                    'period' => '1h',
+                    'limit' => 24,
+                ],
+                '7d' => [
+                    'period' => '1d',
+                    'limit' => 7,
+                ],
+                '30d' => [
+                    'period' => '1d',
+                    'limit' => 30,
+                ],
+                '90d' => [
+                    'period' => '1d',
+                    'limit' => 90,
+                ],
+            ];
 
-        Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
-            foreach ($metrics as $metric) {
-                $limit = $days['limit'];
-                $period = $days['period'];
-                $results = $dbForProject->find('stats', [
-                    Query::equal('period', [$period]),
-                    Query::equal('metric', [$metric]),
-                    Query::limit($limit),
-                    Query::orderDesc('time'),
-                ]);
-                $stats[$metric] = [];
-                foreach ($results as $result) {
-                    $stats[$metric][$result->getAttribute('time')] = [
-                        'value' => $result->getAttribute('value'),
-                    ];
+            $metrics = [
+                "files.{$bucketId}.count.total",
+                "files.{$bucketId}.storage.size",
+                "files.{$bucketId}.requests.create",
+                "files.{$bucketId}.requests.read",
+                "files.{$bucketId}.requests.update",
+                "files.{$bucketId}.requests.delete",
+            ];
+
+            $stats = [];
+
+            Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
+                foreach ($metrics as $metric) {
+                    $limit = $periods[$range]['limit'];
+                    $period = $periods[$range]['period'];
+
+                    $requestDocs = $dbForProject->find('stats', [
+                        Query::equal('period', [$period]),
+                        Query::equal('metric', [$metric]),
+                        Query::limit($limit),
+                        Query::orderDesc('time'),
+                    ]);
+
+                    $stats[$metric] = [];
+                    foreach ($requestDocs as $requestDoc) {
+                        $stats[$metric][] = [
+                            'value' => $requestDoc->getAttribute('value'),
+                            'date' => $requestDoc->getAttribute('time'),
+                        ];
+                    }
+
+                    // backfill metrics with empty values for graphs
+                    $backfill = $limit - \count($requestDocs);
+                    while ($backfill > 0) {
+                        $last = $limit - $backfill - 1; // array index of last added metric
+                        $diff = match ($period) { // convert period to seconds for unix timestamp math
+                            '1h' => 3600,
+                            '1d' => 86400,
+                        };
+                        $stats[$metric][] = [
+                            'value' => 0,
+                            'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
+                        ];
+                        $backfill--;
+                    }
+                    $stats[$metric] = array_reverse($stats[$metric]);
                 }
-            }
-        });
+            });
 
-        $format = (match ($days['period']) {
-            '1h' => 'Y-m-d\TH:00:00.000P',
-            '1d' => 'Y-m-d\T00:00:00.000P',
-        });
-
-        foreach ($metrics as $metric) {
-            $usage[$metric] = [];
-            $leap = time() - ($days['limit'] * $days['factor']);
-
-            while ($leap < time()) {
-                $leap += $days['factor'];
-                $formatDate = date($format, $leap);
-                $usage[$metric][] = [
-                    'value' => $stats[$metric][$formatDate]['value'] ?? 0,
-                    'date' => $formatDate,
-                ];
-            }
+            $usage = new Document([
+                'range' => $range,
+                'filesCount' => $stats[$metrics[0]],
+                'filesStorage' => $stats[$metrics[1]],
+                'filesCreate' => $stats[$metrics[2]],
+                'filesRead' => $stats[$metrics[3]],
+                'filesUpdate' => $stats[$metrics[4]],
+                'filesDelete' => $stats[$metrics[5]],
+            ]);
         }
 
-        $response->dynamic(new Document([
-            'range' => $range,
-            'filesTotal' => $usage[$metrics[0]],
-            'filesStorage' => $usage[$metrics[1]],
-        ]), Response::MODEL_USAGE_BUCKETS);
+        $response->dynamic($usage, Response::MODEL_USAGE_BUCKETS);
     });

app/controllers/api/teams.php

@@ -482,7 +482,8 @@ App::post('/v1/teams/:teamId/memberships')
                     'sessions' => null,
                     'tokens' => null,
                     'memberships' => null,
-                    'search' => implode(' ', [$userId, $email, $name])
+                    'search' => implode(' ', [$userId, $email, $name]),
+                    'accessedAt' => DateTime::now(),
                 ])));
             } catch (Duplicate $th) {
                 throw new Exception(Exception::USER_ALREADY_EXISTS);
@@ -542,44 +543,84 @@ App::post('/v1/teams/:teamId/memberships')
             if (!empty($email)) {
                 $projectName = $project->isEmpty() ? 'Console' : $project->getAttribute('name', '[APP-NAME]');
 
-                $from = $project->isEmpty() || $project->getId() === 'console' ? '' : \sprintf($locale->getText('emails.sender'), $projectName);
-                $body = Template::fromFile(__DIR__ . '/../../config/locale/templates/email-base.tpl');
+                $body = $locale->getText("emails.invitation.body");
                 $subject = \sprintf($locale->getText("emails.invitation.subject"), $team->getAttribute('name'), $projectName);
-
-                $smtpEnabled = $project->getAttribute('smtp', [])['enabled'] ?? false;
                 $customTemplate = $project->getAttribute('templates', [])['email.invitation-' . $locale->default] ?? [];
-                if ($smtpEnabled && !empty($customTemplate)) {
-                    $body = $customTemplate['message'];
-                    $subject = $customTemplate['subject'];
-                    $from = $customTemplate['senderName'];
+
+                $message = Template::fromFile(__DIR__ . '/../../config/locale/templates/email-inner-base.tpl');
+                $message->setParam('{{body}}', $body);
+                $body = $message->render();
+
+                $smtp = $project->getAttribute('smtp', []);
+                $smtpEnabled = $smtp['enabled'] ?? false;
+
+                $senderEmail = App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM);
+                $senderName = App::getEnv('_APP_SYSTEM_EMAIL_NAME', APP_NAME . ' Server');
+                $replyTo = "";
+
+                if ($smtpEnabled) {
+                    if (!empty($smtp['senderEmail'])) {
+                        $senderEmail = $smtp['senderEmail'];
+                    }
+                    if (!empty($smtp['senderName'])) {
+                        $senderName = $smtp['senderName'];
+                    }
+                    if (!empty($smtp['replyTo'])) {
+                        $replyTo = $smtp['replyTo'];
+                    }
+
+                    $mails
+                        ->setSmtpHost($smtp['host'] ?? '')
+                        ->setSmtpPort($smtp['port'] ?? '')
+                        ->setSmtpUsername($smtp['username'] ?? '')
+                        ->setSmtpPassword($smtp['password'] ?? '')
+                        ->setSmtpSecure($smtp['secure'] ?? '');
                 }
 
-                $body->setParam('{{owner}}', $user->getAttribute('name'));
-                $body->setParam('{{team}}', $team->getAttribute('name'));
+                if (!empty($customTemplate)) {
+                    if (!empty($customTemplate['senderEmail'])) {
+                        $senderEmail = $customTemplate['senderEmail'];
+                    }
+                    if (!empty($customTemplate['senderName'])) {
+                        $senderName = $customTemplate['senderName'];
+                    }
+                    if (!empty($customTemplate['replyTo'])) {
+                        $replyTo = $customTemplate['replyTo'];
+                    }
 
-                $body
-                    ->setParam('{{subject}}', $subject)
-                    ->setParam('{{hello}}', $locale->getText("emails.invitation.hello"))
-                    ->setParam('{{name}}', $user->getAttribute('name'))
-                    ->setParam('{{body}}', $locale->getText("emails.invitation.body"))
-                    ->setParam('{{redirect}}', $url)
-                    ->setParam('{{footer}}', $locale->getText("emails.invitation.footer"))
-                    ->setParam('{{thanks}}', $locale->getText("emails.invitation.thanks"))
-                    ->setParam('{{signature}}', $locale->getText("emails.invitation.signature"))
-                    ->setParam('{{project}}', $projectName)
-                    ->setParam('{{direction}}', $locale->getText('settings.direction'))
-                    ->setParam('{{bg-body}}', '#f7f7f7')
-                    ->setParam('{{bg-content}}', '#ffffff')
-                    ->setParam('{{text-content}}', '#000000');
+                    $body = $customTemplate['message'] ?? '';
+                    $subject = $customTemplate['subject'] ?? $subject;
+                }
 
-                $body = $body->render();
+                $mails
+                    ->setSmtpReplyTo($replyTo)
+                    ->setSmtpSenderEmail($senderEmail)
+                    ->setSmtpSenderName($senderName);
+
+                $emailVariables = [
+                    'owner' => $user->getAttribute('name'),
+                    'team' => $team->getAttribute('name'),
+                    'subject' => $subject,
+                    'hello' => $locale->getText("emails.invitation.hello"),
+                    'name' => $user->getAttribute('name'),
+                    'body' => $body,
+                    'redirect' => $url,
+                    'footer' => $locale->getText("emails.invitation.footer"),
+                    'thanks' => $locale->getText("emails.invitation.thanks"),
+                    'signature' => $locale->getText("emails.invitation.signature"),
+                    'project' => $projectName,
+                    'direction' => $locale->getText('settings.direction'),
+                    'bg-body' => '#f7f7f7',
+                    'bg-content' => '#ffffff',
+                    'text-content' => '#000000',
+                ];
 
                 $mails
                     ->setSubject($subject)
                     ->setBody($body)
-                    ->setFrom($from)
                     ->setRecipient($invitee->getAttribute('email'))
                     ->setName($invitee->getAttribute('name'))
+                    ->setVariables($emailVariables)
                     ->trigger()
                 ;
             } elseif (!empty($phone)) {

app/controllers/api/users.php

@@ -8,8 +8,8 @@ use Appwrite\Event\Delete;
 use Appwrite\Event\Event;
 use Appwrite\Network\Validator\Email;
 use Appwrite\Utopia\Database\Validator\CustomId;
-use Utopia\Database\Validator\Queries;
 use Appwrite\Utopia\Database\Validator\Queries\Identities;
+use Utopia\Database\Validator\Queries;
 use Appwrite\Utopia\Database\Validator\Queries\Users;
 use Utopia\Database\Validator\Query\Limit;
 use Utopia\Database\Validator\Query\Offset;
@@ -96,7 +96,8 @@ function createUser(string $hash, mixed $hashOptions, string $userId, ?string $e
             'sessions' => null,
             'tokens' => null,
             'memberships' => null,
-            'search' => implode(' ', [$userId, $email, $phone, $name])
+            'search' => implode(' ', [$userId, $email, $phone, $name]),
+            'accessedAt' => DateTime::now(),
         ]));
     } catch (Duplicate $th) {
         throw new Exception(Exception::USER_ALREADY_EXISTS);
@@ -114,6 +115,7 @@ App::post('/v1/users')
     ->label('scope', 'users.write')
     ->label('audits.event', 'user.create')
     ->label('audits.resource', 'user/{response.$id}')
+    ->label('usage.metric', 'users.{scope}.requests.create')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'create')
@@ -146,6 +148,7 @@ App::post('/v1/users/bcrypt')
     ->label('scope', 'users.write')
     ->label('audits.event', 'user.create')
     ->label('audits.resource', 'user/{response.$id}')
+    ->label('usage.metric', 'users.{scope}.requests.create')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'createBcryptUser')
@@ -176,6 +179,7 @@ App::post('/v1/users/md5')
     ->label('scope', 'users.write')
     ->label('audits.event', 'user.create')
     ->label('audits.resource', 'user/{response.$id}')
+    ->label('usage.metric', 'users.{scope}.requests.create')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'createMD5User')
@@ -206,6 +210,7 @@ App::post('/v1/users/argon2')
     ->label('scope', 'users.write')
     ->label('audits.event', 'user.create')
     ->label('audits.resource', 'user/{response.$id}')
+    ->label('usage.metric', 'users.{scope}.requests.create')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'createArgon2User')
@@ -236,6 +241,7 @@ App::post('/v1/users/sha')
     ->label('scope', 'users.write')
     ->label('audits.event', 'user.create')
     ->label('audits.resource', 'user/{response.$id}')
+    ->label('usage.metric', 'users.{scope}.requests.create')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'createSHAUser')
@@ -273,6 +279,7 @@ App::post('/v1/users/phpass')
     ->label('scope', 'users.write')
     ->label('audits.event', 'user.create')
     ->label('audits.resource', 'user/{response.$id}')
+    ->label('usage.metric', 'users.{scope}.requests.create')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'createPHPassUser')
@@ -303,6 +310,7 @@ App::post('/v1/users/scrypt')
     ->label('scope', 'users.write')
     ->label('audits.event', 'user.create')
     ->label('audits.resource', 'user/{response.$id}')
+    ->label('usage.metric', 'users.{scope}.requests.create')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'createScryptUser')
@@ -346,6 +354,7 @@ App::post('/v1/users/scrypt-modified')
     ->label('scope', 'users.write')
     ->label('audits.event', 'user.create')
     ->label('audits.resource', 'user/{response.$id}')
+    ->label('usage.metric', 'users.{scope}.requests.create')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'createScryptModifiedUser')
@@ -376,6 +385,7 @@ App::get('/v1/users')
     ->desc('List Users')
     ->groups(['api', 'users'])
     ->label('scope', 'users.read')
+    ->label('usage.metric', 'users.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'list')
@@ -424,6 +434,7 @@ App::get('/v1/users/:userId')
     ->desc('Get User')
     ->groups(['api', 'users'])
     ->label('scope', 'users.read')
+    ->label('usage.metric', 'users.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'get')
@@ -449,6 +460,7 @@ App::get('/v1/users/:userId/prefs')
     ->desc('Get User Preferences')
     ->groups(['api', 'users'])
     ->label('scope', 'users.read')
+    ->label('usage.metric', 'users.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'getPrefs')
@@ -476,6 +488,7 @@ App::get('/v1/users/:userId/sessions')
     ->desc('List User Sessions')
     ->groups(['api', 'users'])
     ->label('scope', 'users.read')
+    ->label('usage.metric', 'users.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'listSessions')
@@ -517,6 +530,7 @@ App::get('/v1/users/:userId/memberships')
     ->desc('List User Memberships')
     ->groups(['api', 'users'])
     ->label('scope', 'users.read')
+    ->label('usage.metric', 'users.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'listMemberships')
@@ -556,6 +570,7 @@ App::get('/v1/users/:userId/logs')
     ->desc('List User Logs')
     ->groups(['api', 'users'])
     ->label('scope', 'users.read')
+    ->label('usage.metric', 'users.{scope}.requests.read')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'listLogs')
@@ -690,6 +705,7 @@ App::patch('/v1/users/:userId/status')
     ->label('audits.event', 'user.update')
     ->label('audits.resource', 'user/{response.$id}')
     ->label('audits.userId', '{response.$id}')
+    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'updateStatus')
@@ -725,7 +741,6 @@ App::put('/v1/users/:userId/labels')
     ->label('scope', 'users.write')
     ->label('audits.event', 'user.update')
     ->label('audits.resource', 'user/{response.$id}')
-    ->label('audits.userId', '{response.$id}')
     ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
@@ -757,41 +772,6 @@ App::put('/v1/users/:userId/labels')
         $response->dynamic($user, Response::MODEL_USER);
     });
 
-App::patch('/v1/users/:userId/verification')
-    ->desc('Update Email Verification')
-    ->groups(['api', 'users'])
-    ->label('event', 'users.[userId].update.verification')
-    ->label('scope', 'users.write')
-    ->label('audits.event', 'verification.update')
-    ->label('audits.resource', 'user/{response.$id}')
-    ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
-    ->label('sdk.namespace', 'users')
-    ->label('sdk.method', 'updateEmailVerification')
-    ->label('sdk.description', '/docs/references/users/update-user-email-verification.md')
-    ->label('sdk.response.code', Response::STATUS_CODE_OK)
-    ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
-    ->label('sdk.response.model', Response::MODEL_USER)
-    ->param('userId', '', new UID(), 'User ID.')
-    ->param('emailVerification', false, new Boolean(), 'User email verification status.')
-    ->inject('response')
-    ->inject('dbForProject')
-    ->inject('events')
-    ->action(function (string $userId, bool $emailVerification, Response $response, Database $dbForProject, Event $events) {
-
-        $user = $dbForProject->getDocument('users', $userId);
-
-        if ($user->isEmpty()) {
-            throw new Exception(Exception::USER_NOT_FOUND);
-        }
-
-        $user = $dbForProject->updateDocument('users', $user->getId(), $user->setAttribute('emailVerification', $emailVerification));
-
-        $events
-            ->setParam('userId', $user->getId());
-
-        $response->dynamic($user, Response::MODEL_USER);
-    });
-
 App::patch('/v1/users/:userId/verification/phone')
     ->desc('Update Phone Verification')
     ->groups(['api', 'users'])
@@ -799,6 +779,7 @@ App::patch('/v1/users/:userId/verification/phone')
     ->label('scope', 'users.write')
     ->label('audits.event', 'verification.update')
     ->label('audits.resource', 'user/{response.$id}')
+    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'updatePhoneVerification')
@@ -835,6 +816,7 @@ App::patch('/v1/users/:userId/name')
     ->label('audits.event', 'user.update')
     ->label('audits.resource', 'user/{response.$id}')
     ->label('audits.userId', '{response.$id}')
+    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'updateName')
@@ -872,6 +854,7 @@ App::patch('/v1/users/:userId/password')
     ->label('audits.event', 'user.update')
     ->label('audits.resource', 'user/{response.$id}')
     ->label('audits.userId', '{response.$id}')
+    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'updatePassword')
@@ -936,6 +919,7 @@ App::patch('/v1/users/:userId/email')
     ->label('audits.event', 'user.update')
     ->label('audits.resource', 'user/{response.$id}')
     ->label('audits.userId', '{response.$id}')
+    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'updateEmail')
@@ -991,6 +975,7 @@ App::patch('/v1/users/:userId/phone')
     ->label('scope', 'users.write')
     ->label('audits.event', 'user.update')
     ->label('audits.resource', 'user/{response.$id}')
+    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'updatePhone')
@@ -1035,6 +1020,7 @@ App::patch('/v1/users/:userId/verification')
     ->label('audits.event', 'verification.update')
     ->label('audits.resource', 'user/{request.userId}')
     ->label('audits.userId', '{request.userId}')
+    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'updateEmailVerification')
@@ -1067,6 +1053,7 @@ App::patch('/v1/users/:userId/prefs')
     ->groups(['api', 'users'])
     ->label('event', 'users.[userId].update.prefs')
     ->label('scope', 'users.write')
+    ->label('usage.metric', 'users.{scope}.requests.update')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'updatePrefs')
@@ -1102,6 +1089,7 @@ App::delete('/v1/users/:userId/sessions/:sessionId')
     ->label('scope', 'users.write')
     ->label('audits.event', 'session.delete')
     ->label('audits.resource', 'user/{request.userId}')
+    ->label('usage.metric', 'sessions.{scope}.requests.delete')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'deleteSession')
@@ -1145,6 +1133,7 @@ App::delete('/v1/users/:userId/sessions')
     ->label('scope', 'users.write')
     ->label('audits.event', 'session.delete')
     ->label('audits.resource', 'user/{user.$id}')
+    ->label('usage.metric', 'sessions.{scope}.requests.delete')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'deleteSessions')
@@ -1187,6 +1176,7 @@ App::delete('/v1/users/:userId')
     ->label('scope', 'users.write')
     ->label('audits.event', 'user.delete')
     ->label('audits.resource', 'user/{request.userId}')
+    ->label('usage.metric', 'users.{scope}.requests.delete')
     ->label('sdk.auth', [APP_AUTH_TYPE_KEY])
     ->label('sdk.namespace', 'users')
     ->label('sdk.method', 'delete')
@@ -1265,59 +1255,96 @@ App::get('/v1/users/usage')
     ->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
     ->label('sdk.response.model', Response::MODEL_USAGE_USERS)
     ->param('range', '30d', new WhiteList(['24h', '7d', '30d', '90d'], true), 'Date range.', true)
+    ->param('provider', '', new WhiteList(\array_merge(['email', 'anonymous'], \array_map(fn ($value) => "oauth-" . $value, \array_keys(Config::getParam('providers', [])))), true), 'Provider Name.', true)
     ->inject('response')
     ->inject('dbForProject')
     ->inject('register')
-    ->action(function (string $range, Response $response, Database $dbForProject) {
+    ->action(function (string $range, string $provider, Response $response, Database $dbForProject) {
 
-        $periods = Config::getParam('usage', []);
-        $stats = $usage = [];
-        $days = $periods[$range];
-        $metrics = [
-            METRIC_USERS,
-            METRIC_SESSIONS,
-        ];
-
-        Authorization::skip(function () use ($dbForProject, $days, $metrics, &$stats) {
-            foreach ($metrics as $metric) {
-                $limit = $days['limit'];
-                $period = $days['period'];
-                $results = $dbForProject->find('stats', [
-                    Query::equal('period', [$period]),
-                    Query::equal('metric', [$metric]),
-                    Query::limit($limit),
-                    Query::orderDesc('time'),
-                ]);
-                $stats[$metric] = [];
-                foreach ($results as $result) {
-                    $stats[$metric][$result->getAttribute('time')] = [
-                        'value' => $result->getAttribute('value'),
-                    ];
-                }
-            }
-        });
-
-        $format = match ($days['period']) {
-            '1h' => 'Y-m-d\TH:00:00.000P',
-            '1d' => 'Y-m-d\T00:00:00.000P',
-        };
-
-    foreach ($metrics as $metric) {
-        $usage[$metric] = [];
-        $leap = time() - ($days['limit'] * $days['factor']);
-        while ($leap < time()) {
-            $leap += $days['factor'];
-            $formatDate = date($format, $leap);
-            $usage[$metric][] = [
-                'value' => $stats[$metric][$formatDate]['value'] ?? 0,
-                'date' => $formatDate,
+        $usage = [];
+        if (App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled') {
+            $periods = [
+                '24h' => [
+                    'period' => '1h',
+                    'limit' => 24,
+                ],
+                '7d' => [
+                    'period' => '1d',
+                    'limit' => 7,
+                ],
+                '30d' => [
+                    'period' => '1d',
+                    'limit' => 30,
+                ],
+                '90d' => [
+                    'period' => '1d',
+                    'limit' => 90,
+                ],
             ];
-        }
-    }
 
-        $response->dynamic(new Document([
-            'range' => $range,
-            'usersTotal'   => $usage[$metrics[0]],
-            'sessionsTotal' => $usage[$metrics[1]],
-        ]), Response::MODEL_USAGE_USERS);
+            $metrics = [
+                'users.$all.count.total',
+                'users.$all.requests.create',
+                'users.$all.requests.read',
+                'users.$all.requests.update',
+                'users.$all.requests.delete',
+                'sessions.$all.requests.create',
+                'sessions.$all.requests.delete',
+                "sessions.$provider.requests.create",
+            ];
+
+            $stats = [];
+
+            Authorization::skip(function () use ($dbForProject, $periods, $range, $metrics, &$stats) {
+                foreach ($metrics as $metric) {
+                    $limit = $periods[$range]['limit'];
+                    $period = $periods[$range]['period'];
+
+                    $requestDocs = $dbForProject->find('stats', [
+                        Query::equal('period', [$period]),
+                        Query::equal('metric', [$metric]),
+                        Query::limit($limit),
+                        Query::orderDesc('time'),
+                    ]);
+
+                    $stats[$metric] = [];
+                    foreach ($requestDocs as $requestDoc) {
+                        $stats[$metric][] = [
+                            'value' => $requestDoc->getAttribute('value'),
+                            'date' => $requestDoc->getAttribute('time'),
+                        ];
+                    }
+
+                    // backfill metrics with empty values for graphs
+                    $backfill = $limit - \count($requestDocs);
+                    while ($backfill > 0) {
+                        $last = $limit - $backfill - 1; // array index of last added metric
+                        $diff = match ($period) { // convert period to seconds for unix timestamp math
+                            '1h' => 3600,
+                            '1d' => 86400,
+                        };
+                        $stats[$metric][] = [
+                            'value' => 0,
+                            'date' => DateTime::formatTz(DateTime::addSeconds(new \DateTime($stats[$metric][$last]['date'] ?? null), -1 * $diff)),
+                        ];
+                        $backfill--;
+                    }
+                    $stats[$metric] = array_reverse($stats[$metric]);
+                }
+            });
+
+            $usage = new Document([
+                'range' => $range,
+                'usersCount' => $stats['users.$all.count.total'] ?? [],
+                'usersCreate' => $stats['users.$all.requests.create'] ?? [],
+                'usersRead' => $stats['users.$all.requests.read'] ?? [],
+                'usersUpdate' => $stats['users.$all.requests.update'] ?? [],
+                'usersDelete' => $stats['users.$all.requests.delete'] ?? [],
+                'sessionsCreate' => $stats['sessions.$all.requests.create'] ?? [],
+                'sessionsProviderCreate' => $stats["sessions.$provider.requests.create"] ?? [],
+                'sessionsDelete' => $stats['sessions.$all.requests.delete' ?? []]
+            ]);
+        }
+
+        $response->dynamic($usage, Response::MODEL_USAGE_USERS);
     });

app/controllers/general.php

@@ -8,6 +8,9 @@ use Utopia\Locale\Locale;
 use Utopia\Logger\Logger;
 use Utopia\Logger\Log;
 use Utopia\Logger\Log\User;
+use Swoole\Http\Request as SwooleRequest;
+use Utopia\Cache\Cache;
+use Utopia\Pools\Group;
 use Appwrite\Utopia\Request;
 use Appwrite\Utopia\Response;
 use Appwrite\Utopia\View;
@@ -22,6 +25,7 @@ use Appwrite\Utopia\Response\Filters\V12 as ResponseV12;
 use Appwrite\Utopia\Response\Filters\V13 as ResponseV13;
 use Appwrite\Utopia\Response\Filters\V14 as ResponseV14;
 use Appwrite\Utopia\Response\Filters\V15 as ResponseV15;
+use Appwrite\Utopia\Response\Filters\V16 as ResponseV16;
 use Utopia\CLI\Console;
 use Utopia\Database\Database;
 use Utopia\Database\DateTime;
@@ -33,6 +37,7 @@ use Appwrite\Utopia\Request\Filters\V12 as RequestV12;
 use Appwrite\Utopia\Request\Filters\V13 as RequestV13;
 use Appwrite\Utopia\Request\Filters\V14 as RequestV14;
 use Appwrite\Utopia\Request\Filters\V15 as RequestV15;
+use Appwrite\Utopia\Request\Filters\V16 as RequestV16;
 use Utopia\Validator\Text;
 use Utopia\Validator\WhiteList;
 
@@ -40,9 +45,136 @@ Config::setParam('domainVerification', false);
 Config::setParam('cookieDomain', 'localhost');
 Config::setParam('cookieSamesite', Response::COOKIE_SAMESITE_NONE);
 
+function router(App $utopia, Database $dbForConsole, SwooleRequest $swooleRequest, Request $request, Response $response)
+{
+    $utopia->getRoute()->label('error', __DIR__ . '/../views/general/error.phtml');
+
+    $host = $request->getHostname() ?? '';
+
+    $route = Authorization::skip(
+        fn () => $dbForConsole->find('rules', [
+            Query::equal('domain', [$host]),
+            Query::limit(1)
+        ])
+    )[0] ?? null;
+
+    if ($route === null) {
+        $mainDomain = App::getEnv('_APP_DOMAIN', '');
+
+        if ($mainDomain === 'localhost') {
+            throw new AppwriteException(AppwriteException::ROUTER_DOMAIN_NOT_CONFIGURED);
+        } else {
+            throw new AppwriteException(AppwriteException::ROUTER_HOST_NOT_FOUND);
+        }
+    }
+
+    $projectId = $route->getAttribute('projectId');
+    $project = Authorization::skip(
+        fn () => $dbForConsole->getDocument('projects', $projectId)
+    );
+    if (array_key_exists('proxy', $project->getAttribute('services', []))) {
+        $status = $project->getAttribute('services', [])['proxy'];
+        if (!$status) {
+            throw new AppwriteException(AppwriteException::GENERAL_SERVICE_DISABLED);
+        }
+    }
+
+    // Skip Appwrite Router for ACME challenge. Nessessary for certificate generation
+    $path = ($swooleRequest->server['request_uri'] ?? '/');
+    if (\str_starts_with($path, '/.well-known/acme-challenge')) {
+        return false;
+    }
+
+    $type = $route->getAttribute('resourceType');
+
+    if ($type === 'function') {
+        $functionId = $route->getAttribute('resourceId');
+        $projectId = $route->getAttribute('projectId');
+
+        $path = ($swooleRequest->server['request_uri'] ?? '/');
+        $query = ($swooleRequest->server['query_string'] ?? '');
+        if (!empty($query)) {
+            $path .= '?' . $query;
+        }
+
+        $body = \json_encode([
+            'async' => false,
+            'body' => $swooleRequest->getContent() ?? '',
+            'method' => $swooleRequest->server['request_method'],
+            'path' => $path,
+            'headers' => $swooleRequest->header
+        ]);
+
+        $headers = [
+            'Content-Type: application/json',
+            'Content-Length: ' . \strlen($body),
+            'X-Appwrite-Project: ' . $projectId
+        ];
+
+        $ch = \curl_init();
+        \curl_setopt($ch, CURLOPT_URL, "http://localhost/v1/functions/{$functionId}/executions");
+        \curl_setopt($ch, CURLOPT_CUSTOMREQUEST, "POST");
+        \curl_setopt($ch, CURLOPT_POSTFIELDS, $body);
+        \curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
+        \curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
+        // \curl_setopt($ch, CURLOPT_HEADER, true);
+        \curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10);
+
+        $executionResponse = \curl_exec($ch);
+        $statusCode = \curl_getinfo($ch, CURLINFO_HTTP_CODE);
+        $error = \curl_error($ch);
+        $errNo = \curl_errno($ch);
+
+        \curl_close($ch);
+
+        if ($errNo !== 0) {
+            throw new AppwriteException(AppwriteException::GENERAL_ARGUMENT_INVALID, "Internal error: " . $error);
+        }
+
+        if ($statusCode >= 400) {
+            $error = \json_decode($executionResponse, true)['message'];
+            throw new AppwriteException(AppwriteException::GENERAL_ARGUMENT_INVALID, "Execution error: " . $error);
+        }
+
+        $execution = \json_decode($executionResponse, true);
+
+        $contentType = 'text/plain';
+        foreach ($execution['responseHeaders'] as $header) {
+            if (\strtolower($header['name']) === 'content-type') {
+                $contentType = $header['value'];
+            }
+
+            $response->setHeader($header['name'], $header['value']);
+        }
+
+        $body = $execution['responseBody'] ?? '';
+
+        $encodingKey = \array_search('x-open-runtimes-encoding', \array_column($execution['responseHeaders'], 'name'));
+        if ($encodingKey !== false) {
+            if (($execution['responseHeaders'][$encodingKey]['value'] ?? '') === 'base64') {
+                $body = \base64_decode($body);
+            }
+        }
+
+        $response
+            ->setContentType($contentType)
+            ->setStatusCode($execution['responseStatusCode'] ?? 200)
+            ->send($body);
+
+        return true;
+    } elseif ($type === 'api') {
+        return false;
+    } else {
+        throw new AppwriteException(AppwriteException::GENERAL_SERVER_ERROR, 'Unknown resource type ' . $type);
+    }
+
+    return false;
+}
+
 App::init()
-    ->groups(['api'])
+    ->groups(['api', 'web'])
     ->inject('utopia')
+    ->inject('swooleRequest')
     ->inject('request')
     ->inject('response')
     ->inject('console')
@@ -53,13 +185,30 @@ App::init()
     ->inject('localeCodes')
     ->inject('clients')
     ->inject('servers')
-    ->action(function (App $utopia, Request $request, Response $response, Document $console, Document $project, Database $dbForConsole, Document $user, Locale $locale, array $localeCodes, array $clients, array $servers) {
+    ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Document $console, Document $project, Database $dbForConsole, Document $user, Locale $locale, array $localeCodes, array $clients, array $servers) {
+        /*
+        * Appwrite Router
+        */
+
+        $host = $request->getHostname() ?? '';
+        $mainDomain = App::getEnv('_APP_DOMAIN', '');
+        // Only run Router when external domain
+        if ($host !== $mainDomain && $host !== 'localhost' && $host !== APP_HOSTNAME_INTERNAL) {
+            if (router($utopia, $dbForConsole, $swooleRequest, $request, $response)) {
+                return;
+            }
+        }
+
         /*
         * Request format
         */
-        $route = $utopia->match($request);
+        $route = $utopia->getRoute();
         Request::setRoute($route);
 
+        if ($route === null) {
+            return $response->setStatusCode(404)->send('Not Found');
+        }
+
         $requestFormat = $request->getHeader('x-appwrite-response-format', App::getEnv('_APP_SYSTEM_RESPONSE_FORMAT', ''));
         if ($requestFormat) {
             switch ($requestFormat) {
@@ -75,6 +224,9 @@ App::init()
                 case version_compare($requestFormat, '0.15.3', '<'):
                     Request::setFilter(new RequestV15());
                     break;
+                case version_compare($requestFormat, '1.4.0', '<'):
+                    Request::setFilter(new RequestV16());
+                    break;
                 default:
                     Request::setFilter(null);
             }
@@ -82,60 +234,6 @@ App::init()
             Request::setFilter(null);
         }
 
-        $domain = $request->getHostname();
-        $domains = Config::getParam('domains', []);
-        if (!array_key_exists($domain, $domains)) {
-            $domain = new Domain(!empty($domain) ? $domain : '');
-
-            if (empty($domain->get()) || !$domain->isKnown() || $domain->isTest()) {
-                $domains[$domain->get()] = false;
-                Console::warning($domain->get() . ' is not a publicly accessible domain. Skipping SSL certificate generation.');
-            } elseif (str_starts_with($request->getURI(), '/.well-known/acme-challenge')) {
-                Console::warning('Skipping SSL certificates generation on ACME challenge.');
-            } else {
-                Authorization::disable();
-
-                $envDomain = App::getEnv('_APP_DOMAIN', '');
-                $mainDomain = null;
-                if (!empty($envDomain) && $envDomain !== 'localhost') {
-                    $mainDomain = $envDomain;
-                } else {
-                    $domainDocument = $dbForConsole->findOne('domains', [Query::orderAsc('_id')]);
-                    $mainDomain = $domainDocument ? $domainDocument->getAttribute('domain') : $domain->get();
-                }
-
-                if ($mainDomain !== $domain->get()) {
-                    Console::warning($domain->get() . ' is not a main domain. Skipping SSL certificate generation.');
-                } else {
-                    $domainDocument = $dbForConsole->findOne('domains', [
-                        Query::equal('domain', [$domain->get()])
-                    ]);
-
-                    if (!$domainDocument) {
-                        $domainDocument = new Document([
-                            'domain' => $domain->get(),
-                            'tld' => $domain->getSuffix(),
-                            'registerable' => $domain->getRegisterable(),
-                            'verification' => false,
-                            'certificateId' => null,
-                        ]);
-
-                        $domainDocument = $dbForConsole->createDocument('domains', $domainDocument);
-
-                        Console::info('Issuing a TLS certificate for the main domain (' . $domain->get() . ') in a few seconds...');
-
-                        (new Certificate())
-                            ->setDomain($domainDocument)
-                            ->trigger();
-                    }
-                }
-                $domains[$domain->get()] = true;
-
-                Authorization::reset(); // ensure authorization is re-enabled
-            }
-            Config::setParam('domains', $domains);
-        }
-
         $localeParam = (string) $request->getParam('locale', $request->getHeader('x-appwrite-locale', ''));
         if (\in_array($localeParam, $localeCodes)) {
             $locale->setDefault($localeParam);
@@ -172,7 +270,7 @@ App::init()
         Config::setParam(
             'domainVerification',
             ($selfDomain->getRegisterable() === $endDomain->getRegisterable()) &&
-            $endDomain->getRegisterable() !== ''
+                $endDomain->getRegisterable() !== ''
         );
 
         $isLocalHost = $request->getHostname() === 'localhost' || $request->getHostname() === 'localhost:' . $request->getPort();
@@ -212,6 +310,9 @@ App::init()
                 case version_compare($responseFormat, '0.15.3', '<='):
                     Response::setFilter(new ResponseV15());
                     break;
+                case version_compare($responseFormat, '1.4.0', '<'):
+                    Response::setFilter(new ResponseV16());
+                    break;
                 default:
                     Response::setFilter(null);
             }
@@ -226,7 +327,7 @@ App::init()
         * @see https://www.owasp.org/index.php/List_of_useful_HTTP_headers
         */
         if (App::getEnv('_APP_OPTIONS_FORCE_HTTPS', 'disabled') === 'enabled') { // Force HTTPS
-            if ($request->getProtocol() !== 'https') {
+            if ($request->getProtocol() !== 'https' && ($swooleRequest->header['host'] ?? '') !== 'localhost' && ($swooleRequest->header['host'] ?? '') !== APP_HOSTNAME_INTERNAL) { // Localhost allowed for proxy, APP_HOSTNAME_INTERNAL allowed for migrations
                 if ($request->getMethod() !== Request::METHOD_GET) {
                     throw new AppwriteException(AppwriteException::GENERAL_PROTOCOL_UNSUPPORTED, 'Method unsupported over HTTP.');
                 }
@@ -246,8 +347,7 @@ App::init()
             ->addHeader('Access-Control-Allow-Headers', 'Origin, Cookie, Set-Cookie, X-Requested-With, Content-Type, Access-Control-Allow-Origin, Access-Control-Request-Headers, Accept, X-Appwrite-Project, X-Appwrite-Key, X-Appwrite-Locale, X-Appwrite-Mode, X-Appwrite-JWT, X-Appwrite-Response-Format, X-SDK-Version, X-SDK-Name, X-SDK-Language, X-SDK-Platform, X-SDK-GraphQL, X-Appwrite-ID, X-Appwrite-Timestamp, Content-Range, Range, Cache-Control, Expires, Pragma')
             ->addHeader('Access-Control-Expose-Headers', 'X-Fallback-Cookies')
             ->addHeader('Access-Control-Allow-Origin', $refDomain)
-            ->addHeader('Access-Control-Allow-Credentials', 'true')
-        ;
+            ->addHeader('Access-Control-Allow-Credentials', 'true');
 
         /*
         * Validate Client Domain - Check to avoid CSRF attack
@@ -274,7 +374,7 @@ App::init()
             : Role::users()->toString();
 
         // Add user roles
-        $memberships = $user->find('teamId', $project->getAttribute('teamId', null), 'memberships');
+        $memberships = $user->find('teamId', $project->getAttribute('teamId'), 'memberships');
 
         if ($memberships) {
             foreach ($memberships->getAttribute('roles', []) as $memberRole) {
@@ -320,7 +420,7 @@ App::init()
 
                 $expire = $key->getAttribute('expire');
                 if (!empty($expire) && $expire < DateTime::formatTz(DateTime::now())) {
-                    throw new AppwriteException(AppwriteException:: PROJECT_KEY_EXPIRED);
+                    throw new AppwriteException(AppwriteException::PROJECT_KEY_EXPIRED);
                 }
 
                 Authorization::setRole(Auth::USER_ROLE_APPS);
@@ -385,9 +485,23 @@ App::init()
     });
 
 App::options()
+    ->inject('utopia')
+    ->inject('swooleRequest')
     ->inject('request')
     ->inject('response')
-    ->action(function (Request $request, Response $response) {
+    ->inject('dbForConsole')
+    ->action(function (App $utopia, SwooleRequest $swooleRequest, Request $request, Response $response, Database $dbForConsole) {
+        /*
+        * Appwrite Router
+        */
+        $host = $request->getHostname() ?? '';
+        $mainDomain = App::getEnv('_APP_DOMAIN', '');
+        // Only run Router when external domain
+        if ($host !== $mainDomain && $host !== 'localhost' && $host !== APP_HOSTNAME_INTERNAL) {
+            if (router($utopia, $dbForConsole, $swooleRequest, $request, $response)) {
+                return;
+            }
+        }
 
         $origin = $request->getOrigin();
 
@@ -412,7 +526,7 @@ App::error()
     ->action(function (Throwable $error, App $utopia, Request $request, Response $response, Document $project, ?Logger $logger, array $loggerBreadcrumbs) {
 
         $version = App::getEnv('_APP_VERSION', 'UNKNOWN');
-        $route = $utopia->match($request);
+        $route = $utopia->getRoute();
 
         if ($logger) {
             if ($error->getCode() >= 500 || $error->getCode() === 0) {
@@ -548,8 +662,7 @@ App::error()
             ->addHeader('Cache-Control', 'no-cache, no-store, must-revalidate')
             ->addHeader('Expires', '0')
             ->addHeader('Pragma', 'no-cache')
-            ->setStatusCode($code)
-        ;
+            ->setStatusCode($code);
 
         $template = ($route) ? $route->getLabel('error', null) : null;
 
@@ -564,8 +677,7 @@ App::error()
                 ->setParam('message', $error->getMessage())
                 ->setParam('type', $type)
                 ->setParam('code', $code)
-                ->setParam('trace', $trace)
-            ;
+                ->setParam('trace', $trace);
 
             $response->html($layout->render());
         }
@@ -649,6 +761,13 @@ App::get('/.well-known/acme-challenge/*')
 include_once __DIR__ . '/shared/api.php';
 include_once __DIR__ . '/shared/api/auth.php';
 
+App::wildcard()
+    ->groups(['api'])
+    ->label('scope', 'global')
+    ->action(function () {
+        throw new AppwriteException(AppwriteException::GENERAL_ROUTE_NOT_FOUND);
+    });
+
 foreach (Config::getParam('services', []) as $service) {
     include_once $service['controller'];
 }

app/controllers/mock.php

@@ -198,35 +198,6 @@ App::delete('/v1/mock/tests/bar')
     ->action(function ($required, $default, $z) {
     });
 
-/** Endpoint to test if required headers are sent from the SDK */
-App::get('/v1/mock/tests/general/headers')
-    ->desc('Get headers')
-    ->groups(['mock'])
-    ->label('scope', 'public')
-    ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
-    ->label('sdk.namespace', 'general')
-    ->label('sdk.method', 'headers')
-    ->label('sdk.description', 'Return headers from the request')
-    ->label('sdk.response.code', Response::STATUS_CODE_OK)
-    ->label('sdk.response.model', Response::MODEL_MOCK)
-    ->label('sdk.mock', true)
-    ->inject('request')
-    ->inject('response')
-    ->action(function (Request $request, Response $response) {
-        $res = [
-            'x-sdk-name' => $request->getHeader('x-sdk-name'),
-            'x-sdk-platform' => $request->getHeader('x-sdk-platform'),
-            'x-sdk-language' => $request->getHeader('x-sdk-language'),
-            'x-sdk-version' => $request->getHeader('x-sdk-version'),
-        ];
-        $res = array_map(function ($key, $value) {
-            return $key . ': ' . $value;
-        }, array_keys($res), $res);
-        $res = implode("; ", $res);
-
-        $response->dynamic(new Document(['result' => $res]), Response::MODEL_MOCK);
-    });
-
 App::get('/v1/mock/tests/general/download')
     ->desc('Download File')
     ->groups(['mock'])
@@ -443,35 +414,6 @@ App::post('/v1/mock/tests/general/nullable')
     ->action(function (string $required, string $nullable, ?string $optional) {
     });
 
-/** Endpoint to test if required headers are sent from the SDK */
-App::get('/v1/mock/tests/general/headers')
-    ->desc('Get headers')
-    ->groups(['mock'])
-    ->label('scope', 'public')
-    ->label('sdk.auth', [APP_AUTH_TYPE_SESSION, APP_AUTH_TYPE_KEY, APP_AUTH_TYPE_JWT])
-    ->label('sdk.namespace', 'general')
-    ->label('sdk.method', 'headers')
-    ->label('sdk.description', 'Return headers from the request')
-    ->label('sdk.response.code', Response::STATUS_CODE_OK)
-    ->label('sdk.response.model', Response::MODEL_MOCK)
-    ->label('sdk.mock', true)
-    ->inject('request')
-    ->inject('response')
-    ->action(function (Request $request, Response $response) {
-        $res = [
-            'x-sdk-name' => $request->getHeader('x-sdk-name'),
-            'x-sdk-platform' => $request->getHeader('x-sdk-platform'),
-            'x-sdk-language' => $request->getHeader('x-sdk-language'),
-            'x-sdk-version' => $request->getHeader('x-sdk-version'),
-        ];
-        $res = array_map(function ($key, $value) {
-            return $key . ': ' . $value;
-        }, array_keys($res), $res);
-        $res = implode("; ", $res);
-
-        $response->dynamic(new Document(['result' => $res]), Response::MODEL_MOCK);
-    });
-
 App::get('/v1/mock/tests/general/400-error')
     ->desc('400 Error')
     ->groups(['mock'])
@@ -643,7 +585,7 @@ App::shutdown()
     ->action(function (App $utopia, Response $response, Request $request) {
 
         $result = [];
-        $route  = $utopia->match($request);
+        $route  = $utopia->getRoute();
         $path   = APP_STORAGE_CACHE . '/tests.json';
         $tests  = (\file_exists($path)) ? \json_decode(\file_get_contents($path), true) : [];
 

app/controllers/shared/api.php

@@ -8,8 +8,8 @@ use Appwrite\Event\Event;
 use Appwrite\Event\Func;
 use Appwrite\Event\Mail;
 use Appwrite\Extend\Exception;
-use Appwrite\Event\Usage;
 use Appwrite\Messaging\Adapter\Realtime;
+use Appwrite\Usage\Stats;
 use Appwrite\Utopia\Response;
 use Appwrite\Utopia\Request;
 use Utopia\App;
@@ -48,99 +48,43 @@ $parseLabel = function (string $label, array $responsePayload, array $requestPar
     return $label;
 };
 
-$databaseListener = function (string $event, Document $document, Document $project, Usage $queueForUsage, Database $dbForProject) {
-
-    $value = 1;
+$databaseListener = function (string $event, Document $document, Stats $usage) {
+    $multiplier = 1;
     if ($event === Database::EVENT_DOCUMENT_DELETE) {
-        $value = -1;
+        $multiplier = -1;
     }
 
-    switch (true) {
-        case $document->getCollection() === 'teams':
-            $queueForUsage
-                ->addMetric(METRIC_TEAMS, $value); // per project
+    $collection = $document->getCollection();
+    switch ($collection) {
+        case 'users':
+            $usage->setParam('users.{scope}.count.total', 1 * $multiplier);
             break;
-        case $document->getCollection() === 'users':
-            $queueForUsage
-                ->addMetric(METRIC_USERS, $value); // per project
-            if ($event === Database::EVENT_DOCUMENT_DELETE) {
-                $queueForUsage
-                    ->addReduce($document);
-            }
+        case 'databases':
+            $usage->setParam('databases.{scope}.count.total', 1 * $multiplier);
             break;
-        case $document->getCollection() === 'sessions': // sessions
-            $queueForUsage
-                ->addMetric(METRIC_SESSIONS, $value); //per project
+        case 'buckets':
+            $usage->setParam('buckets.{scope}.count.total', 1 * $multiplier);
             break;
-        case $document->getCollection() === 'databases': // databases
-            $queueForUsage
-                ->addMetric(METRIC_DATABASES, $value); // per project
-
-            if ($event === Database::EVENT_DOCUMENT_DELETE) {
-                $queueForUsage
-                    ->addReduce($document);
-            }
-            break;
-        case str_starts_with($document->getCollection(), 'database_') && !str_contains($document->getCollection(), 'collection'): //collections
-            $parts = explode('_', $document->getCollection());
-            $databaseInternalId = $parts[1] ?? 0;
-            $queueForUsage
-                ->addMetric(METRIC_COLLECTIONS, $value) // per project
-                ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_COLLECTIONS), $value) // per database
-            ;
-
-            if ($event === Database::EVENT_DOCUMENT_DELETE) {
-                $queueForUsage
-                    ->addReduce($document);
-            }
-            break;
-        case str_starts_with($document->getCollection(), 'database_') && str_contains($document->getCollection(), '_collection_'): //documents
-            $parts = explode('_', $document->getCollection());
-            $databaseInternalId   = $parts[1] ?? 0;
-            $collectionInternalId = $parts[3] ?? 0;
-            $queueForUsage
-                ->addMetric(METRIC_DOCUMENTS, $value)  // per project
-                ->addMetric(str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_DOCUMENTS), $value) // per database
-                ->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $collectionInternalId], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS), $value);  // per collection
-            break;
-        case $document->getCollection() === 'buckets': //buckets
-            $queueForUsage
-                ->addMetric(METRIC_BUCKETS, $value); // per project
-            if ($event === Database::EVENT_DOCUMENT_DELETE) {
-                $queueForUsage
-                    ->addReduce($document);
-            }
-            break;
-        case str_starts_with($document->getCollection(), 'bucket_'): // files
-            $queueForUsage
-                ->addMetric(METRIC_FILES, $value) // per project
-                ->addMetric(METRIC_FILES_STORAGE, $document->getAttribute('sizeOriginal') * $value) // per project
-                ->addMetric(str_replace('{bucketInternalId}', $document->getAttribute('bucketInternalId'), METRIC_BUCKET_ID_FILES), $value) // per bucket
-                ->addMetric(str_replace('{bucketInternalId}', $document->getAttribute('bucketInternalId'), METRIC_BUCKET_ID_FILES_STORAGE), $document->getAttribute('sizeOriginal') * $value); // per bucket
-            break;
-        case $document->getCollection() === 'functions':
-            $queueForUsage
-                ->addMetric(METRIC_FUNCTIONS, $value); // per project
-
-            if ($event === Database::EVENT_DOCUMENT_DELETE) {
-                $queueForUsage
-                    ->addReduce($document);
-            }
-            break;
-        case $document->getCollection() === 'deployments':
-            $queueForUsage
-                ->addMetric(METRIC_DEPLOYMENTS, $value) // per project
-                ->addMetric(METRIC_DEPLOYMENTS_STORAGE, $document->getAttribute('size') * $value) // per project
-                ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$document->getAttribute('resourceType'), $document->getAttribute('resourceInternalId')], METRIC_FUNCTION_ID_DEPLOYMENTS), $value)// per function
-                ->addMetric(str_replace(['{resourceType}', '{resourceInternalId}'], [$document->getAttribute('resourceType'), $document->getAttribute('resourceInternalId')], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE), $document->getAttribute('size') * $value);// per function
-
-            break;
-        case $document->getCollection() === 'executions':
-            $queueForUsage
-                ->addMetric(METRIC_EXECUTIONS, $value) // per project
-                ->addMetric(str_replace('{functionInternalId}', $document->getAttribute('functionInternalId'), METRIC_FUNCTION_ID_EXECUTIONS), $value);// per function
+        case 'deployments':
+            $usage->setParam('deployments.{scope}.storage.size', $document->getAttribute('size') * $multiplier);
             break;
         default:
+            if (strpos($collection, 'bucket_') === 0) {
+                $usage
+                    ->setParam('bucketId', $document->getAttribute('bucketId'))
+                    ->setParam('files.{scope}.storage.size', $document->getAttribute('sizeOriginal') * $multiplier)
+                    ->setParam('files.{scope}.count.total', 1 * $multiplier);
+            } elseif (strpos($collection, 'database_') === 0) {
+                $usage
+                    ->setParam('databaseId', $document->getAttribute('databaseId'));
+                if (strpos($collection, '_collection_') !== false) {
+                    $usage
+                        ->setParam('collectionId', $document->getAttribute('$collectionId'))
+                        ->setParam('documents.{scope}.count.total', 1 * $multiplier);
+                } else {
+                    $usage->setParam('collections.{scope}.count.total', 1 * $multiplier);
+                }
+            }
             break;
     }
 };
@@ -157,12 +101,12 @@ App::init()
     ->inject('deletes')
     ->inject('database')
     ->inject('dbForProject')
-    ->inject('queueForUsage')
     ->inject('mode')
     ->inject('mails')
-    ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $events, Audit $audits, Delete $deletes, EventDatabase $database, Database $dbForProject, Usage $queueForUsage, string $mode, Mail $mails) use ($databaseListener) {
+    ->inject('usage')
+    ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $events, Audit $audits, Delete $deletes, EventDatabase $database, Database $dbForProject, string $mode, Mail $mails, Stats $usage) use ($databaseListener) {
 
-        $route = $utopia->match($request);
+        $route = $utopia->getRoute();
 
         if ($project->isEmpty() && $route->getLabel('abuse-limit', 0) > 0) { // Abuse limit requires an active project scope
             throw new Exception(Exception::PROJECT_UNKNOWN);
@@ -242,25 +186,19 @@ App::init()
             ->setProject($project)
             ->setUser($user);
 
-        $smtp = $project->getAttribute('smtp', []);
-        if (!empty($smtp) && ($smtp['enabled'] ?? false)) {
-            $mails
-                ->setSmtpHost($smtp['host'] ?? '')
-                ->setSmtpPort($smtp['port'] ?? 25)
-                ->setSmtpUsername($smtp['username'] ?? '')
-                ->setSmtpPassword($smtp['password'] ?? '')
-                ->setSmtpSenderEmail($smtp['sender'] ?? '')
-                ->setSmtpReplyTo($smtp['replyTo'] ?? '');
-        }
+        $usage
+            ->setParam('projectInternalId', $project->getInternalId())
+            ->setParam('projectId', $project->getId())
+            ->setParam('project.{scope}.network.requests', 1)
+            ->setParam('httpMethod', $request->getMethod())
+            ->setParam('project.{scope}.network.inbound', 0)
+            ->setParam('project.{scope}.network.outbound', 0);
 
         $deletes->setProject($project);
         $database->setProject($project);
 
-        $calculateUsage = fn ($event, Document $document) => $databaseListener($event, $document, $project, $queueForUsage, $dbForProject);
-
-        $dbForProject
-            ->on(Database::EVENT_DOCUMENT_CREATE, 'calculate-usage', $calculateUsage)
-            ->on(Database::EVENT_DOCUMENT_DELETE, 'calculate-usage', $calculateUsage);
+        $dbForProject->on(Database::EVENT_DOCUMENT_CREATE, 'calculate-usage', fn ($event, Document $document) => $databaseListener($event, $document, $usage));
+        $dbForProject->on(Database::EVENT_DOCUMENT_DELETE, 'calculate-usage', fn ($event, Document $document) => $databaseListener($event, $document, $usage));
 
         $useCache = $route->getLabel('cache', false);
 
@@ -331,7 +269,7 @@ App::init()
     ->inject('project')
     ->action(function (App $utopia, Request $request, Document $project) {
 
-        $route = $utopia->match($request);
+        $route = $utopia->getRoute();
 
         $isPrivilegedUser = Auth::isPrivilegedUser(Authorization::getRoles());
         $isAppUser = Auth::isAppUser(Authorization::getRoles());
@@ -426,14 +364,14 @@ App::shutdown()
     ->inject('user')
     ->inject('events')
     ->inject('audits')
+    ->inject('usage')
     ->inject('deletes')
     ->inject('database')
     ->inject('dbForProject')
     ->inject('queueForFunctions')
-    ->inject('queueForUsage')
     ->inject('mode')
     ->inject('dbForConsole')
-    ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $events, Audit $audits, Delete $deletes, EventDatabase $database, Database $dbForProject, Func $queueForFunctions, Usage $queueForUsage, string $mode, Database $dbForConsole) use ($parseLabel) {
+    ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $events, Audit $audits, Stats $usage, Delete $deletes, EventDatabase $database, Database $dbForProject, Func $queueForFunctions, string $mode, Database $dbForConsole) use ($parseLabel) {
 
         $responsePayload = $response->getPayload();
 
@@ -492,7 +430,7 @@ App::shutdown()
             }
         }
 
-        $route = $utopia->match($request);
+        $route = $utopia->getRoute();
         $requestParams = $route->getParamsValues();
 
         /**
@@ -586,48 +524,35 @@ App::shutdown()
             }
         }
 
+        if (
+            App::getEnv('_APP_USAGE_STATS', 'enabled') == 'enabled'
+            && $project->getId()
+            && !empty($route->getLabel('sdk.namespace', null))
+        ) { // Don't calculate console usage on admin mode
+            $metric = $route->getLabel('usage.metric', '');
+            $usageParams = $route->getLabel('usage.params', []);
 
-
-        if ($project->getId() !== 'console') {
-            if ($mode !== APP_MODE_ADMIN) {
-                $fileSize = 0;
-                $file = $request->getFiles('file');
-                if (!empty($file)) {
-                    $fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
-                }
-
-                $queueForUsage
-                    ->addMetric(METRIC_NETWORK_REQUESTS, 1)
-                    ->addMetric(METRIC_NETWORK_INBOUND, $request->getSize() + $fileSize)
-                    ->addMetric(METRIC_NETWORK_OUTBOUND, $response->getSize());
-            }
-
-            $queueForUsage
-                ->setProject($project)
-                ->trigger();
-        }
-
-        /**
-         * Update user last activity
-         */
-        if (!$user->isEmpty()) {
-            $accessedAt = $user->getAttribute('accessedAt', '');
-            if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_USER_ACCCESS)) > $accessedAt) {
-                $user->setAttribute('accessedAt', DateTime::now());
-
-                if (APP_MODE_ADMIN !== $mode) {
-                    $dbForProject->updateDocument('users', $user->getId(), $user);
-                } else {
-                    $dbForConsole->updateDocument('users', $user->getId(), $user);
+            if (!empty($metric)) {
+                $usage->setParam($metric, 1);
+                foreach ($usageParams as $param) {
+                    $param = $parseLabel($param, $responsePayload, $requestParams, $user);
+                    $parts = explode(':', $param);
+                    if (count($parts) != 2) {
+                        throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Usage params not properly set');
+                    }
+                    $usage->setParam($parts[0], $parts[1]);
                 }
             }
-        }
-    });
 
-App::init()
-    ->groups(['usage'])
-    ->action(function () {
-        if (App::getEnv('_APP_USAGE_STATS', 'enabled') !== 'enabled') {
-            throw new Exception(Exception::GENERAL_USAGE_DISABLED);
+            $fileSize = 0;
+            $file = $request->getFiles('file');
+            if (!empty($file)) {
+                $fileSize = (\is_array($file['size']) && isset($file['size'][0])) ? $file['size'][0] : $file['size'];
+            }
+
+            $usage
+                ->setParam('project.{scope}.network.inbound', $request->getSize() + $fileSize)
+                ->setParam('project.{scope}.network.outbound', $response->getSize())
+                ->submit();
         }
     });

app/controllers/web/console.php

@@ -1,5 +1,6 @@
 <?php
 
+use Appwrite\Extend\Exception;
 use Appwrite\Utopia\Request;
 use Appwrite\Utopia\Response;
 use Utopia\App;
@@ -30,6 +31,13 @@ App::get('/console/*')
     ->inject('request')
     ->inject('response')
     ->action(function (Request $request, Response $response) {
+        // Serve static files (console) only for main domain
+        $host = $request->getHostname() ?? '';
+        $mainDomain = App::getEnv('_APP_DOMAIN', '');
+        if ($host !== $mainDomain && $host !== 'localhost' && $host !== APP_HOSTNAME_INTERNAL) {
+            throw new Exception(Exception::GENERAL_ROUTE_NOT_FOUND);
+        }
+
         $fallback = file_get_contents(__DIR__ . '/../../../console/index.html');
 
         // Card SSR

app/controllers/web/home.php

@@ -6,7 +6,7 @@ use Utopia\Config\Config;
 
 App::get('/versions')
     ->desc('Get Version')
-    ->groups(['home'])
+    ->groups(['home', 'web'])
     ->label('scope', 'public')
     ->inject('response')
     ->action(function (Response $response) {

app/http.php

@@ -61,8 +61,9 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) {
     $app = new App('UTC');
 
     go(function () use ($register, $app) {
-        $pools = $register->get('pools'); /** @var Group $pools */
-        App::setResource('pools', fn() => $pools);
+        $pools = $register->get('pools');
+        /** @var Group $pools */
+        App::setResource('pools', fn () => $pools);
 
         // wait for database to be ready
         $attempts = 0;
@@ -72,7 +73,8 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) {
         do {
             try {
                 $attempts++;
-                $dbForConsole = $app->getResource('dbForConsole'); /** @var Utopia\Database\Database $dbForConsole */
+                $dbForConsole = $app->getResource('dbForConsole');
+                /** @var Utopia\Database\Database $dbForConsole */
                 break; // leave the do-while if successful
             } catch (\Exception $e) {
                 Console::warning("Database not ready. Retrying connection ({$attempts})...");
@@ -221,25 +223,33 @@ $http->on('start', function (Server $http) use ($payloadSize, $register) {
 });
 
 $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swooleResponse) use ($register) {
+    App::setResource('swooleRequest', fn () => $swooleRequest);
+    App::setResource('swooleResponse', fn () => $swooleResponse);
+
     $request = new Request($swooleRequest);
     $response = new Response($swooleResponse);
 
-    if (Files::isFileLoaded($request->getURI())) {
-        $time = (60 * 60 * 24 * 365 * 2); // 45 days cache
+    // Serve static files (console) only for main domain
+    $host = $request->getHostname() ?? '';
+    $mainDomain = App::getEnv('_APP_DOMAIN', '');
+    if ($host === $mainDomain || $host === 'localhost') {
+        if (Files::isFileLoaded($request->getURI())) {
+            $time = (60 * 60 * 24 * 365 * 2); // 45 days cache
 
-        $response
-            ->setContentType(Files::getFileMimeType($request->getURI()))
-            ->addHeader('Cache-Control', 'public, max-age=' . $time)
-            ->addHeader('Expires', \date('D, d M Y H:i:s', \time() + $time) . ' GMT') // 45 days cache
-            ->send(Files::getFileContents($request->getURI()));
+            $response
+                ->setContentType(Files::getFileMimeType($request->getURI()))
+                ->addHeader('Cache-Control', 'public, max-age=' . $time)
+                ->addHeader('Expires', \date('D, d M Y H:i:s', \time() + $time) . ' GMT') // 45 days cache
+                ->send(Files::getFileContents($request->getURI()));
 
-        return;
+            return;
+        }
     }
 
     $app = new App('UTC');
 
     $pools = $register->get('pools');
-    App::setResource('pools', fn() => $pools);
+    App::setResource('pools', fn () => $pools);
 
     try {
         Authorization::cleanRoles();
@@ -259,7 +269,7 @@ $http->on('request', function (SwooleRequest $swooleRequest, SwooleResponse $swo
             }
 
             $loggerBreadcrumbs = $app->getResource("loggerBreadcrumbs");
-            $route = $app->match($request);
+            $route = $app->getRoute();
 
             $log = new Utopia\Logger\Log();
 

app/init.php

@@ -32,6 +32,7 @@ use Appwrite\Network\Validator\Email;
 use Appwrite\Network\Validator\Origin;
 use Appwrite\OpenSSL\OpenSSL;
 use Appwrite\URL\URL as AppwriteURL;
+use Appwrite\Usage\Stats;
 use Utopia\App;
 use Utopia\Logger\Logger;
 use Utopia\Cache\Adapter\Redis as RedisCache;
@@ -68,6 +69,7 @@ use Utopia\Pools\Group;
 use Utopia\Pools\Pool;
 use Ahc\Jwt\JWT;
 use Ahc\Jwt\JWTException;
+use Appwrite\Auth\OAuth2\Github;
 use Appwrite\Event\Func;
 use MaxMind\Db\Reader;
 use PHPMailer\PHPMailer\PHPMailer;
@@ -75,6 +77,7 @@ use Swoole\Database\PDOProxy;
 use Utopia\Queue;
 use Utopia\Queue\Connection;
 use Utopia\Storage\Storage;
+use Utopia\VCS\Adapter\Git\GitHub as VcsGitHub;
 use Utopia\Validator\Range;
 use Utopia\Validator\IP;
 use Utopia\Validator\URL;
@@ -105,8 +108,8 @@ const APP_LIMIT_LIST_DEFAULT = 25; // Default maximum number of items to return
 const APP_KEY_ACCCESS = 24 * 60 * 60; // 24 hours
 const APP_USER_ACCCESS = 24 * 60 * 60; // 24 hours
 const APP_CACHE_UPDATE = 24 * 60 * 60; // 24 hours
-const APP_CACHE_BUSTER = 506;
-const APP_VERSION_STABLE = '1.3.8';
+const APP_CACHE_BUSTER = 507;
+const APP_VERSION_STABLE = '1.4.0';
 const APP_DATABASE_ATTRIBUTE_EMAIL = 'email';
 const APP_DATABASE_ATTRIBUTE_ENUM = 'enum';
 const APP_DATABASE_ATTRIBUTE_IP = 'ip';
@@ -133,6 +136,7 @@ const APP_SOCIAL_DISCORD_CHANNEL = '564160730845151244';
 const APP_SOCIAL_DEV = 'https://dev.to/appwrite';
 const APP_SOCIAL_STACKSHARE = 'https://stackshare.io/appwrite';
 const APP_SOCIAL_YOUTUBE = 'https://www.youtube.com/c/appwrite?sub_confirmation=1';
+const APP_HOSTNAME_INTERNAL = 'appwrite';
 // Database Reconnect
 const DATABASE_RECONNECT_SLEEP = 2;
 const DATABASE_RECONNECT_MAX_ATTEMPTS = 10;
@@ -156,10 +160,11 @@ const DELETE_TYPE_TEAMS = 'teams';
 const DELETE_TYPE_EXECUTIONS = 'executions';
 const DELETE_TYPE_AUDIT = 'audit';
 const DELETE_TYPE_ABUSE = 'abuse';
-const DELETE_TYPE_CERTIFICATES = 'certificates';
 const DELETE_TYPE_USAGE = 'usage';
 const DELETE_TYPE_REALTIME = 'realtime';
 const DELETE_TYPE_BUCKETS = 'buckets';
+const DELETE_TYPE_INSTALLATIONS = 'installations';
+const DELETE_TYPE_RULES = 'rules';
 const DELETE_TYPE_SESSIONS = 'sessions';
 const DELETE_TYPE_CACHE_BY_TIMESTAMP = 'cacheByTimeStamp';
 const DELETE_TYPE_CACHE_BY_RESOURCE  = 'cacheByResource';
@@ -181,6 +186,9 @@ const APP_AUTH_TYPE_KEY = 'Key';
 const APP_AUTH_TYPE_ADMIN = 'Admin';
 // Response related
 const MAX_OUTPUT_CHUNK_SIZE = 2 * 1024 * 1024; // 2MB
+// Function headers
+const FUNCTION_ALLOWLIST_HEADERS_REQUEST = ['content-type', 'agent', 'content-length', 'host'];
+const FUNCTION_ALLOWLIST_HEADERS_RESPONSE = ['content-type', 'content-length'];
 // Usage metrics
 const METRIC_TEAMS = 'teams';
 const METRIC_USERS = 'users';
@@ -229,7 +237,6 @@ Config::load('providers', __DIR__ . '/config/providers.php');
 Config::load('platforms', __DIR__ . '/config/platforms.php');
 Config::load('collections', __DIR__ . '/config/collections.php');
 Config::load('runtimes', __DIR__ . '/config/runtimes.php');
-Config::load('usage', __DIR__ . '/config/usage.php');
 Config::load('roles', __DIR__ . '/config/roles.php');  // User roles and scopes
 Config::load('scopes', __DIR__ . '/config/scopes.php');  // User roles and scopes
 Config::load('services', __DIR__ . '/config/services.php');  // List of services
@@ -375,20 +382,6 @@ Database::addFilter(
     }
 );
 
-Database::addFilter(
-    'subQueryDomains',
-    function (mixed $value) {
-        return null;
-    },
-    function (mixed $value, Document $document, Database $database) {
-        return $database
-            ->find('domains', [
-                Query::equal('projectInternalId', [$document->getInternalId()]),
-                Query::limit(APP_LIMIT_SUBQUERY),
-            ]);
-    }
-);
-
 Database::addFilter(
     'subQueryKeys',
     function (mixed $value) {
@@ -466,7 +459,8 @@ Database::addFilter(
     function (mixed $value, Document $document, Database $database) {
         return $database
             ->find('variables', [
-                Query::equal('functionInternalId', [$document->getInternalId()]),
+                Query::equal('resourceInternalId', [$document->getInternalId()]),
+                Query::equal('resourceType', ['function']),
                 Query::limit(APP_LIMIT_SUBQUERY),
             ]);
     }
@@ -498,6 +492,21 @@ Database::addFilter(
     }
 );
 
+// READ-ONLY! TO update, write directly to 'variables' collection. After update to vars, make sure to deleteCachedDocument()
+Database::addFilter(
+    'subQueryProjectVariables',
+    function (mixed $value) {
+        return null;
+    },
+    function (mixed $value, Document $document, Database $database) {
+        return $database
+            ->find('variables', [
+                Query::equal('resourceType', ['project']),
+                Query::limit(APP_LIMIT_SUBQUERY)
+            ]);
+    }
+);
+
 Database::addFilter(
     'userSearch',
     function (mixed $value, Document $user) {
@@ -580,14 +589,15 @@ $register->set('logger', function () {
 $register->set('pools', function () {
     $group = new Group();
 
-    $fallbackForDB = AppwriteURL::unparse([
+    $fallbackForDB = 'db_main=' . AppwriteURL::unparse([
         'scheme' => 'mariadb',
         'host' => App::getEnv('_APP_DB_HOST', 'mariadb'),
         'port' => App::getEnv('_APP_DB_PORT', '3306'),
         'user' => App::getEnv('_APP_DB_USER', ''),
         'pass' => App::getEnv('_APP_DB_PASS', ''),
+        'path' => App::getEnv('_APP_DB_SCHEMA', ''),
     ]);
-    $fallbackForRedis = AppwriteURL::unparse([
+    $fallbackForRedis = 'redis_main=' . AppwriteURL::unparse([
         'scheme' => 'redis',
         'host' => App::getEnv('_APP_REDIS_HOST', 'redis'),
         'port' => App::getEnv('_APP_REDIS_PORT', '6379'),
@@ -761,6 +771,31 @@ $register->set('pools', function () {
     return $group;
 });
 
+$register->set('influxdb', function () {
+
+ // Register DB connection
+    $host = App::getEnv('_APP_INFLUXDB_HOST', '');
+    $port = App::getEnv('_APP_INFLUXDB_PORT', '');
+
+    if (empty($host) || empty($port)) {
+        return;
+    }
+    $driver = new InfluxDB\Driver\Curl("http://{$host}:{$port}");
+    $client = new InfluxDB\Client($host, $port, '', '', false, false, 5);
+    $client->setDriver($driver);
+
+    return $client;
+});
+$register->set('statsd', function () {
+    // Register DB connection
+    $host = App::getEnv('_APP_STATSD_HOST', 'telegraf');
+    $port = App::getEnv('_APP_STATSD_PORT', 8125);
+
+    $connection = new \Domnikl\Statsd\Connection\UdpSocket($host, $port);
+    $statsd = new \Domnikl\Statsd\Client($connection);
+
+    return $statsd;
+});
 $register->set('smtp', function () {
     $mail = new PHPMailer(true);
 
@@ -864,9 +899,9 @@ App::setResource('queue', function (Group $pools) {
 App::setResource('queueForFunctions', function (Connection $queue) {
     return new Func($queue);
 }, ['queue']);
-App::setResource('queueForUsage', function (Connection $queue) {
-    return new Usage($queue);
-}, ['queue']);
+App::setResource('usage', function ($register) {
+    return new Stats($register->get('statsd'));
+}, ['register']);
 App::setResource('clients', function ($request, $console, $project) {
     $console->setAttribute('platforms', [ // Always allow current host
         '$collection' => ID::custom('platforms'),
@@ -949,7 +984,11 @@ App::setResource('user', function ($mode, $project, $console, $request, $respons
         if ($project->isEmpty()) {
             $user = new Document([]);
         } else {
-            $user = $dbForProject->getDocument('users', Auth::$unique);
+            if ($project->getId() === 'console') {
+                $user = $dbForConsole->getDocument('users', Auth::$unique);
+            } else {
+                $user = $dbForProject->getDocument('users', Auth::$unique);
+            }
         }
     } else {
         $user = $dbForConsole->getDocument('users', Auth::$unique);
@@ -1078,11 +1117,44 @@ App::setResource('dbForConsole', function (Group $pools, Cache $cache) {
 
     $database = new Database($dbAdapter, $cache);
 
-    $database->setNamespace('console');
+    $database->setNamespace('_console');
 
     return $database;
 }, ['pools', 'cache']);
 
+App::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $cache) {
+    $databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools
+
+    $getProjectDB = function (Document $project) use ($pools, $dbForConsole, $cache, &$databases) {
+        if ($project->isEmpty() || $project->getId() === 'console') {
+            return $dbForConsole;
+        }
+
+        $databaseName = $project->getAttribute('database');
+
+        if (isset($databases[$databaseName])) {
+            $database = $databases[$databaseName];
+            $database->setNamespace('_' . $project->getInternalId());
+            return $database;
+        }
+
+        $dbAdapter = $pools
+            ->get($databaseName)
+            ->pop()
+            ->getResource();
+
+        $database = new Database($dbAdapter, $cache);
+
+        $databases[$databaseName] = $database;
+
+        $database->setNamespace('_' . $project->getInternalId());
+
+        return $database;
+    };
+
+    return $getProjectDB;
+}, ['pools', 'dbForConsole', 'cache']);
+
 App::setResource('cache', function (Group $pools) {
     $list = Config::getParam('pools-cache', []);
     $adapters = [];
@@ -1118,38 +1190,81 @@ function getDevice($root): Device
 {
     $connection = App::getEnv('_APP_CONNECTIONS_STORAGE', '');
 
-    $acl = 'private';
-    $device = Storage::DEVICE_LOCAL;
-    $accessKey = '';
-    $accessSecret = '';
-    $bucket = '';
-    $region = '';
+    if (!empty($connection)) {
+        $acl = 'private';
+        $device = Storage::DEVICE_LOCAL;
+        $accessKey = '';
+        $accessSecret = '';
+        $bucket = '';
+        $region = '';
 
-    try {
-        $dsn = new DSN($connection);
-        $device = $dsn->getScheme();
-        $accessKey = $dsn->getUser();
-        $accessSecret = $dsn->getPassword();
-        $bucket = $dsn->getPath();
-        $region = $dsn->getParam('region');
-    } catch (\Exception $e) {
-        Console::error($e->getMessage() . 'Invalid DSN. Defaulting to Local device.');
-    }
+        try {
+            $dsn = new DSN($connection);
+            $device = $dsn->getScheme();
+            $accessKey = $dsn->getUser() ?? '';
+            $accessSecret = $dsn->getPassword() ?? '';
+            $bucket = $dsn->getPath() ?? '';
+            $region = $dsn->getParam('region');
+        } catch (\Exception $e) {
+            Console::warning($e->getMessage() . 'Invalid DSN. Defaulting to Local device.');
+        }
 
-    switch ($device) {
-        case Storage::DEVICE_S3:
-            return new S3($root, $accessKey, $accessSecret, $bucket, $region, $acl);
-        case STORAGE::DEVICE_DO_SPACES:
-            return new DOSpaces($root, $accessKey, $accessSecret, $bucket, $region, $acl);
-        case Storage::DEVICE_BACKBLAZE:
-            return new Backblaze($root, $accessKey, $accessSecret, $bucket, $region, $acl);
-        case Storage::DEVICE_LINODE:
-            return new Linode($root, $accessKey, $accessSecret, $bucket, $region, $acl);
-        case Storage::DEVICE_WASABI:
-            return new Wasabi($root, $accessKey, $accessSecret, $bucket, $region, $acl);
-        case Storage::DEVICE_LOCAL:
-        default:
-            return new Local($root);
+        switch ($device) {
+            case Storage::DEVICE_S3:
+                return new S3($root, $accessKey, $accessSecret, $bucket, $region, $acl);
+            case STORAGE::DEVICE_DO_SPACES:
+                return new DOSpaces($root, $accessKey, $accessSecret, $bucket, $region, $acl);
+            case Storage::DEVICE_BACKBLAZE:
+                return new Backblaze($root, $accessKey, $accessSecret, $bucket, $region, $acl);
+            case Storage::DEVICE_LINODE:
+                return new Linode($root, $accessKey, $accessSecret, $bucket, $region, $acl);
+            case Storage::DEVICE_WASABI:
+                return new Wasabi($root, $accessKey, $accessSecret, $bucket, $region, $acl);
+            case Storage::DEVICE_LOCAL:
+            default:
+                return new Local($root);
+        }
+    } else {
+        switch (strtolower(App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL) ?? '')) {
+            case Storage::DEVICE_LOCAL:
+            default:
+                return new Local($root);
+            case Storage::DEVICE_S3:
+                $s3AccessKey = App::getEnv('_APP_STORAGE_S3_ACCESS_KEY', '');
+                $s3SecretKey = App::getEnv('_APP_STORAGE_S3_SECRET', '');
+                $s3Region = App::getEnv('_APP_STORAGE_S3_REGION', '');
+                $s3Bucket = App::getEnv('_APP_STORAGE_S3_BUCKET', '');
+                $s3Acl = 'private';
+                return new S3($root, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl);
+            case Storage::DEVICE_DO_SPACES:
+                $doSpacesAccessKey = App::getEnv('_APP_STORAGE_DO_SPACES_ACCESS_KEY', '');
+                $doSpacesSecretKey = App::getEnv('_APP_STORAGE_DO_SPACES_SECRET', '');
+                $doSpacesRegion = App::getEnv('_APP_STORAGE_DO_SPACES_REGION', '');
+                $doSpacesBucket = App::getEnv('_APP_STORAGE_DO_SPACES_BUCKET', '');
+                $doSpacesAcl = 'private';
+                return new DOSpaces($root, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl);
+            case Storage::DEVICE_BACKBLAZE:
+                $backblazeAccessKey = App::getEnv('_APP_STORAGE_BACKBLAZE_ACCESS_KEY', '');
+                $backblazeSecretKey = App::getEnv('_APP_STORAGE_BACKBLAZE_SECRET', '');
+                $backblazeRegion = App::getEnv('_APP_STORAGE_BACKBLAZE_REGION', '');
+                $backblazeBucket = App::getEnv('_APP_STORAGE_BACKBLAZE_BUCKET', '');
+                $backblazeAcl = 'private';
+                return new Backblaze($root, $backblazeAccessKey, $backblazeSecretKey, $backblazeBucket, $backblazeRegion, $backblazeAcl);
+            case Storage::DEVICE_LINODE:
+                $linodeAccessKey = App::getEnv('_APP_STORAGE_LINODE_ACCESS_KEY', '');
+                $linodeSecretKey = App::getEnv('_APP_STORAGE_LINODE_SECRET', '');
+                $linodeRegion = App::getEnv('_APP_STORAGE_LINODE_REGION', '');
+                $linodeBucket = App::getEnv('_APP_STORAGE_LINODE_BUCKET', '');
+                $linodeAcl = 'private';
+                return new Linode($root, $linodeAccessKey, $linodeSecretKey, $linodeBucket, $linodeRegion, $linodeAcl);
+            case Storage::DEVICE_WASABI:
+                $wasabiAccessKey = App::getEnv('_APP_STORAGE_WASABI_ACCESS_KEY', '');
+                $wasabiSecretKey = App::getEnv('_APP_STORAGE_WASABI_SECRET', '');
+                $wasabiRegion = App::getEnv('_APP_STORAGE_WASABI_REGION', '');
+                $wasabiBucket = App::getEnv('_APP_STORAGE_WASABI_BUCKET', '');
+                $wasabiAcl = 'private';
+                return new Wasabi($root, $wasabiAccessKey, $wasabiSecretKey, $wasabiBucket, $wasabiRegion, $wasabiAcl);
+        }
     }
 }
 
@@ -1309,6 +1424,10 @@ App::setResource('heroes', function () {
     return $list;
 });
 
+App::setResource('gitHub', function (Cache $cache) {
+    return new VcsGitHub($cache);
+}, ['cache']);
+
 App::setResource('requestTimestamp', function ($request) {
     //TODO: Move this to the Request class itself
     $timestampHeader = $request->getHeader('x-appwrite-timestamp');

app/realtime.php

@@ -47,7 +47,7 @@ function getConsoleDB(): Database
 
     $database = new Database($dbAdapter, getCache());
 
-    $database->setNamespace('console');
+    $database->setNamespace('_console');
 
     return $database;
 }

app/views/install/compose.phtml

@@ -80,6 +80,7 @@ services:
       - mariadb
       - redis
 #      - clamav
+      - influxdb
     environment:
       - _APP_ENV
       - _APP_WORKER_PER_CORE
@@ -87,8 +88,6 @@ services:
       - _APP_CONSOLE_WHITELIST_ROOT
       - _APP_CONSOLE_WHITELIST_EMAILS
       - _APP_CONSOLE_WHITELIST_IPS
-      - _APP_CONSOLE_INVITES
-      - _APP_CONSOLE_ROOT_SESSION
       - _APP_SYSTEM_EMAIL_NAME
       - _APP_SYSTEM_EMAIL_ADDRESS
       - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS
@@ -98,28 +97,50 @@ services:
       - _APP_OPENSSL_KEY_V1
       - _APP_DOMAIN
       - _APP_DOMAIN_TARGET
+      - _APP_DOMAIN_FUNCTIONS
+      - _APP_REDIS_HOST	
+      - _APP_REDIS_PORT	
+      - _APP_REDIS_USER	
+      - _APP_REDIS_PASS
       - _APP_DB_HOST
       - _APP_DB_PORT
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
-      - _APP_CONNECTIONS_MAX
-      - _APP_REDIS_HOST
-      - _APP_REDIS_PORT
-      - _APP_REDIS_USER
-      - _APP_REDIS_PASS
       - _APP_SMTP_HOST
       - _APP_SMTP_PORT
       - _APP_SMTP_SECURE
       - _APP_SMTP_USERNAME
       - _APP_SMTP_PASSWORD
       - _APP_USAGE_STATS
+      - _APP_INFLUXDB_HOST
+      - _APP_INFLUXDB_PORT
       - _APP_STORAGE_LIMIT
       - _APP_STORAGE_PREVIEW_LIMIT
       - _APP_STORAGE_ANTIVIRUS
       - _APP_STORAGE_ANTIVIRUS_HOST
       - _APP_STORAGE_ANTIVIRUS_PORT
-      - _APP_CONNECTIONS_STORAGE
+      - _APP_STORAGE_DEVICE	
+      - _APP_STORAGE_S3_ACCESS_KEY	
+      - _APP_STORAGE_S3_SECRET	
+      - _APP_STORAGE_S3_REGION	
+      - _APP_STORAGE_S3_BUCKET	
+      - _APP_STORAGE_DO_SPACES_ACCESS_KEY	
+      - _APP_STORAGE_DO_SPACES_SECRET	
+      - _APP_STORAGE_DO_SPACES_REGION	
+      - _APP_STORAGE_DO_SPACES_BUCKET	
+      - _APP_STORAGE_BACKBLAZE_ACCESS_KEY	
+      - _APP_STORAGE_BACKBLAZE_SECRET	
+      - _APP_STORAGE_BACKBLAZE_REGION	
+      - _APP_STORAGE_BACKBLAZE_BUCKET	
+      - _APP_STORAGE_LINODE_ACCESS_KEY	
+      - _APP_STORAGE_LINODE_SECRET	
+      - _APP_STORAGE_LINODE_REGION	
+      - _APP_STORAGE_LINODE_BUCKET	
+      - _APP_STORAGE_WASABI_ACCESS_KEY	
+      - _APP_STORAGE_WASABI_SECRET	
+      - _APP_STORAGE_WASABI_REGION	
+      - _APP_STORAGE_WASABI_BUCKET
       - _APP_FUNCTIONS_SIZE_LIMIT
       - _APP_FUNCTIONS_TIMEOUT
       - _APP_FUNCTIONS_BUILD_TIMEOUT
@@ -130,6 +151,8 @@ services:
       - _APP_EXECUTOR_HOST
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
+      - _APP_STATSD_HOST
+      - _APP_STATSD_PORT
       - _APP_MAINTENANCE_INTERVAL
       - _APP_MAINTENANCE_RETENTION_EXECUTION
       - _APP_MAINTENANCE_RETENTION_CACHE
@@ -142,6 +165,15 @@ services:
       - _APP_GRAPHQL_MAX_BATCH_SIZE
       - _APP_GRAPHQL_MAX_COMPLEXITY
       - _APP_GRAPHQL_MAX_DEPTH
+      - _APP_VCS_GITHUB_APP_NAME	
+      - _APP_VCS_GITHUB_PRIVATE_KEY	
+      - _APP_VCS_GITHUB_APP_ID	
+      - _APP_VCS_GITHUB_WEBHOOK_SECRET	
+      - _APP_VCS_GITHUB_CLIENT_SECRET	
+      - _APP_VCS_GITHUB_CLIENT_ID	
+      - _APP_MIGRATIONS_FIREBASE_CLIENT_ID	
+      - _APP_MIGRATIONS_FIREBASE_CLIENT_SECRET	
+      - _APP_ASSISTANT_OPENAI_API_KEY
 
   appwrite-realtime:
     image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
@@ -174,16 +206,15 @@ services:
       - _APP_WORKER_PER_CORE
       - _APP_OPTIONS_ABUSE
       - _APP_OPENSSL_KEY_V1
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
       - _APP_DB_HOST
       - _APP_DB_PORT
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
-      - _APP_CONNECTIONS_MAX
-      - _APP_REDIS_HOST
-      - _APP_REDIS_PORT
-      - _APP_REDIS_USER
-      - _APP_REDIS_PASS
       - _APP_USAGE_STATS
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
@@ -201,16 +232,17 @@ services:
       - mariadb
     environment:
       - _APP_ENV
+      - _APP_WORKER_PER_CORE
       - _APP_OPENSSL_KEY_V1
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
       - _APP_DB_HOST
       - _APP_DB_PORT
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
-      - _APP_REDIS_HOST
-      - _APP_REDIS_PORT
-      - _APP_REDIS_USER
-      - _APP_REDIS_PASS
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
 
@@ -227,6 +259,7 @@ services:
       - mariadb
     environment:
       - _APP_ENV
+      - _APP_WORKER_PER_CORE
       - _APP_OPENSSL_KEY_V1
       - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS
       - _APP_REDIS_HOST
@@ -255,17 +288,38 @@ services:
       - appwrite-certificates:/storage/certificates:rw
     environment:
       - _APP_ENV
+      - _APP_WORKER_PER_CORE
       - _APP_OPENSSL_KEY_V1
+      - _APP_REDIS_HOST	
+      - _APP_REDIS_PORT	
+      - _APP_REDIS_USER	
+      - _APP_REDIS_PASS
       - _APP_DB_HOST
       - _APP_DB_PORT
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
-      - _APP_REDIS_HOST
-      - _APP_REDIS_PORT
-      - _APP_REDIS_USER
-      - _APP_REDIS_PASS
-      - _APP_CONNECTIONS_STORAGE
+      - _APP_STORAGE_DEVICE	
+      - _APP_STORAGE_S3_ACCESS_KEY	
+      - _APP_STORAGE_S3_SECRET	
+      - _APP_STORAGE_S3_REGION	
+      - _APP_STORAGE_S3_BUCKET	
+      - _APP_STORAGE_DO_SPACES_ACCESS_KEY	
+      - _APP_STORAGE_DO_SPACES_SECRET	
+      - _APP_STORAGE_DO_SPACES_REGION	
+      - _APP_STORAGE_DO_SPACES_BUCKET	
+      - _APP_STORAGE_BACKBLAZE_ACCESS_KEY	
+      - _APP_STORAGE_BACKBLAZE_SECRET	
+      - _APP_STORAGE_BACKBLAZE_REGION	
+      - _APP_STORAGE_BACKBLAZE_BUCKET	
+      - _APP_STORAGE_LINODE_ACCESS_KEY	
+      - _APP_STORAGE_LINODE_SECRET	
+      - _APP_STORAGE_LINODE_REGION	
+      - _APP_STORAGE_LINODE_BUCKET	
+      - _APP_STORAGE_WASABI_ACCESS_KEY	
+      - _APP_STORAGE_WASABI_SECRET	
+      - _APP_STORAGE_WASABI_REGION	
+      - _APP_STORAGE_WASABI_BUCKET
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
       - _APP_EXECUTOR_SECRET
@@ -284,16 +338,17 @@ services:
       - mariadb
     environment:
       - _APP_ENV
+      - _APP_WORKER_PER_CORE
       - _APP_OPENSSL_KEY_V1
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
       - _APP_DB_HOST
       - _APP_DB_PORT
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
-      - _APP_REDIS_HOST
-      - _APP_REDIS_PORT
-      - _APP_REDIS_USER
-      - _APP_REDIS_PASS
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
 
@@ -310,20 +365,30 @@ services:
       - mariadb
     environment:
       - _APP_ENV
+      - _APP_WORKER_PER_CORE
       - _APP_OPENSSL_KEY_V1
       - _APP_EXECUTOR_SECRET
       - _APP_EXECUTOR_HOST
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
       - _APP_DB_HOST
       - _APP_DB_PORT
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
-      - _APP_REDIS_HOST
-      - _APP_REDIS_PORT
-      - _APP_REDIS_USER
-      - _APP_REDIS_PASS
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
+      - _APP_VCS_GITHUB_APP_NAME	
+      - _APP_VCS_GITHUB_PRIVATE_KEY	
+      - _APP_VCS_GITHUB_APP_ID	
+      - _APP_FUNCTIONS_TIMEOUT	
+      - _APP_FUNCTIONS_BUILD_TIMEOUT	
+      - _APP_FUNCTIONS_CPUS	
+      - _APP_FUNCTIONS_MEMORY	
+      - _APP_OPTIONS_FORCE_HTTPS	
+      - _APP_DOMAIN
 
   appwrite-worker-certificates:
     image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
@@ -341,19 +406,21 @@ services:
       - appwrite-certificates:/storage/certificates:rw
     environment:
       - _APP_ENV
+      - _APP_WORKER_PER_CORE
       - _APP_OPENSSL_KEY_V1
       - _APP_DOMAIN
       - _APP_DOMAIN_TARGET
+      - _APP_DOMAIN_FUNCTIONS
       - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS
+      - _APP_REDIS_HOST	
+      - _APP_REDIS_PORT	
+      - _APP_REDIS_USER	
+      - _APP_REDIS_PASS
       - _APP_DB_HOST
       - _APP_DB_PORT
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
-      - _APP_REDIS_HOST
-      - _APP_REDIS_PORT
-      - _APP_REDIS_USER
-      - _APP_REDIS_PASS
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
 
@@ -371,20 +438,28 @@ services:
       - openruntimes-executor
     environment:
       - _APP_ENV
+      - _APP_WORKER_PER_CORE
       - _APP_OPENSSL_KEY_V1
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
       - _APP_DB_HOST
       - _APP_DB_PORT
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
-      - _APP_REDIS_HOST
-      - _APP_REDIS_PORT
-      - _APP_REDIS_USER
-      - _APP_REDIS_PASS
       - _APP_FUNCTIONS_TIMEOUT
+      - _APP_FUNCTIONS_BUILD_TIMEOUT	
+      - _APP_FUNCTIONS_CPUS	
+      - _APP_FUNCTIONS_MEMORY
       - _APP_EXECUTOR_SECRET
       - _APP_EXECUTOR_HOST
       - _APP_USAGE_STATS
+      - _APP_DOCKER_HUB_USERNAME	
+      - _APP_DOCKER_HUB_PASSWORD	
+      - _APP_LOGGING_CONFIG	
+      - _APP_LOGGING_PROVIDER
 
   appwrite-worker-mails:
     image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
@@ -398,6 +473,7 @@ services:
       - redis
     environment:
       - _APP_ENV
+      - _APP_WORKER_PER_CORE
       - _APP_OPENSSL_KEY_V1
       - _APP_SYSTEM_EMAIL_NAME
       - _APP_SYSTEM_EMAIL_ADDRESS
@@ -425,6 +501,7 @@ services:
       - redis
     environment:
       - _APP_ENV
+      - _APP_WORKER_PER_CORE
       - _APP_REDIS_HOST
       - _APP_REDIS_PORT
       - _APP_REDIS_USER
@@ -446,24 +523,57 @@ services:
       - redis
     environment:
       - _APP_ENV
+      - _APP_WORKER_PER_CORE
       - _APP_OPENSSL_KEY_V1
       - _APP_DOMAIN
       - _APP_DOMAIN_TARGET
+      - _APP_DOMAIN_FUNCTIONS
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
       - _APP_DB_HOST
       - _APP_DB_PORT
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
-      - _APP_REDIS_HOST
-      - _APP_REDIS_PORT
-      - _APP_REDIS_USER
-      - _APP_REDIS_PASS
       - _APP_MAINTENANCE_INTERVAL
       - _APP_MAINTENANCE_RETENTION_EXECUTION
       - _APP_MAINTENANCE_RETENTION_CACHE
       - _APP_MAINTENANCE_RETENTION_ABUSE
       - _APP_MAINTENANCE_RETENTION_AUDIT
       - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY
+      - _APP_MAINTENANCE_RETENTION_SCHEDULES
+
+  appwrite-usage:
+    image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
+    entrypoint: usage
+    container_name: appwrite-usage
+    <<: *x-logging
+    restart: unless-stopped
+    networks:
+      - appwrite
+    depends_on:
+      - influxdb
+      - mariadb
+    environment:
+      - _APP_ENV
+      - _APP_WORKER_PER_CORE
+      - _APP_OPENSSL_KEY_V1
+      - _APP_DB_HOST
+      - _APP_DB_PORT
+      - _APP_DB_SCHEMA
+      - _APP_DB_USER
+      - _APP_DB_PASS
+      - _APP_INFLUXDB_HOST
+      - _APP_INFLUXDB_PORT
+      - _APP_USAGE_AGGREGATION_INTERVAL
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
+      - _APP_LOGGING_PROVIDER
+      - _APP_LOGGING_CONFIG
 
   appwrite-schedule:
     image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
@@ -477,17 +587,23 @@ services:
       - redis
     environment:
       - _APP_ENV
+      - _APP_WORKER_PER_CORE
       - _APP_REDIS_HOST
       - _APP_REDIS_PORT
       - _APP_REDIS_USER
       - _APP_REDIS_PASS
+      - _APP_DB_HOST	
+      - _APP_DB_PORT	
+      - _APP_DB_SCHEMA	
+      - _APP_DB_USER	
+      - _APP_DB_PASS
 
   openruntimes-executor:
     container_name: openruntimes-executor
-    hostname: exc1
+    hostname: executor
     <<: *x-logging
     stop_signal: SIGINT
-    image: openruntimes/executor:0.1.4
+    image: openruntimes/executor:0.3.5
     networks:
       - appwrite
       - runtimes
@@ -497,7 +613,6 @@ services:
       - appwrite-functions:/storage/functions:rw
       - /tmp:/tmp:rw
     environment:
-      - OPR_EXECUTOR_CONNECTION_STORAGE=$_APP_CONNECTIONS_STORAGE
       - OPR_EXECUTOR_INACTIVE_TRESHOLD=$_APP_FUNCTIONS_INACTIVE_THRESHOLD
       - OPR_EXECUTOR_MAINTENANCE_INTERVAL=$_APP_FUNCTIONS_MAINTENANCE_INTERVAL
       - OPR_EXECUTOR_NETWORK=$_APP_FUNCTIONS_RUNTIMES_NETWORK
@@ -508,6 +623,27 @@ services:
       - OPR_EXECUTOR_SECRET=$_APP_EXECUTOR_SECRET
       - OPR_EXECUTOR_LOGGING_PROVIDER=$_APP_LOGGING_PROVIDER
       - OPR_EXECUTOR_LOGGING_CONFIG=$_APP_LOGGING_CONFIG
+      - OPR_EXECUTOR_STORAGE_DEVICE=$_APP_STORAGE_DEVICE
+      - OPR_EXECUTOR_STORAGE_S3_ACCESS_KEY=$_APP_STORAGE_S3_ACCESS_KEY
+      - OPR_EXECUTOR_STORAGE_S3_SECRET=$_APP_STORAGE_S3_SECRET
+      - OPR_EXECUTOR_STORAGE_S3_REGION=$_APP_STORAGE_S3_REGION
+      - OPR_EXECUTOR_STORAGE_S3_BUCKET=$_APP_STORAGE_S3_BUCKET
+      - OPR_EXECUTOR_STORAGE_DO_SPACES_ACCESS_KEY=$_APP_STORAGE_DO_SPACES_ACCESS_KEY
+      - OPR_EXECUTOR_STORAGE_DO_SPACES_SECRET=$_APP_STORAGE_DO_SPACES_SECRET
+      - OPR_EXECUTOR_STORAGE_DO_SPACES_REGION=$_APP_STORAGE_DO_SPACES_REGION
+      - OPR_EXECUTOR_STORAGE_DO_SPACES_BUCKET=$_APP_STORAGE_DO_SPACES_BUCKET
+      - OPR_EXECUTOR_STORAGE_BACKBLAZE_ACCESS_KEY=$_APP_STORAGE_BACKBLAZE_ACCESS_KEY
+      - OPR_EXECUTOR_STORAGE_BACKBLAZE_SECRET=$_APP_STORAGE_BACKBLAZE_SECRET
+      - OPR_EXECUTOR_STORAGE_BACKBLAZE_REGION=$_APP_STORAGE_BACKBLAZE_REGION
+      - OPR_EXECUTOR_STORAGE_BACKBLAZE_BUCKET=$_APP_STORAGE_BACKBLAZE_BUCKET
+      - OPR_EXECUTOR_STORAGE_LINODE_ACCESS_KEY=$_APP_STORAGE_LINODE_ACCESS_KEY
+      - OPR_EXECUTOR_STORAGE_LINODE_SECRET=$_APP_STORAGE_LINODE_SECRET
+      - OPR_EXECUTOR_STORAGE_LINODE_REGION=$_APP_STORAGE_LINODE_REGION
+      - OPR_EXECUTOR_STORAGE_LINODE_BUCKET=$_APP_STORAGE_LINODE_BUCKET
+      - OPR_EXECUTOR_STORAGE_WASABI_ACCESS_KEY=$_APP_STORAGE_WASABI_ACCESS_KEY
+      - OPR_EXECUTOR_STORAGE_WASABI_SECRET=$_APP_STORAGE_WASABI_SECRET
+      - OPR_EXECUTOR_STORAGE_WASABI_REGION=$_APP_STORAGE_WASABI_REGION
+      - OPR_EXECUTOR_STORAGE_WASABI_BUCKET=$_APP_STORAGE_WASABI_BUCKET
 
   mariadb:
     image: mariadb:10.7 # fix issues when upgrading using: mysql_upgrade -u root -p
@@ -549,41 +685,26 @@ services:
   #   volumes:
   #     - appwrite-uploads:/storage/uploads
 
-  appwrite-worker-usage:
-    image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
-    entrypoint: worker-usage
+  influxdb:
+    image: appwrite/influxdb:1.5.0
+    container_name: appwrite-influxdb
     <<: *x-logging
-    container_name: appwrite-worker-usage
+    restart: unless-stopped
     networks:
       - appwrite
     volumes:
-      - ./app:/usr/src/code/app
-      - ./src:/usr/src/code/src
-    depends_on:
-      - redis
-      - mariadb
+      - appwrite-influxdb:/var/lib/influxdb:rw
+
+  telegraf:
+    image: appwrite/telegraf:1.4.0
+    container_name: appwrite-telegraf
+    <<: *x-logging
+    restart: unless-stopped
+    networks:
+      - appwrite
     environment:
-      - _APP_ENV
-      - _APP_WORKER_PER_CORE
-      - _APP_CONNECTIONS_MAX
-      - _APP_POOL_CLIENTS
-      - _APP_OPENSSL_KEY_V1
-      - _APP_DB_HOST
-      - _APP_DB_PORT
-      - _APP_DB_SCHEMA
-      - _APP_DB_USER
-      - _APP_DB_PASS
-      - _APP_REDIS_HOST
-      - _APP_REDIS_PORT
-      - _APP_REDIS_USER
-      - _APP_REDIS_PASS
-      - _APP_CONNECTIONS_DB_CONSOLE
-      - _APP_CONNECTIONS_DB_PROJECT
-      - _APP_CONNECTIONS_CACHE
-      - _APP_CONNECTIONS_QUEUE
-      - _APP_USAGE_STATS
-      - _APP_LOGGING_PROVIDER
-      - _APP_LOGGING_CONFIG
+      - _APP_INFLUXDB_HOST
+      - _APP_INFLUXDB_PORT
 
 networks:
   gateway:
@@ -599,6 +720,7 @@ volumes:
   appwrite-cache:
   appwrite-uploads:
   appwrite-certificates:
-  appwrite-config:
   appwrite-functions:
   appwrite-builds:
+  appwrite-influxdb:
+  appwrite-config:

app/worker.php

@@ -4,14 +4,17 @@ require_once __DIR__ . '/init.php';
 
 use Appwrite\Event\Func;
 use Appwrite\Event\Usage;
+use Appwrite\Usage\Stats;
 use Swoole\Runtime;
 use Utopia\App;
 use Utopia\Cache\Adapter\Sharding;
 use Utopia\Cache\Cache;
+use Utopia\CLI\CLI;
 use Utopia\CLI\Console;
 use Utopia\Config\Config;
 use Utopia\Database\Database;
 use Utopia\Database\Document;
+use Utopia\Database\Validator\Authorization;
 use Utopia\Queue\Adapter\Swoole;
 use Utopia\Queue\Message;
 use Utopia\Queue\Server;
@@ -35,7 +38,7 @@ Server::setResource('dbForConsole', function (Cache $cache, Registry $register)
     ;
 
     $adapter = new Database($database, $cache);
-    $adapter->setNamespace('console');
+    $adapter->setNamespace('_console');
 
     return $adapter;
 }, ['cache', 'register']);
@@ -86,22 +89,16 @@ Server::setResource('queueForFunctions', function (Registry $register) {
     );
 }, ['register']);
 
-Server::setResource('queueForUsage', function (Registry $register) {
-    $pools = $register->get('pools');
-    return new Usage(
-        $pools
-            ->get('queue')
-            ->pop()
-            ->getResource()
-    );
-}, ['register']);
-
 Server::setResource('log', fn() => new Log());
 
 Server::setResource('logger', function ($register) {
     return $register->get('logger');
 }, ['register']);
 
+Server::setResource('statsd', function ($register) {
+    return $register->get('statsd');
+}, ['register']);
+
 Server::setResource('pools', function ($register) {
     return $register->get('pools');
 }, ['register']);
@@ -111,7 +108,7 @@ $connection = $pools->get('queue')->pop()->getResource();
 $workerNumber = swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6));
 
 if (empty(App::getEnv('QUEUE'))) {
-    throw new Exception('Please configure "QUEUE" environemnt variable.');
+    throw new Exception('Please configure "QUEUE" environment variable.');
 }
 
 $adapter = new Swoole($connection, $workerNumber, App::getEnv('QUEUE'));
@@ -149,7 +146,7 @@ $server
             $log->addExtra('line', $error->getLine());
             $log->addExtra('trace', $error->getTraceAsString());
             $log->addExtra('detailedTrace', $error->getTrace());
-            $log->addExtra('roles', \Utopia\Database\Validator\Authorization::$roles);
+            $log->addExtra('roles', Authorization::getRoles());
 
             $isProduction = App::getEnv('_APP_ENV', 'development') === 'production';
             $log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING);

app/workers/builds.php

@@ -1,11 +1,14 @@
 <?php
 
+use Swoole\Coroutine as Co;
 use Appwrite\Event\Event;
 use Appwrite\Event\Func;
 use Appwrite\Messaging\Adapter\Realtime;
 use Appwrite\Resque\Worker;
 use Appwrite\Utopia\Response\Model\Deployment;
 use Executor\Executor;
+use Appwrite\Usage\Stats;
+use Appwrite\Vcs\Comment;
 use Utopia\Database\DateTime;
 use Utopia\App;
 use Utopia\CLI\Console;
@@ -13,8 +16,11 @@ use Utopia\Database\Helpers\ID;
 use Utopia\DSN\DSN;
 use Utopia\Database\Document;
 use Utopia\Config\Config;
-use Utopia\Database\Validator\Authorization;
+use Utopia\Database\Database;
+use Utopia\Database\Query;
 use Utopia\Storage\Storage;
+use Utopia\Database\Validator\Authorization;
+use Utopia\VCS\Adapter\Git\GitHub;
 
 require_once __DIR__ . '/../init.php';
 
@@ -42,12 +48,14 @@ class BuildsV1 extends Worker
         $project = new Document($this->args['project'] ?? []);
         $resource = new Document($this->args['resource'] ?? []);
         $deployment = new Document($this->args['deployment'] ?? []);
+        $template = new Document($this->args['template'] ?? []);
 
         switch ($type) {
             case BUILD_TYPE_DEPLOYMENT:
             case BUILD_TYPE_RETRY:
                 Console::info('Creating build for deployment: ' . $deployment->getId());
-                $this->buildDeployment($project, $resource, $deployment);
+                $github = new GitHub($this->getCache());
+                $this->buildDeployment($github, $project, $resource, $deployment, $template);
                 break;
 
             default:
@@ -61,11 +69,12 @@ class BuildsV1 extends Worker
      * @throws \Utopia\Database\Exception\Structure
      * @throws Throwable
      */
-    protected function buildDeployment(Document $project, Document $function, Document $deployment)
+    protected function buildDeployment(GitHub $github, Document $project, Document $function, Document $deployment, Document $template)
     {
         global $register;
 
         $dbForProject = $this->getProjectDB($project);
+        $dbForConsole = $this->getConsoleDB();
 
         $function = $dbForProject->getDocument('functions', $function->getId());
         if ($function->isEmpty()) {
@@ -77,6 +86,10 @@ class BuildsV1 extends Worker
             throw new Exception('Deployment not found', 404);
         }
 
+        if (empty($deployment->getAttribute('entrypoint', ''))) {
+            throw new Exception('Entrypoint for your Appwrite Function is missing. Please specify it when making deployment or update the entrypoint under your function\'s "Settings" > "Configuration" > "Entrypoint".', 500);
+        }
+
         $runtimes = Config::getParam('runtimes', []);
         $key = $function->getAttribute('runtime');
         $runtime = isset($runtimes[$key]) ? $runtimes[$key] : null;
@@ -84,18 +97,20 @@ class BuildsV1 extends Worker
             throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported');
         }
 
-        $connection = App::getEnv('_APP_CONNECTIONS_STORAGE', ''); /** @TODO : move this to the registry or someplace else */
-        $device = Storage::DEVICE_LOCAL;
-        try {
-            $dsn = new DSN($connection);
-            $device = $dsn->getScheme();
-        } catch (\Exception $e) {
-            Console::error($e->getMessage() . 'Invalid DSN. Defaulting to Local device.');
-        }
+        // Realtime preparation
+        $allEvents = Event::generateEvents('functions.[functionId].deployments.[deploymentId].update', [
+            'functionId' => $function->getId(),
+            'deploymentId' => $deployment->getId()
+        ]);
+
+        $startTime = DateTime::now();
+        $durationStart = \microtime(true);
 
         $buildId = $deployment->getAttribute('buildId', '');
-        $startTime = DateTime::now();
-        if (empty($buildId)) {
+
+        $isNewBuild = empty($buildId);
+
+        if ($isNewBuild) {
             $buildId = ID::unique();
             $build = $dbForProject->createDocument('builds', new Document([
                 '$id' => $buildId,
@@ -104,15 +119,16 @@ class BuildsV1 extends Worker
                 'deploymentInternalId' => $deployment->getInternalId(),
                 'deploymentId' => $deployment->getId(),
                 'status' => 'processing',
-                'outputPath' => '',
+                'path' => '',
                 'runtime' => $function->getAttribute('runtime'),
-                'source' => $deployment->getAttribute('path'),
-                'sourceType' => $device,
-                'stdout' => '',
-                'stderr' => '',
+                'source' => $deployment->getAttribute('path', ''),
+                'sourceType' => strtolower(App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL)),
+                'logs' => '',
                 'endTime' => null,
-                'duration' => 0
+                'duration' => 0,
+                'size' => 0
             ]));
+
             $deployment->setAttribute('buildId', $build->getId());
             $deployment->setAttribute('buildInternalId', $build->getInternalId());
             $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment);
@@ -120,91 +136,307 @@ class BuildsV1 extends Worker
             $build = $dbForProject->getDocument('builds', $buildId);
         }
 
-        /** Request the executor to build the code... */
-        $build->setAttribute('status', 'building');
-        $build = $dbForProject->updateDocument('builds', $buildId, $build);
+        $source = $deployment->getAttribute('path', '');
+        $installationId = $deployment->getAttribute('installationId', '');
+        $providerRepositoryId = $deployment->getAttribute('providerRepositoryId', '');
+        $providerCommitHash = $deployment->getAttribute('providerCommitHash', '');
+        $isVcsEnabled = $providerRepositoryId ? true : false;
+        $owner = '';
+        $repositoryName = '';
 
-        /** Trigger Webhook */
-        $deploymentModel = new Deployment();
+        if ($isVcsEnabled) {
+            $installation = $dbForConsole->getDocument('installations', $installationId);
+            $providerInstallationId = $installation->getAttribute('providerInstallationId');
+            $privateKey = App::getEnv('_APP_VCS_GITHUB_PRIVATE_KEY');
+            $githubAppId = App::getEnv('_APP_VCS_GITHUB_APP_ID');
 
-        $deploymentUpdate = new Event(Event::WEBHOOK_QUEUE_NAME, Event::WEBHOOK_CLASS_NAME);
-        $deploymentUpdate
-            ->setProject($project)
-            ->setEvent('functions.[functionId].deployments.[deploymentId].update')
-            ->setParam('functionId', $function->getId())
-            ->setParam('deploymentId', $deployment->getId())
-            ->setPayload($deployment->getArrayCopy(array_keys($deploymentModel->getRules())))
-            ->trigger();
-
-        /** Trigger Functions */
-        $pools = $register->get('pools');
-        $connection = $pools->get('queue')->pop();
-
-        $functions = new Func($connection->getResource());
-        $functions
-            ->from($deploymentUpdate)
-            ->trigger();
-
-        $connection->reclaim();
-
-        /** Trigger Realtime */
-        $allEvents = Event::generateEvents('functions.[functionId].deployments.[deploymentId].update', [
-            'functionId' => $function->getId(),
-            'deploymentId' => $deployment->getId()
-        ]);
-        $target = Realtime::fromPayload(
-            // Pass first, most verbose event pattern
-            event: $allEvents[0],
-            payload: $build,
-            project: $project
-        );
-
-        Realtime::send(
-            projectId: 'console',
-            payload: $build->getArrayCopy(),
-            events: $allEvents,
-            channels: $target['channels'],
-            roles: $target['roles']
-        );
-
-        $source = $deployment->getAttribute('path');
-
-        $vars = array_reduce($function->getAttribute('vars', []), function (array $carry, Document $var) {
-            $carry[$var->getAttribute('key')] = $var->getAttribute('value');
-            return $carry;
-        }, []);
+            $github->initializeVariables($providerInstallationId, $privateKey, $githubAppId);
+        }
 
         try {
-            $response = $this->executor->createRuntime(
-                deploymentId: $deployment->getId(),
-                projectId: $project->getId(),
-                source: $source,
-                image: $runtime['image'],
-                remove: true,
-                entrypoint: $deployment->getAttribute('entrypoint'),
-                workdir: '/usr/code',
-                destination: APP_STORAGE_BUILDS . "/app-{$project->getId()}",
-                variables: $vars,
-                commands: [
-                    'sh', '-c',
-                    'tar -zxf /tmp/code.tar.gz -C /usr/code && \
-                    cd /usr/local/src/ && ./build.sh'
-                ]
+            if ($isNewBuild && $isVcsEnabled) {
+                $tmpDirectory = '/tmp/builds/' . $buildId . '/code';
+                $rootDirectory = $function->getAttribute('providerRootDirectory', '');
+                $rootDirectory = \rtrim($rootDirectory, '/');
+                $rootDirectory = \ltrim($rootDirectory, '.');
+                $rootDirectory = \ltrim($rootDirectory, '/');
+
+                $owner = $github->getOwnerName($providerInstallationId);
+                $repositoryName = $github->getRepositoryName($providerRepositoryId);
+
+                $cloneOwner = $deployment->getAttribute('providerRepositoryOwner', $owner);
+                $cloneRepository = $deployment->getAttribute('providerRepositoryName', $repositoryName);
+
+                $branchName = $deployment->getAttribute('providerBranch');
+                $commitHash = $deployment->getAttribute('providerCommitHash', '');
+                $gitCloneCommand = $github->generateCloneCommand($cloneOwner, $cloneRepository, $branchName, $tmpDirectory, $rootDirectory, $commitHash);
+                $stdout = '';
+                $stderr = '';
+                Console::execute('mkdir -p /tmp/builds/' . \escapeshellcmd($buildId), '', $stdout, $stderr);
+                $exit = Console::execute($gitCloneCommand, '', $stdout, $stderr);
+
+                if ($exit !== 0) {
+                    throw new \Exception('Unable to clone code repository: ' . $stderr);
+                }
+
+                // Build from template
+                $templateRepositoryName = $template->getAttribute('repositoryName', '');
+                $templateOwnerName = $template->getAttribute('ownerName', '');
+                $templateBranch = $template->getAttribute('branch', '');
+
+                $templateRootDirectory =  $template->getAttribute('rootDirectory', '');
+                $templateRootDirectory = \rtrim($templateRootDirectory, '/');
+                $templateRootDirectory = \ltrim($templateRootDirectory, '.');
+                $templateRootDirectory = \ltrim($templateRootDirectory, '/');
+
+                if (!empty($templateRepositoryName) && !empty($templateOwnerName) && !empty($templateBranch)) {
+                    // Clone template repo
+                    $tmpTemplateDirectory = '/tmp/builds/' . \escapeshellcmd($buildId) . '/template';
+                    $gitCloneCommandForTemplate = $github->generateCloneCommand($templateOwnerName, $templateRepositoryName, $templateBranch, $tmpTemplateDirectory, $templateRootDirectory);
+                    $exit = Console::execute($gitCloneCommandForTemplate, '', $stdout, $stderr);
+
+                    if ($exit !== 0) {
+                        throw new \Exception('Unable to clone code repository: ' . $stderr);
+                    }
+
+                    // Ensure directories
+                    Console::execute('mkdir -p ' . $tmpTemplateDirectory . '/' . $templateRootDirectory, '', $stdout, $stderr);
+                    Console::execute('mkdir -p ' . $tmpDirectory . '/' . $rootDirectory, '', $stdout, $stderr);
+
+                    // Merge template into user repo
+                    Console::execute('cp -rfn ' . $tmpTemplateDirectory . '/' . $templateRootDirectory . '/* ' . $tmpDirectory . '/' . $rootDirectory, '', $stdout, $stderr);
+
+                    // Commit and push
+                    $exit = Console::execute('git config --global user.email "team@appwrite.io" && git config --global user.name "Appwrite" && cd ' . $tmpDirectory . ' && git add . && git commit -m "Create \'' . \escapeshellcmd($function->getAttribute('name', '')) .  '\' function" && git push origin ' . \escapeshellcmd($branchName), '', $stdout, $stderr);
+
+                    if ($exit !== 0) {
+                        throw new \Exception('Unable to push code repository: ' . $stderr);
+                    }
+
+                    $exit = Console::execute('cd ' . $tmpDirectory . ' && git rev-parse HEAD', '', $stdout, $stderr);
+
+                    if ($exit !== 0) {
+                        throw new \Exception('Unable to get vcs commit SHA: ' . $stderr);
+                    }
+
+                    $providerCommitHash = \trim($stdout);
+                    $authorUrl = "https://github.com/$cloneOwner";
+
+                    $deployment->setAttribute('providerCommitHash', $providerCommitHash ?? '');
+                    $deployment->setAttribute('providerCommitAuthorUrl', $authorUrl);
+                    $deployment->setAttribute('providerCommitAuthor', 'Appwrite');
+                    $deployment->setAttribute('providerCommitMessage', "Create '" .  $function->getAttribute('name', '') . "' function");
+                    $deployment->setAttribute('providerCommitUrl', "https://github.com/$cloneOwner/$cloneRepository/commit/$providerCommitHash");
+                    $deployment = $dbForProject->updateDocument('deployments', $deployment->getId(), $deployment);
+
+                    /**
+                     * Send realtime Event
+                     */
+                    $target = Realtime::fromPayload(
+                        // Pass first, most verbose event pattern
+                        event: $allEvents[0],
+                        payload: $build,
+                        project: $project
+                    );
+                    Realtime::send(
+                        projectId: 'console',
+                        payload: $build->getArrayCopy(),
+                        events: $allEvents,
+                        channels: $target['channels'],
+                        roles: $target['roles']
+                    );
+                }
+
+                Console::execute('tar --exclude code.tar.gz -czf /tmp/builds/' . \escapeshellcmd($buildId) . '/code.tar.gz -C /tmp/builds/' . \escapeshellcmd($buildId) . '/code' . (empty($rootDirectory) ? '' : '/' . $rootDirectory) . ' .', '', $stdout, $stderr);
+
+                $deviceFunctions = $this->getFunctionsDevice($project->getId());
+
+                $fileName = 'code.tar.gz';
+                $fileTmpName = '/tmp/builds/' . $buildId . '/code.tar.gz';
+
+                $path = $deviceFunctions->getPath($deployment->getId() . '.' . \pathinfo($fileName, PATHINFO_EXTENSION));
+
+                $result = $deviceFunctions->move($fileTmpName, $path);
+
+                if (!$result) {
+                    throw new \Exception("Unable to move file");
+                }
+
+                Console::execute('rm -rf /tmp/builds/' . \escapeshellcmd($buildId), '', $stdout, $stderr);
+
+                $source = $path;
+
+                $build = $dbForProject->updateDocument('builds', $build->getId(), $build->setAttribute('source', $source));
+
+                $this->runGitAction('processing', $github, $providerCommitHash, $owner, $repositoryName, $project, $function, $deployment->getId(), $dbForProject, $dbForConsole);
+            }
+
+            /** Request the executor to build the code... */
+            $build->setAttribute('status', 'building');
+            $build = $dbForProject->updateDocument('builds', $buildId, $build);
+
+            if ($isVcsEnabled) {
+                $this->runGitAction('building', $github, $providerCommitHash, $owner, $repositoryName, $project, $function, $deployment->getId(), $dbForProject, $dbForConsole);
+            }
+
+            /** Trigger Webhook */
+            $deploymentModel = new Deployment();
+
+            $deploymentUpdate = new Event(Event::WEBHOOK_QUEUE_NAME, Event::WEBHOOK_CLASS_NAME);
+            $deploymentUpdate
+                ->setProject($project)
+                ->setEvent('functions.[functionId].deployments.[deploymentId].update')
+                ->setParam('functionId', $function->getId())
+                ->setParam('deploymentId', $deployment->getId())
+                ->setPayload($deployment->getArrayCopy(array_keys($deploymentModel->getRules())))
+                ->trigger();
+
+            /** Trigger Functions */
+            $pools = $register->get('pools');
+            $connection = $pools->get('queue')->pop();
+
+            $functions = new Func($connection->getResource());
+            $functions
+                ->from($deploymentUpdate)
+                ->trigger();
+
+            $connection->reclaim();
+
+            /** Trigger Realtime */
+            $target = Realtime::fromPayload(
+                // Pass first, most verbose event pattern
+                event: $allEvents[0],
+                payload: $build,
+                project: $project
             );
 
-            $endTime = new \DateTime();
-            $endTime->setTimestamp($response['endTimeUnix']);
+            Realtime::send(
+                projectId: 'console',
+                payload: $build->getArrayCopy(),
+                events: $allEvents,
+                channels: $target['channels'],
+                roles: $target['roles']
+            );
+
+            $vars = [];
+
+            // Global vars
+            $varsFromProject = $dbForProject->find('variables', [
+                Query::equal('resourceType', ['project']),
+                Query::limit(APP_LIMIT_SUBQUERY)
+            ]);
+
+            foreach ($varsFromProject as $var) {
+                $vars[$var->getAttribute('key')] = $var->getAttribute('value') ?? '';
+            }
+
+            // Function vars
+            $vars = \array_merge($vars, array_reduce($function->getAttribute('vars', []), function (array $carry, Document $var) {
+                $carry[$var->getAttribute('key')] = $var->getAttribute('value');
+                return $carry;
+            }, []));
+
+            // Appwrite vars
+            $vars = \array_merge($vars, [
+                'APPWRITE_FUNCTION_ID' => $function->getId(),
+                'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name'),
+                'APPWRITE_FUNCTION_DEPLOYMENT' => $deployment->getId(),
+                'APPWRITE_FUNCTION_PROJECT_ID' => $project->getId(),
+                'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'] ?? '',
+                'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'] ?? '',
+            ]);
+
+            $command = $deployment->getAttribute('commands', '');
+            $command = \str_replace('"', '\\"', $command);
+
+            $response = null;
+
+            $err = null;
+
+            // TODO: Remove run() wrapper when switching to new utopia queue. That should be done on Swoole adapter in the libary
+            Co\run(function () use ($project, $deployment, &$response, $source, $function, $runtime, $vars, $command, &$build, $dbForProject, $allEvents, &$err) {
+                Co::join([
+                    Co\go(function () use (&$response, $project, $deployment, $source, $function, $runtime, $vars, $command, &$err) {
+                        try {
+                            $response = $this->executor->createRuntime(
+                                deploymentId: $deployment->getId(),
+                                projectId: $project->getId(),
+                                source: $source,
+                                image: $runtime['image'],
+                                version: $function->getAttribute('version'),
+                                remove: true,
+                                entrypoint: $deployment->getAttribute('entrypoint'),
+                                destination: APP_STORAGE_BUILDS . "/app-{$project->getId()}",
+                                variables: $vars,
+                                command: 'tar -zxf /tmp/code.tar.gz -C /mnt/code && helpers/build.sh "' . $command . '"'
+                            );
+                        } catch (Exception $error) {
+                            $err = $error;
+                        }
+                    }),
+                    Co\go(function () use ($project, $deployment, &$response, &$build, $dbForProject, $allEvents, &$err) {
+                        try {
+                            $this->executor->getLogs(
+                                deploymentId: $deployment->getId(),
+                                projectId: $project->getId(),
+                                callback: function ($logs) use (&$response, &$build, $dbForProject, $allEvents, $project) {
+                                    if ($response === null) {
+                                        $build = $dbForProject->getDocument('builds', $build->getId());
+
+                                        if ($build->isEmpty()) {
+                                            throw new Exception('Build not found', 404);
+                                        }
+
+                                        $build = $build->setAttribute('logs', $build->getAttribute('logs', '') . $logs);
+                                        $build = $dbForProject->updateDocument('builds', $build->getId(), $build);
+
+                                        /**
+                                         * Send realtime Event
+                                         */
+                                        $target = Realtime::fromPayload(
+                                            // Pass first, most verbose event pattern
+                                            event: $allEvents[0],
+                                            payload: $build,
+                                            project: $project
+                                        );
+                                        Realtime::send(
+                                            projectId: 'console',
+                                            payload: $build->getArrayCopy(),
+                                            events: $allEvents,
+                                            channels: $target['channels'],
+                                            roles: $target['roles']
+                                        );
+                                    }
+                                }
+                            );
+                        } catch (Exception $error) {
+                            if (empty($err)) {
+                                $err = $error;
+                            }
+                        }
+                    }),
+                ]);
+            });
+
+            if ($err) {
+                throw $err;
+            }
+
+            $endTime = DateTime::now();
+            $durationEnd = \microtime(true);
 
             /** Update the build document */
-            $build->setAttribute('endTime', DateTime::format($endTime));
-            $build->setAttribute('duration', \intval($response['duration']));
-            $build->setAttribute('status', $response['status']);
-            $build->setAttribute('outputPath', $response['outputPath']);
-            $build->setAttribute('stderr', $response['stderr']);
-            $build->setAttribute('stdout', $response['stdout']);
+            $build->setAttribute('startTime', DateTime::format((new \DateTime())->setTimestamp($response['startTime'])));
+            $build->setAttribute('endTime', $endTime);
+            $build->setAttribute('duration', \intval(\ceil($durationEnd - $durationStart)));
+            $build->setAttribute('status', 'ready');
+            $build->setAttribute('path', $response['path']);
+            $build->setAttribute('size', $response['size']);
+            $build->setAttribute('logs', $response['output']);
 
-            /* Also update the deployment buildTime */
-            $deployment->setAttribute('buildTime', $response['duration']);
+            if ($isVcsEnabled) {
+                $this->runGitAction('ready', $github, $providerCommitHash, $owner, $repositoryName, $project, $function, $deployment->getId(), $dbForProject, $dbForConsole);
+            }
 
             Console::success("Build id: $buildId created");
 
@@ -212,28 +444,33 @@ class BuildsV1 extends Worker
             if ($deployment->getAttribute('activate') === true) {
                 $function->setAttribute('deploymentInternalId', $deployment->getInternalId());
                 $function->setAttribute('deployment', $deployment->getId());
+                $function->setAttribute('live', true);
                 $function = $dbForProject->updateDocument('functions', $function->getId(), $function);
             }
 
             /** Update function schedule */
             $dbForConsole = $this->getConsoleDB();
+            // Inform scheduler if function is still active
             $schedule = $dbForConsole->getDocument('schedules', $function->getAttribute('scheduleId'));
-            $schedule->setAttribute('resourceUpdatedAt', DateTime::now());
-
             $schedule
+                ->setAttribute('resourceUpdatedAt', DateTime::now())
                 ->setAttribute('schedule', $function->getAttribute('schedule'))
                 ->setAttribute('active', !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment')));
-
-
             Authorization::skip(fn () => $dbForConsole->updateDocument('schedules', $schedule->getId(), $schedule));
         } catch (\Throwable $th) {
             $endTime = DateTime::now();
-            $interval = (new \DateTime($endTime))->diff(new \DateTime($startTime));
+            $durationEnd = \microtime(true);
             $build->setAttribute('endTime', $endTime);
-            $build->setAttribute('duration', $interval->format('%s') + 0);
+            $build->setAttribute('duration', \intval(\ceil($durationEnd - $durationStart)));
             $build->setAttribute('status', 'failed');
-            $build->setAttribute('stderr', $th->getMessage());
+            $build->setAttribute('logs', $th->getMessage());
             Console::error($th->getMessage());
+            Console::error($th->getFile() . ':' . $th->getLine());
+            Console::error($th->getTraceAsString());
+
+            if ($isVcsEnabled) {
+                $this->runGitAction('failed', $github, $providerCommitHash, $owner, $repositoryName, $project, $function, $deployment->getId(), $dbForProject, $dbForConsole);
+            }
         } finally {
             $build = $dbForProject->updateDocument('builds', $buildId, $build);
 
@@ -241,7 +478,7 @@ class BuildsV1 extends Worker
              * Send realtime Event
              */
             $target = Realtime::fromPayload(
-                // Pass first, most verbose event pattern
+            // Pass first, most verbose event pattern
                 event: $allEvents[0],
                 payload: $build,
                 project: $project
@@ -253,20 +490,93 @@ class BuildsV1 extends Worker
                 channels: $target['channels'],
                 roles: $target['roles']
             );
+
+            /** Update usage stats */
+            if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') {
+                $statsd = $register->get('statsd');
+                $usage = new Stats($statsd);
+                $usage
+                    ->setParam('projectInternalId', $project->getInternalId())
+                    ->setParam('projectId', $project->getId())
+                    ->setParam('functionId', $function->getId())
+                    ->setParam('builds.{scope}.compute', 1)
+                    ->setParam('buildStatus', $build->getAttribute('status', ''))
+                    ->setParam('buildTime', $build->getAttribute('duration'))
+                    ->setParam('networkRequestSize', 0)
+                    ->setParam('networkResponseSize', 0)
+                    ->submit();
+            }
+        }
+    }
+
+    protected function runGitAction(string $status, GitHub $github, string $providerCommitHash, string $owner, string $repositoryName, Document $project, Document $function, string $deploymentId, Database $dbForProject, Database $dbForConsole): void
+    {
+        if ($function->getAttribute('providerSilentMode', false) === true) {
+            return;
         }
 
-        /** Trigger usage queue */
-        $this
-            ->getUsageQueue()
-            ->setProject($project)
-            ->addMetric(METRIC_BUILDS, 1) // per project
-            ->addMetric(METRIC_BUILDS_STORAGE, $build->getAttribute('size', 0))
-            ->addMetric(METRIC_BUILDS_COMPUTE, (int)$build->getAttribute('duration', 0) * 1000)
-            ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS), 1) // per function
-            ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE), $build->getAttribute('size', 0))
-            ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE), (int)$build->getAttribute('duration', 0) * 1000)
-            ->trigger()
-        ;
+        $deployment = $dbForProject->getDocument('deployments', $deploymentId);
+        $commentId = $deployment->getAttribute('providerCommentId', '');
+
+        if (!empty($providerCommitHash)) {
+            $message = match ($status) {
+                'ready' => 'Build succeeded.',
+                'failed' => 'Build failed.',
+                'processing' => 'Building...',
+                default => $status
+            };
+
+            $state = match ($status) {
+                'ready' => 'success',
+                'failed' => 'failure',
+                'processing' => 'pending',
+                default => $status
+            };
+
+            $functionName = $function->getAttribute('name');
+            $projectName = $project->getAttribute('name');
+
+            $name = "{$functionName} ({$projectName})";
+
+            $protocol = App::getEnv('_APP_OPTIONS_FORCE_HTTPS') == 'disabled' ? 'http' : 'https';
+            $hostname = App::getEnv('_APP_DOMAIN');
+            $functionId = $function->getId();
+            $projectId = $project->getId();
+            $providerTargetUrl = $protocol . '://' . $hostname . "/console/project-$projectId/functions/function-$functionId";
+
+            $github->updateCommitStatus($repositoryName, $providerCommitHash, $owner, $state, $message, $providerTargetUrl, $name);
+        }
+
+        if (!empty($commentId)) {
+            $retries = 0;
+
+            while (true) {
+                $retries++;
+
+                try {
+                    $dbForConsole->createDocument('vcsCommentLocks', new Document([
+                        '$id' => $commentId
+                    ]));
+                    break;
+                } catch (Exception $err) {
+                    if ($retries >= 9) {
+                        throw $err;
+                    }
+
+                    \sleep(1);
+                }
+            }
+
+            // Wrap in try/finally to ensure lock file gets deleted
+            try {
+                $comment = new Comment();
+                $comment->parseComment($github->getComment($owner, $repositoryName, $commentId));
+                $comment->addBuild($project, $function, $status, $deployment->getId(), ['type' => 'logs']);
+                $github->updateComment($owner, $repositoryName, $commentId, $comment->generateComment());
+            } finally {
+                $dbForConsole->deleteDocument('vcsCommentLocks', $commentId);
+            }
+        }
     }
 
     public function shutdown(): void

app/workers/certificates.php

@@ -2,6 +2,9 @@
 
 use Appwrite\Event\Mail;
 use Appwrite\Network\Validator\CNAME;
+use Appwrite\Utopia\Response\Model\Rule;
+use Appwrite\Messaging\Adapter\Realtime;
+use Appwrite\Event\Event;
 use Appwrite\Resque\Worker;
 use Appwrite\Template\Template;
 use Utopia\App;
@@ -46,7 +49,7 @@ class CertificatesV1 extends Worker
          * 4. Validate security email. Cannot be empty, required by LetsEncrypt
          * 5. Validate renew date with certificate file, unless requested to skip by parameter
          * 6. Issue a certificate using certbot CLI
-         * 7. Update 'log' attribute on certificate document with Certbot message
+         * 7. Update 'logs' attribute on certificate document with Certbot message
          * 8. Create storage folder for certificate, if not ready already
          * 9. Move certificates from Certbot location to our Storage
          * 10. Create/Update our Storage with new Traefik config with new certificate paths
@@ -56,7 +59,7 @@ class CertificatesV1 extends Worker
          * If at any point unexpected error occurs, program stops without applying changes to document, and error is thrown into worker
          *
          * If code stops with expected error:
-         * 1. 'log' attribute on document is updated with error message
+         * 1. 'logs' attribute on document is updated with error message
          * 2. 'attempts' amount is increased
          * 3. Console log is shown
          * 4. Email is sent to security email
@@ -84,6 +87,8 @@ class CertificatesV1 extends Worker
             $certificate->setAttribute('domain', $domain->get());
         }
 
+        $success = false;
+
         try {
             // Email for alerts is required by LetsEncrypt
             $email = App::getEnv('_APP_SYSTEM_SECURITY_EMAIL_ADDRESS');
@@ -110,11 +115,8 @@ class CertificatesV1 extends Worker
             $letsEncryptData = $this->issueCertificate($folder, $domain->get(), $email);
 
             // Command succeeded, store all data into document
-            // We store stderr too, because it may include warnings
-            $certificate->setAttribute('log', \json_encode([
-                'stdout' => $letsEncryptData['stdout'],
-                'stderr' => $letsEncryptData['stderr'],
-            ]));
+            $logs = 'Certificate successfully generated.';
+            $certificate->setAttribute('logs', \mb_strcut($logs, 0, 1000000));// Limit to 1MB
 
             // Give certificates to Traefik
             $this->applyCertificateFiles($folder, $domain->get(), $letsEncryptData);
@@ -123,9 +125,13 @@ class CertificatesV1 extends Worker
             $certificate->setAttribute('renewDate', $this->getRenewDate($domain->get()));
             $certificate->setAttribute('attempts', 0);
             $certificate->setAttribute('issueDate', DateTime::now());
+
+            $success = true;
         } catch (Throwable $e) {
+            $logs = $e->getMessage();
+
             // Set exception as log in certificate document
-            $certificate->setAttribute('log', $e->getMessage());
+            $certificate->setAttribute('logs', \mb_strcut($logs, 0, 1000000));// Limit to 1MB
 
             // Increase attempts count
             $attempts = $certificate->getAttribute('attempts', 0) + 1;
@@ -141,7 +147,7 @@ class CertificatesV1 extends Worker
             $certificate->setAttribute('updated', DateTime::now());
 
             // Save all changes we made to certificate document into database
-            $this->saveCertificateDocument($domain->get(), $certificate);
+            $this->saveCertificateDocument($domain->get(), $certificate, $success);
         }
     }
 
@@ -154,10 +160,11 @@ class CertificatesV1 extends Worker
      *
      * @param string $domain Domain name that certificate is for
      * @param Document $certificate Certificate document that we need to save
+     * @param bool $success Was certificate generation successful?
      *
      * @return void
      */
-    private function saveCertificateDocument(string $domain, Document $certificate): void
+    private function saveCertificateDocument(string $domain, Document $certificate, bool $success): void
     {
         // Check if update or insert required
         $certificateDocument = $this->dbForConsole->findOne('certificates', [Query::equal('domain', [$domain])]);
@@ -171,7 +178,7 @@ class CertificatesV1 extends Worker
         }
 
         $certificateId = $certificate->getId();
-        $this->updateDomainDocuments($certificateId, $domain);
+        $this->updateDomainDocuments($certificateId, $domain, $success);
     }
 
     /**
@@ -184,11 +191,6 @@ class CertificatesV1 extends Worker
         $envDomain = App::getEnv('_APP_DOMAIN', '');
         if (!empty($envDomain) && $envDomain !== 'localhost') {
             return $envDomain;
-        } else {
-            $domainDocument = $this->dbForConsole->findOne('domains', [Query::orderAsc('_id')]);
-            if ($domainDocument) {
-                return $domainDocument->getAttribute('domain');
-            }
         }
 
         return null;
@@ -279,7 +281,7 @@ class CertificatesV1 extends Worker
         $stderr = '';
 
         $staging = (App::isProduction()) ? '' : ' --dry-run';
-        $exit = Console::execute("certbot certonly --webroot --noninteractive --agree-tos{$staging}"
+        $exit = Console::execute("certbot certonly -v --webroot --noninteractive --agree-tos{$staging}"
             . " --email " . $email
             . " --cert-name " . $folder
             . " -w " . APP_STORAGE_CERTIFICATES
@@ -420,25 +422,64 @@ class CertificatesV1 extends Worker
      *
      * @param string $certificateId ID of a new or updated certificate document
      * @param string $domain Domain that is affected by new certificate
+     * @param bool $success Was certificate generation successful?
      *
      * @return void
      */
-    private function updateDomainDocuments(string $certificateId, string $domain): void
+    private function updateDomainDocuments(string $certificateId, string $domain, bool $success): void
     {
-        $domains = $this->dbForConsole->find('domains', [
+        $rule = $this->dbForConsole->findOne('rules', [
             Query::equal('domain', [$domain]),
-            Query::limit(1000),
         ]);
 
-        foreach ($domains as $domainDocument) {
-            $domainDocument->setAttribute('updated', DateTime::now());
-            $domainDocument->setAttribute('certificateId', $certificateId);
+        if ($rule !== false && !$rule->isEmpty()) {
+            $rule->setAttribute('certificateId', $certificateId);
+            $rule->setAttribute('status', $success ? 'verified' : 'unverified');
+            $this->dbForConsole->updateDocument('rules', $rule->getId(), $rule);
 
-            $this->dbForConsole->updateDocument('domains', $domainDocument->getId(), $domainDocument);
+            $projectId = $rule->getAttribute('projectId');
+            $project = $this->dbForConsole->getDocument('projects', $projectId);
 
-            if ($domainDocument->getAttribute('projectId')) {
-                $this->dbForConsole->deleteCachedDocument('projects', $domainDocument->getAttribute('projectId'));
-            }
+            /** Trigger Webhook */
+            $ruleModel = new Rule();
+            $ruleUpdate = new Event(Event::WEBHOOK_QUEUE_NAME, Event::WEBHOOK_CLASS_NAME);
+            $ruleUpdate
+                ->setProject($project)
+                ->setEvent('rules.[ruleId].update')
+                ->setParam('ruleId', $rule->getId())
+                ->setPayload($rule->getArrayCopy(array_keys($ruleModel->getRules())))
+                ->trigger();
+
+            /** Trigger Functions */
+            $ruleUpdate
+                ->setClass(Event::FUNCTIONS_CLASS_NAME)
+                ->setQueue(Event::FUNCTIONS_QUEUE_NAME)
+                ->trigger();
+
+            /** Trigger realtime event */
+            $allEvents = Event::generateEvents('rules.[ruleId].update', [
+                'ruleId' => $rule->getId(),
+            ]);
+            $target = Realtime::fromPayload(
+                // Pass first, most verbose event pattern
+                event: $allEvents[0],
+                payload: $rule,
+                project: $project
+            );
+            Realtime::send(
+                projectId: 'console',
+                payload: $rule->getArrayCopy(),
+                events: $allEvents,
+                channels: $target['channels'],
+                roles: $target['roles']
+            );
+            Realtime::send(
+                projectId: $project->getId(),
+                payload: $rule->getArrayCopy(),
+                events: $allEvents,
+                channels: $target['channels'],
+                roles: $target['roles']
+            );
         }
     }
 }

app/workers/deletes.php

@@ -1,6 +1,7 @@
 <?php
 
 use Appwrite\Auth\Auth;
+use Executor\Executor;
 use Utopia\App;
 use Utopia\Cache\Adapter\Filesystem;
 use Utopia\Cache\Cache;
@@ -66,6 +67,12 @@ class DeletesV1 extends Worker
                     case DELETE_TYPE_BUCKETS:
                         $this->deleteBucket($document, $project);
                         break;
+                    case DELETE_TYPE_INSTALLATIONS:
+                        $this->deleteInstallation($document, $project);
+                        break;
+                    case DELETE_TYPE_RULES:
+                        $this->deleteRule($document, $project);
+                        break;
                     default:
                         if (\str_starts_with($document->getCollection(), 'database_')) {
                             $this->deleteCollection($document, $project);
@@ -106,11 +113,6 @@ class DeletesV1 extends Worker
                 $this->deleteExpiredSessions();
                 break;
 
-            case DELETE_TYPE_CERTIFICATES:
-                $document = new Document($this->args['document']);
-                $this->deleteCertificates($document);
-                break;
-
             case DELETE_TYPE_USAGE:
                 $this->deleteUsageStats($this->args['hourlyUsageRetentionDatetime']);
                 break;
@@ -589,15 +591,29 @@ class DeletesV1 extends Worker
     {
         $projectId = $project->getId();
         $dbForProject = $this->getProjectDB($project);
+        $dbForConsole = $this->getConsoleDB();
         $functionId = $document->getId();
         $functionInternalId = $document->getInternalId();
 
+        /**
+         * Delete rules
+         */
+        Console::info("Deleting rules for function " . $functionId);
+        $this->deleteByGroup('rules', [
+            Query::equal('resourceType', ['function']),
+            Query::equal('resourceInternalId', [$functionInternalId]),
+            Query::equal('projectInternalId', [$project->getInternalId()])
+        ], $dbForConsole, function (Document $document) use ($project) {
+            $this->deleteRule($document, $project);
+        });
+
         /**
          * Delete Variables
          */
         Console::info("Deleting variables for function " . $functionId);
         $this->deleteByGroup('variables', [
-            Query::equal('functionInternalId', [$functionInternalId])
+            Query::equal('resourceType', ['function']),
+            Query::equal('resourceInternalId', [$functionInternalId])
         ], $dbForProject);
 
         /**
@@ -605,11 +621,11 @@ class DeletesV1 extends Worker
          */
         Console::info("Deleting deployments for function " . $functionId);
         $storageFunctions = $this->getFunctionsDevice($projectId);
-        $deploymentIds = [];
+        $deploymentInternalIds = [];
         $this->deleteByGroup('deployments', [
-            Query::equal('resourceId', [$functionId])
-        ], $dbForProject, function (Document $document) use ($storageFunctions, &$deploymentIds) {
-            $deploymentIds[] = $document->getId();
+            Query::equal('resourceInternalId', [$functionInternalId])
+        ], $dbForProject, function (Document $document) use ($storageFunctions, &$deploymentInternalIds) {
+            $deploymentInternalIds[] = $document->getInternalId();
             if ($storageFunctions->delete($document->getAttribute('path', ''), true)) {
                 Console::success('Deleted deployment files: ' . $document->getAttribute('path', ''));
             } else {
@@ -622,14 +638,14 @@ class DeletesV1 extends Worker
          */
         Console::info("Deleting builds for function " . $functionId);
         $storageBuilds = $this->getBuildsDevice($projectId);
-        foreach ($deploymentIds as $deploymentId) {
+        foreach ($deploymentInternalIds as $deploymentInternalId) {
             $this->deleteByGroup('builds', [
-                Query::equal('deploymentId', [$deploymentId])
-            ], $dbForProject, function (Document $document) use ($storageBuilds, $deploymentId) {
-                if ($storageBuilds->delete($document->getAttribute('outputPath', ''), true)) {
-                    Console::success('Deleted build files: ' . $document->getAttribute('outputPath', ''));
+                Query::equal('deploymentInternalId', [$deploymentInternalId])
+            ], $dbForProject, function (Document $document) use ($storageBuilds) {
+                if ($storageBuilds->delete($document->getAttribute('path', ''), true)) {
+                    Console::success('Deleted build files: ' . $document->getAttribute('path', ''));
                 } else {
-                    Console::error('Failed to delete build files: ' . $document->getAttribute('outputPath', ''));
+                    Console::error('Failed to delete build files: ' . $document->getAttribute('path', ''));
                 }
             });
         }
@@ -639,10 +655,14 @@ class DeletesV1 extends Worker
          */
         Console::info("Deleting executions for function " . $functionId);
         $this->deleteByGroup('executions', [
-            Query::equal('functionId', [$functionId])
+            Query::equal('functionInternalId', [$functionInternalId])
         ], $dbForProject);
 
-        // TODO: Request executor to delete runtime
+        /**
+         * Request executor to delete all deployment containers
+         */
+        Console::info("Requesting executor to delete all deployment containers for function " . $functionId);
+        $this->deleteRuntimes($document, $project);
     }
 
     /**
@@ -654,7 +674,7 @@ class DeletesV1 extends Worker
         $projectId = $project->getId();
         $dbForProject = $this->getProjectDB($project);
         $deploymentId = $document->getId();
-        $functionId = $document->getAttribute('resourceId');
+        $deploymentInternalId = $document->getInternalId();
 
         /**
          * Delete deployment files
@@ -673,16 +693,21 @@ class DeletesV1 extends Worker
         Console::info("Deleting builds for deployment " . $deploymentId);
         $storageBuilds = $this->getBuildsDevice($projectId);
         $this->deleteByGroup('builds', [
-            Query::equal('deploymentId', [$deploymentId])
+            Query::equal('deploymentInternalId', [$deploymentInternalId])
         ], $dbForProject, function (Document $document) use ($storageBuilds) {
-            if ($storageBuilds->delete($document->getAttribute('outputPath', ''), true)) {
-                Console::success('Deleted build files: ' . $document->getAttribute('outputPath', ''));
+            if ($storageBuilds->delete($document->getAttribute('path', ''), true)) {
+                Console::success('Deleted build files: ' . $document->getAttribute('path', ''));
             } else {
-                Console::error('Failed to delete build files: ' . $document->getAttribute('outputPath', ''));
+                Console::error('Failed to delete build files: ' . $document->getAttribute('path', ''));
             }
         });
 
-        // TODO: Request executor to delete runtime
+
+        /**
+         * Request executor to delete all deployment containers
+         */
+        Console::info("Requesting executor to delete deployment container for deployment " . $deploymentId);
+        $this->deleteRuntimes($document, $project);
     }
 
 
@@ -833,44 +858,18 @@ class DeletesV1 extends Worker
     }
 
     /**
-     * @param Document $document certificates document
-     * @throws \Utopia\Database\Exception\Authorization
+     * @param Document $document rule document
+     * @param Document $project project document
      */
-    protected function deleteCertificates(Document $document): void
+    protected function deleteRule(Document $document, Document $project): void
     {
         $consoleDB = $this->getConsoleDB();
 
-        // If domain has certificate generated
-        if (isset($document['certificateId'])) {
-            $domainUsingCertificate = $consoleDB->findOne('domains', [
-                Query::equal('certificateId', [$document['certificateId']])
-            ]);
-
-            if (!$domainUsingCertificate) {
-                $mainDomain = App::getEnv('_APP_DOMAIN_TARGET', '');
-                if ($mainDomain === $document->getAttribute('domain')) {
-                    $domainUsingCertificate = $mainDomain;
-                }
-            }
-
-            // If certificate is still used by some domain, mark we can't delete.
-            // Current domain should not be found, because we only have copy. Original domain is already deleted from database.
-            if ($domainUsingCertificate) {
-                Console::warning("Skipping certificate deletion, because a domain is still using it.");
-                return;
-            }
-        }
-
         $domain = $document->getAttribute('domain');
         $directory = APP_STORAGE_CERTIFICATES . '/' . $domain;
         $checkTraversal = realpath($directory) === $directory;
 
-        if ($domain && $checkTraversal && is_dir($directory)) {
-            // Delete certificate document, so Appwrite is aware of change
-            if (isset($document['certificateId'])) {
-                $consoleDB->deleteDocument('certificates', $document['certificateId']);
-            }
-
+        if ($checkTraversal && is_dir($directory)) {
             // Delete files, so Traefik is aware of change
             array_map('unlink', glob($directory . '/*.*'));
             rmdir($directory);
@@ -878,6 +877,11 @@ class DeletesV1 extends Worker
         } else {
             Console::info("No certificate files found for {$domain}");
         }
+
+        // Delete certificate document, so Appwrite is aware of change
+        if (isset($document['certificateId'])) {
+            $consoleDB->deleteDocument('certificates', $document['certificateId']);
+        }
     }
 
     protected function deleteBucket(Document $document, Document $project)
@@ -890,4 +894,72 @@ class DeletesV1 extends Worker
 
         $device->deletePath($document->getId());
     }
+
+    protected function deleteInstallation(Document $document, Document $project)
+    {
+        $dbForProject = $this->getProjectDB($project);
+        $dbForConsole = $this->getConsoleDB();
+
+        $this->listByGroup('functions', [
+            Query::equal('installationInternalId', [$document->getInternalId()])
+        ], $dbForProject, function ($function) use ($dbForProject, $dbForConsole) {
+            $dbForConsole->deleteDocument('repositories', $function->getAttribute('repositoryId'));
+
+            $function = $function
+                ->setAttribute('installationId', '')
+                ->setAttribute('installationInternalId', '')
+                ->setAttribute('providerRepositoryId', '')
+                ->setAttribute('providerBranch', '')
+                ->setAttribute('providerSilentMode', false)
+                ->setAttribute('providerRootDirectory', '')
+                ->setAttribute('repositoryId', '')
+                ->setAttribute('repositoryInternalId', '');
+            $dbForProject->updateDocument('functions', $function->getId(), $function);
+        });
+    }
+
+    protected function deleteRuntimes(?Document $function, Document $project)
+    {
+        $executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST'));
+
+        $deleteByFunction = function (Document $function) use ($project, $executor) {
+            $this->listByGroup(
+                'deployments',
+                [
+                    Query::equal('resourceInternalId', [$function->getInternalId()]),
+                    Query::equal('resourceType', ['functions']),
+                ],
+                $this->getProjectDB($project),
+                function (Document $deployment) use ($project, $executor) {
+                    $deploymentId = $deployment->getId();
+
+                    try {
+                        $executor->deleteRuntime($project->getId(), $deploymentId);
+                        Console::info("Runtime for deployment {$deploymentId} deleted.");
+                    } catch (Throwable $th) {
+                        Console::warning("Runtime for deployment {$deploymentId} skipped:");
+                        Console::error('[Error] Type: ' . get_class($th));
+                        Console::error('[Error] Message: ' . $th->getMessage());
+                        Console::error('[Error] File: ' . $th->getFile());
+                        Console::error('[Error] Line: ' . $th->getLine());
+                    }
+                }
+            );
+        };
+
+        if ($function !== null) {
+            // Delete function runtimes
+            $deleteByFunction($function);
+        } else {
+            // Delete all project runtimes
+            $this->listByGroup(
+                'functions',
+                [],
+                $this->getProjectDB($project),
+                function (Document $function) use ($deleteByFunction) {
+                    $deleteByFunction($function);
+                }
+            );
+        }
+    }
 }

app/workers/functions.php

@@ -2,11 +2,12 @@
 
 require_once __DIR__ . '/../worker.php';
 
-use Appwrite\Event\Usage;
+use Domnikl\Statsd\Client;
 use Utopia\Queue\Message;
 use Appwrite\Event\Event;
 use Appwrite\Event\Func;
 use Appwrite\Messaging\Adapter\Realtime;
+use Appwrite\Usage\Stats;
 use Appwrite\Utopia\Response\Model\Execution;
 use Executor\Executor;
 use Utopia\App;
@@ -30,11 +31,14 @@ Server::setResource('execute', function () {
         Log $log,
         Func $queueForFunctions,
         Database $dbForProject,
-        Usage $queueForUsage,
+        Client $statsd,
         Document $project,
         Document $function,
         string $trigger,
         string $data = null,
+        string $path,
+        string $method,
+        array $headers,
         ?Document $user = null,
         string $jwt = null,
         string $event = null,
@@ -43,7 +47,6 @@ Server::setResource('execute', function () {
     ) {
         $user ??= new Document();
         $functionId = $function->getId();
-        $functionInternalId = $function->getInternalId();
         $deploymentId = $function->getAttribute('deployment', '');
 
         $log->addTag('functionId', $functionId);
@@ -51,7 +54,6 @@ Server::setResource('execute', function () {
 
         /** Check if deployment exists */
         $deployment = $dbForProject->getDocument('deployments', $deploymentId);
-        $deploymentInternalId = $deployment->getInternalId();
 
         if ($deployment->getAttribute('resourceId') !== $functionId) {
             throw new Exception('Deployment not found. Create deployment before trying to execute a function');
@@ -80,100 +82,146 @@ Server::setResource('execute', function () {
 
         $runtime = $runtimes[$function->getAttribute('runtime')];
 
+        $headers['x-appwrite-trigger'] = $trigger;
+        $headers['x-appwrite-event'] = $event ?? '';
+        $headers['x-appwrite-user-id'] = $user->getId() ?? '';
+        $headers['x-appwrite-user-jwt'] = $jwt ?? '';
+
         /** Create execution or update execution status */
         $execution = $dbForProject->getDocument('executions', $executionId ?? '');
         if ($execution->isEmpty()) {
+            $headersFiltered = [];
+            foreach ($headers as $key => $value) {
+                if (\in_array(\strtolower($key), FUNCTION_ALLOWLIST_HEADERS_REQUEST)) {
+                    $headersFiltered[] = [ 'name' => $key, 'value' => $value ];
+                }
+            }
+
             $executionId = ID::unique();
-            $execution = $dbForProject->createDocument('executions', new Document([
+            $execution = new Document([
                 '$id' => $executionId,
                 '$permissions' => $user->isEmpty() ? [] : [Permission::read(Role::user($user->getId()))],
-                'functionId' => $functionId,
-                'functionInternalId' => $functionInternalId,
-                'deploymentInternalId' => $deploymentInternalId,
-                'deploymentId' => $deploymentId,
-                'trigger' => $trigger,
-                'status' => 'waiting',
-                'statusCode' => 0,
-                'response' => '',
-                'stderr' => '',
+                'functionInternalId' => $function->getInternalId(),
+                'functionId' => $function->getId(),
+                'deploymentInternalId' => $deployment->getInternalId(),
+                'deploymentId' => $deployment->getId(),
+                'trigger' => 'http',
+                'status' => 'processing',
+                'responseStatusCode' => 0,
+                'responseHeaders' => [],
+                'requestPath' => $path,
+                'requestMethod' => $method,
+                'requestHeaders' => $headersFiltered,
+                'errors' => '',
+                'logs' => '',
                 'duration' => 0.0,
-                'search' => implode(' ', [$function->getId(), $executionId]),
-            ]));
+                'search' => implode(' ', [$functionId, $executionId]),
+            ]);
+
+            if ($function->getAttribute('logging')) {
+                $execution = $dbForProject->createDocument('executions', $execution);
+            }
 
             // TODO: @Meldiron Trigger executions.create event here
 
             if ($execution->isEmpty()) {
                 throw new Exception('Failed to create or read execution');
             }
-
-            /**
-             * Usage
-             */
-
-            $queueForUsage
-                ->addMetric(METRIC_EXECUTIONS, 1) // per project
-                ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS), 1); // per function
         }
 
-        $execution->setAttribute('status', 'processing');
-        $execution = $dbForProject->updateDocument('executions', $executionId, $execution);
+        if ($execution->getAttribute('status') !== 'processing') {
+            $execution->setAttribute('status', 'processing');
 
-        $vars = array_reduce($function->getAttribute('vars', []), function (array $carry, Document $var) {
+            if ($function->getAttribute('logging')) {
+                $execution = $dbForProject->updateDocument('executions', $executionId, $execution);
+            }
+        }
+
+        $durationStart = \microtime(true);
+
+        $vars = [];
+
+        // Shared vars
+        $varsShared = $project->getAttribute('variables', []);
+        $vars = \array_merge($vars, \array_reduce($varsShared, function (array $carry, Document $var) {
+            $carry[$var->getAttribute('key')] = $var->getAttribute('value') ?? '';
+            return $carry;
+        }, []));
+
+        // Function vars
+        $vars = \array_merge($vars, array_reduce($function->getAttribute('vars', []), function (array $carry, Document $var) {
             $carry[$var->getAttribute('key')] = $var->getAttribute('value');
             return $carry;
-        }, []);
+        }, []));
 
-        /** Collect environment variables */
+        // Appwrite vars
         $vars = \array_merge($vars, [
             'APPWRITE_FUNCTION_ID' => $functionId,
             'APPWRITE_FUNCTION_NAME' => $function->getAttribute('name'),
             'APPWRITE_FUNCTION_DEPLOYMENT' => $deploymentId,
-            'APPWRITE_FUNCTION_TRIGGER' => $trigger,
             'APPWRITE_FUNCTION_PROJECT_ID' => $project->getId(),
             'APPWRITE_FUNCTION_RUNTIME_NAME' => $runtime['name'] ?? '',
             'APPWRITE_FUNCTION_RUNTIME_VERSION' => $runtime['version'] ?? '',
-            'APPWRITE_FUNCTION_EVENT' => $event ?? '',
-            'APPWRITE_FUNCTION_EVENT_DATA' => $eventData ?? '',
-            'APPWRITE_FUNCTION_DATA' => $data ?? '',
-            'APPWRITE_FUNCTION_USER_ID' => $user->getId() ?? '',
-            'APPWRITE_FUNCTION_JWT' => $jwt ?? '',
         ]);
 
+        $body = $eventData ?? '';
+        if (empty($body)) {
+            $body = $data ?? '';
+        }
+
         /** Execute function */
         try {
-            $client = new Executor(App::getEnv('_APP_EXECUTOR_HOST'));
-            $executionResponse = $client->createExecution(
+            $command = $runtime['startCommand'];
+            $executor = new Executor(App::getEnv('_APP_EXECUTOR_HOST'));
+            $executionResponse = $executor->createExecution(
                 projectId: $project->getId(),
                 deploymentId: $deploymentId,
-                payload: $vars['APPWRITE_FUNCTION_DATA'] ?? '',
+                body: \strlen($body) > 0 ? $body : null,
                 variables: $vars,
                 timeout: $function->getAttribute('timeout', 0),
                 image: $runtime['image'],
-                source: $build->getAttribute('outputPath', ''),
+                source: $build->getAttribute('path', ''),
                 entrypoint: $deployment->getAttribute('entrypoint', ''),
+                version: $function->getAttribute('version'),
+                path: $path,
+                method: $method,
+                headers: $headers,
+                runtimeEntrypoint: 'cp /tmp/code.tar.gz /mnt/code/code.tar.gz && nohup helpers/start.sh "' . $command . '"'
             );
 
+            $status = $executionResponse['statusCode'] >= 400 ? 'failed' : 'completed';
+
+            $headersFiltered = [];
+            foreach ($executionResponse['headers'] as $key => $value) {
+                if (\in_array(\strtolower($key), FUNCTION_ALLOWLIST_HEADERS_RESPONSE)) {
+                    $headersFiltered[] = [ 'name' => $key, 'value' => $value ];
+                }
+            }
+
             /** Update execution status */
             $execution
-                ->setAttribute('status', $executionResponse['status'])
-                ->setAttribute('statusCode', $executionResponse['statusCode'])
-                ->setAttribute('response', $executionResponse['response'])
-                ->setAttribute('stdout', $executionResponse['stdout'])
-                ->setAttribute('stderr', $executionResponse['stderr'])
+                ->setAttribute('status', $status)
+                ->setAttribute('responseStatusCode', $executionResponse['statusCode'])
+                ->setAttribute('responseHeaders', $headersFiltered)
+                ->setAttribute('logs', $executionResponse['logs'])
+                ->setAttribute('errors', $executionResponse['errors'])
                 ->setAttribute('duration', $executionResponse['duration']);
         } catch (\Throwable $th) {
-            $interval = (new \DateTime())->diff(new \DateTime($execution->getCreatedAt()));
+            $durationEnd = \microtime(true);
+
             $execution
-                ->setAttribute('duration', (float)$interval->format('%s.%f'))
+                ->setAttribute('duration', $durationEnd - $durationStart)
                 ->setAttribute('status', 'failed')
-                ->setAttribute('statusCode', $th->getCode())
-                ->setAttribute('stderr', $th->getMessage());
+                ->setAttribute('responseStatusCode', 500)
+                ->setAttribute('errors', $th->getMessage() . '\nError Code: ' . $th->getCode());
 
             $error = $th->getMessage();
             $errorCode = $th->getCode();
         }
 
-        $execution = $dbForProject->updateDocument('executions', $executionId, $execution);
+        if ($function->getAttribute('logging')) {
+            $execution = $dbForProject->updateDocument('executions', $executionId, $execution);
+        }
 
         /** Trigger Webhook */
         $executionModel = new Execution();
@@ -217,13 +265,24 @@ Server::setResource('execute', function () {
             roles: $target['roles']
         );
 
-        /** Trigger usage queue */
-        $queueForUsage
-            ->setProject($project)
-            ->addMetric(METRIC_EXECUTIONS_COMPUTE, (int)($execution->getAttribute('duration') * 1000))// per project
-            ->addMetric(str_replace('{functionInternalId}', $function->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE), (int)($execution->getAttribute('duration') * 1000))
-            ->trigger()
-        ;
+        /** Update usage stats */
+        if (App::getEnv('_APP_USAGE_STATS', 'enabled') === 'enabled') {
+            $usage = new Stats($statsd);
+            $usage
+                ->setParam('projectId', $project->getId())
+                ->setParam('projectInternalId', $project->getInternalId())
+                ->setParam('functionId', $function->getId()) // TODO: We should use functionInternalId in usage stats
+                ->setParam('executions.{scope}.compute', 1)
+                ->setParam('executionStatus', $execution->getAttribute('status', ''))
+                ->setParam('executionTime', $execution->getAttribute('duration'))
+                ->setParam('networkRequestSize', 0)
+                ->setParam('networkResponseSize', 0)
+                ->submit();
+        }
+
+        if (!empty($error)) {
+            throw new Exception($error, $errorCode);
+        }
     };
 });
 
@@ -231,10 +290,10 @@ $server->job()
     ->inject('message')
     ->inject('dbForProject')
     ->inject('queueForFunctions')
-    ->inject('queueForUsage')
+    ->inject('statsd')
     ->inject('execute')
     ->inject('log')
-    ->action(function (Message $message, Database $dbForProject, Func $queueForFunctions, Usage $queueForUsage, callable $execute, Log $log) {
+    ->action(function (Message $message, Database $dbForProject, Func $queueForFunctions, Client $statsd, callable $execute, Log $log) {
         $payload = $message->getPayload() ?? [];
 
         if (empty($payload)) {
@@ -243,7 +302,7 @@ $server->job()
 
         $type = $payload['type'] ?? '';
         $events = $payload['events'] ?? [];
-        $data = $payload['data'] ?? '';
+        $data = $payload['body'] ?? '';
         $eventData = $payload['payload'] ?? '';
         $project = new Document($payload['project'] ?? []);
         $function = new Document($payload['function'] ?? []);
@@ -275,26 +334,26 @@ $server->job()
                         continue;
                     }
                     Console::success('Iterating function: ' . $function->getAttribute('name'));
-                    try {
-                        $execute(
-                            log: $log,
-                            queueForUsage: $queueForUsage,
-                            dbForProject: $dbForProject,
-                            project: $project,
-                            function: $function,
-                            queueForFunctions: $queueForFunctions,
-                            trigger: 'event',
-                            event: $events[0],
-                            eventData: \is_string($eventData) ? $eventData : \json_encode($eventData),
-                            user: $user,
-                            data: null,
-                            executionId: null,
-                            jwt: null
-                        );
-                        Console::success('Triggered function: ' . $events[0]);
-                    } catch (\Throwable $th) {
-                        Console::error("Failed to execute " . $function->getId() . " with error: " . $th->getMessage());
-                    }
+                    $execute(
+                        statsd: $statsd,
+                        dbForProject: $dbForProject,
+                        project: $project,
+                        function: $function,
+                        queueForFunctions: $queueForFunctions,
+                        trigger: 'event',
+                        event: $events[0],
+                        eventData: \is_string($eventData) ? $eventData : \json_encode($eventData),
+                        user: $user,
+                        data: null,
+                        executionId: null,
+                        jwt: null,
+                        path: '/',
+                        method: 'POST',
+                        headers: [
+                            'user-agent' => 'Appwrite/' . APP_VERSION_STABLE
+                        ],
+                    );
+                    Console::success('Triggered function: ' . $events[0]);
                 }
             }
             return;
@@ -321,7 +380,10 @@ $server->job()
                     data: $data,
                     user: $user,
                     jwt: $jwt,
-                    queueForUsage: $queueForUsage,
+                    path: $payload['path'],
+                    method: $payload['method'],
+                    headers: $payload['headers'],
+                    statsd: $statsd,
                 );
                 break;
             case 'schedule':
@@ -338,7 +400,10 @@ $server->job()
                     data: null,
                     user: null,
                     jwt: null,
-                    queueForUsage: $queueForUsage,
+                    path: $payload['path'],
+                    method: $payload['method'],
+                    headers: $payload['headers'],
+                    statsd: $statsd,
                 );
                 break;
         }

app/workers/mails.php

@@ -1,6 +1,7 @@
 <?php
 
 use Appwrite\Resque\Worker;
+use Appwrite\Template\Template;
 use Utopia\App;
 use Utopia\CLI\Console;
 use PHPMailer\PHPMailer\PHPMailer;
@@ -32,12 +33,19 @@ class MailsV1 extends Worker
             return;
         }
 
-
         $recipient = $this->args['recipient'];
         $subject = $this->args['subject'];
         $name = $this->args['name'];
         $body = $this->args['body'];
-        $from = $this->args['from'];
+        $variables = $this->args['variables'];
+
+        $body = Template::fromFile(__DIR__ . '/../config/locale/templates/email-base.tpl');
+
+        foreach ($variables as $key => $value) {
+            $body->setParam('{{' . $key . '}}', $value);
+        }
+
+        $body = $body->render();
 
         /** @var \PHPMailer\PHPMailer\PHPMailer $mail */
         $mail = empty($smtp) ? $register->get('smtp') : $this->getMailer($smtp);
@@ -48,16 +56,16 @@ class MailsV1 extends Worker
         $mail->clearAttachments();
         $mail->clearBCCs();
         $mail->clearCCs();
-
-        $mail->setFrom(App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM), (empty($from) ? \urldecode(App::getEnv('_APP_SYSTEM_EMAIL_NAME', APP_NAME . ' Server')) : $from));
         $mail->addAddress($recipient, $name);
-        if (isset($smtp['replyTo'])) {
-            $mail->addReplyTo($smtp['replyTo']);
-        }
         $mail->Subject = $subject;
         $mail->Body = $body;
         $mail->AltBody = \strip_tags($body);
 
+        $mail->setFrom($smtp['senderEmail'], $smtp['senderName']);
+        if (!empty($smtp['replyTo'])) {
+            $mail->addReplyTo($smtp['replyTo'], $smtp['senderName']);
+        }
+
         try {
             $mail->send();
         } catch (\Exception $error) {
@@ -84,12 +92,6 @@ class MailsV1 extends Worker
         $mail->SMTPAutoTLS = false;
         $mail->CharSet = 'UTF-8';
 
-        $from = \urldecode($smtp['senderName'] ?? App::getEnv('_APP_SYSTEM_EMAIL_NAME', APP_NAME . ' Server'));
-        $email = $smtp['senderEmail'] ?? App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM);
-
-        $mail->setFrom($email, $from);
-        $mail->addReplyTo($email, $from);
-
         $mail->isHTML(true);
 
         return $mail;

app/workers/usage.php

@@ -1,288 +0,0 @@
-<?php
-
-require_once __DIR__ . '/../worker.php';
-
-use Swoole\Timer;
-use Utopia\App;
-use Utopia\Cache\Cache;
-use Utopia\Database\Database;
-use Utopia\Database\DateTime;
-use Utopia\Database\Document;
-use Utopia\Database\Exception\Duplicate;
-use Utopia\Database\Validator\Authorization;
-use Utopia\Queue\Message;
-use Utopia\CLI\Console;
-use Utopia\Queue\Server;
-use Utopia\Registry\Registry;
-
-Authorization::disable();
-Authorization::setDefaultStatus(false);
-
-$stats = [];
-
-$periods['1h']  = 'Y-m-d H:00';
-$periods['1d']  = 'Y-m-d 00:00';
-//$periods['1m']  = 'Y-m-1 00:00';
-$periods['inf'] = '0000-00-00 00:00';
-
-const INFINITY_PERIOD = '_inf_';
-
-/**
- * On Documents that tied by relations like functions>deployments>build || documents>collection>database || buckets>files.
- * When we remove a parent document we need to deduct his children aggregation from the project scope.
- */
-Server::setResource('reduce', function (Cache $cache, Registry $register, $pools) {
-    return function ($database, $projectInternalId, Document $document, array &$metrics) use ($pools, $cache, $register): void {
-        try {
-            $dbForProject = new Database(
-                $pools
-                    ->get($database)
-                    ->pop()
-                    ->getResource(),
-                $cache
-            );
-
-            $dbForProject->setNamespace('_' . $projectInternalId);
-
-            switch (true) {
-                case $document->getCollection() === 'users': // users
-                    $sessions = count($document->getAttribute(METRIC_SESSIONS, 0));
-                    if (!empty($sessions)) {
-                        $metrics[] = [
-                            'key' => METRIC_SESSIONS,
-                            'value' => ($sessions * -1),
-                        ];
-                    }
-                    break;
-                case $document->getCollection() === 'databases': // databases
-                    $collections = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_COLLECTIONS)));
-                    $documents = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace('{databaseInternalId}', $document->getInternalId(), METRIC_DATABASE_ID_DOCUMENTS)));
-                    if (!empty($collections['value'])) {
-                        $metrics[] = [
-                            'key' => METRIC_COLLECTIONS,
-                            'value' => ($collections['value'] * -1),
-                        ];
-                    }
-
-                    if (!empty($documents['value'])) {
-                        $metrics[] = [
-                            'key' => METRIC_DOCUMENTS,
-                            'value' => ($documents['value'] * -1),
-                        ];
-                    }
-                    break;
-                case str_starts_with($document->getCollection(), 'database_') && !str_contains($document->getCollection(), 'collection'): //collections
-                    $parts = explode('_', $document->getCollection());
-                    $databaseInternalId = $parts[1] ?? 0;
-                    $documents = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$databaseInternalId, $document->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_DOCUMENTS)));
-
-                    if (!empty($documents['value'])) {
-                        $metrics[] = [
-                            'key' => METRIC_DOCUMENTS,
-                            'value' => ($documents['value'] * -1),
-                        ];
-                        $metrics[] = [
-                            'key' => str_replace('{databaseInternalId}', $databaseInternalId, METRIC_DATABASE_ID_DOCUMENTS),
-                            'value' => ($documents['value'] * -1),
-                        ];
-                    }
-                    break;
-
-                case $document->getCollection() === 'buckets':
-                    $files = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES)));
-                    $storage = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace('{bucketInternalId}', $document->getInternalId(), METRIC_BUCKET_ID_FILES_STORAGE)));
-
-                    if (!empty($files['value'])) {
-                        $metrics[] = [
-                            'key' => METRIC_FILES,
-                            'value' => ($files['value'] * -1),
-                        ];
-                    }
-
-                    if (!empty($storage['value'])) {
-                        $metrics[] = [
-                            'key' => METRIC_FILES_STORAGE,
-                            'value' => ($storage['value'] * -1),
-                        ];
-                    }
-                    break;
-
-                case $document->getCollection() === 'functions':
-                    $deployments = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS)));
-                    $deploymentsStorage = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace(['{resourceType}', '{resourceInternalId}'], ['functions', $document->getInternalId()], METRIC_FUNCTION_ID_DEPLOYMENTS_STORAGE)));
-                    $builds = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD .  str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS)));
-                    $buildsStorage = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_STORAGE)));
-                    $buildsCompute = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_BUILDS_COMPUTE)));
-                    $executions = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD .  str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS)));
-                    $executionsCompute = $dbForProject->getDocument('stats', md5(INFINITY_PERIOD . str_replace('{functionInternalId}', $document->getInternalId(), METRIC_FUNCTION_ID_EXECUTIONS_COMPUTE)));
-
-                    if (!empty($deployments['value'])) {
-                        $metrics[] = [
-                            'key' => METRIC_DEPLOYMENTS,
-                            'value' => ($deployments['value'] * -1),
-                        ];
-                    }
-
-                    if (!empty($deploymentsStorage['value'])) {
-                        $metrics[] = [
-                            'key' => METRIC_DEPLOYMENTS_STORAGE,
-                            'value' => ($deploymentsStorage['value'] * -1),
-                        ];
-                    }
-
-                    if (!empty($builds['value'])) {
-                        $metrics[] = [
-                            'key' => METRIC_BUILDS,
-                            'value' => ($builds['value'] * -1),
-                        ];
-                    }
-
-                    if (!empty($buildsStorage['value'])) {
-                        $metrics[] = [
-                            'key' => METRIC_BUILDS_STORAGE,
-                            'value' => ($buildsStorage['value'] * -1),
-                        ];
-                    }
-
-                    if (!empty($buildsCompute['value'])) {
-                        $metrics[] = [
-                            'key' => METRIC_BUILDS_COMPUTE,
-                            'value' => ($buildsCompute['value'] * -1),
-                        ];
-                    }
-
-                    if (!empty($executions['value'])) {
-                        $metrics[] = [
-                            'key' => METRIC_EXECUTIONS,
-                            'value' => ($executions['value'] * -1),
-                        ];
-                    }
-
-                    if (!empty($executionsCompute['value'])) {
-                        $metrics[] = [
-                            'key' => METRIC_EXECUTIONS_COMPUTE,
-                            'value' => ($executionsCompute['value'] * -1),
-                        ];
-                    }
-                    break;
-                default:
-                    break;
-            }
-        } catch (\Exception $e) {
-            console::error("[reducer] " . " {DateTime::now()} " .  " {$projectInternalId} " . " {$e->getMessage()}");
-        } finally {
-            $pools->reclaim();
-        }
-    };
-}, ['cache', 'register', 'pools']);
-
-
-$server->job()
-    ->inject('message')
-    ->inject('reduce')
-
-    ->action(function (Message $message, callable $reduce) use (&$stats) {
-
-        $payload = $message->getPayload() ?? [];
-        $project = new Document($payload['project'] ?? []);
-        $projectId = $project->getInternalId();
-        foreach ($payload['reduce'] ?? [] as $document) {
-            if (empty($document)) {
-                continue;
-            }
-
-             $reduce(
-                 database: $project->getAttribute('database'),
-                 projectInternalId: $project->getInternalId(),
-                 document: new Document($document),
-                 metrics:  $payload['metrics'],
-             );
-        }
-
-        $stats[$projectId]['database'] = $project->getAttribute('database');
-        foreach ($payload['metrics'] ?? [] as $metric) {
-            if (!isset($stats[$projectId]['keys'][$metric['key']])) {
-                $stats[$projectId]['keys'][$metric['key']] = $metric['value'];
-                continue;
-            }
-            $stats[$projectId]['keys'][$metric['key']] += $metric['value'];
-        }
-    });
-
-$server
-    ->workerStart()
-    ->inject('register')
-    ->inject('cache')
-    ->inject('pools')
-    ->action(function ($register, $cache, $pools) use ($periods, &$stats) {
-        Timer::tick(30000, function () use ($register, $cache, $pools, $periods, &$stats) {
-
-            $offset = count($stats);
-            $projects = array_slice($stats, 0, $offset, true);
-            array_splice($stats, 0, $offset);
-
-            foreach ($projects as $projectInternalId => $project) {
-                try {
-                    $dbForProject = new Database(
-                        $pools
-                        ->get($project['database'])
-                        ->pop()
-                        ->getResource(),
-                        $cache
-                    );
-
-                    $dbForProject->setNamespace('_' . $projectInternalId);
-
-                    foreach ($project['keys'] ?? [] as $key => $value) {
-                        if ($value == 0) {
-                            continue;
-                        }
-
-                        foreach ($periods as $period => $format) {
-                            $time = 'inf' === $period ? null : date($format, time());
-                            $id = \md5("{$time}_{$period}_{$key}");
-
-                            try {
-                                $dbForProject->createDocument('stats', new Document([
-                                    '$id' => $id,
-                                    'period' => $period,
-                                    'time' => $time,
-                                    'metric' => $key,
-                                    'value' => $value,
-                                    'region' => App::getEnv('_APP_REGION', 'default'),
-                                ]));
-                            } catch (Duplicate $th) {
-                                if ($value < 0) {
-                                    $dbForProject->decreaseDocumentAttribute(
-                                        'stats',
-                                        $id,
-                                        'value',
-                                        abs($value)
-                                    );
-                                } else {
-                                    $dbForProject->increaseDocumentAttribute(
-                                        'stats',
-                                        $id,
-                                        'value',
-                                        $value
-                                    );
-                                }
-                            }
-                        }
-                    }
-                    if (!empty($project['keys'])) {
-                        $dbForProject->createDocument('statsLogger', new Document([
-                            'time' => DateTime::now(),
-                            'metrics' => $project['keys'],
-                        ]));
-                    }
-                } catch (\Exception $e) {
-                    $now = DateTime::now();
-                    console::error("[Error] " . " Time: {$now} " .  " projectInternalId: {$projectInternalId}" . " File: {$e->getFile()}" . " Line: {$e->getLine()} " . " message: {$e->getMessage()}");
-                } finally {
-                    $pools->reclaim();
-                }
-            }
-        });
-    });
-$server->start();

bin/usage

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+php /usr/src/code/app/cli.php usage $@

bin/worker-usage

@@ -1,3 +0,0 @@
-#!/bin/sh
-
-QUEUE=v1-usage php /usr/src/code/app/workers/usage.php $@

composer.json

@@ -41,18 +41,18 @@
         "ext-openssl": "*",
         "ext-zlib": "*",
         "ext-sockets": "*",
+        "appwrite/php-runtimes": "0.12.0",
         "appwrite/php-clamav": "2.0.*",
-        "appwrite/php-runtimes": "0.11.*",
         "utopia-php/abuse": "0.31.*",
         "utopia-php/analytics": "0.10.*",
         "utopia-php/audit": "0.33.*",
         "utopia-php/cache": "0.8.*",
         "utopia-php/cli": "0.15.*",
         "utopia-php/config": "0.2.*",
-        "utopia-php/database": "0.42.*",
+        "utopia-php/database": "0.43.*",
         "utopia-php/domains": "0.3.*",
         "utopia-php/dsn": "0.1.*",
-        "utopia-php/framework": "0.28.*",
+        "utopia-php/framework": "0.30.0",
         "utopia-php/image": "0.5.*",
         "utopia-php/locale": "0.4.*",
         "utopia-php/logger": "0.3.*",
@@ -64,18 +64,20 @@
         "utopia-php/queue": "0.5.*",
         "utopia-php/registry": "0.5.*",
         "utopia-php/storage": "0.14.*",
-        "utopia-php/swoole": "0.8.*",
+        "utopia-php/swoole": "0.5.*",
+        "utopia-php/vcs": "0.3.*",
         "utopia-php/websocket": "0.1.*",
         "resque/php-resque": "1.3.6",
         "matomo/device-detector": "6.1.*",
         "dragonmantank/cron-expression": "3.3.2",
+        "influxdb/influxdb-php": "1.15.2",
         "phpmailer/phpmailer": "6.8.0",
         "chillerlan/php-qrcode": "4.3.4",
         "adhocore/jwt": "1.1.2",
         "webonyx/graphql-php": "14.11.*",
         "slickdeals/statsd": "3.1.0",
         "league/csv": "9.7.1",
-        "utopia-php/migration": "^0.2.0"
+        "utopia-php/migration": "^0.3.0"
     },
     "repositories": [
         {
@@ -84,7 +86,7 @@
         }
     ],
     "require-dev": {
-        "appwrite/sdk-generator": "0.33.*",
+        "appwrite/sdk-generator": "0.34.*",
         "ext-fileinfo": "*",
         "phpunit/phpunit": "9.5.20",
         "squizlabs/php_codesniffer": "^3.7",

docker-compose.yml

@@ -53,7 +53,6 @@ services:
         DEBUG: false
         TESTING: true
         VERSION: dev
-        VITE_CONSOLE_MODE: cloud
     ports: 
       - 9501:80
     networks:
@@ -97,14 +96,10 @@ services:
     environment:
       - _APP_ENV
       - _APP_WORKER_PER_CORE
-      - _APP_SERVER_MULTIPROCESS=enabled
       - _APP_LOCALE
       - _APP_CONSOLE_WHITELIST_ROOT
       - _APP_CONSOLE_WHITELIST_EMAILS
-      - _APP_CONSOLE_WHITELIST_CODES
       - _APP_CONSOLE_WHITELIST_IPS
-      - _APP_CONSOLE_INVITES
-      - _APP_CONSOLE_ROOT_SESSION
       - _APP_SYSTEM_EMAIL_NAME
       - _APP_SYSTEM_EMAIL_ADDRESS
       - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS
@@ -114,35 +109,50 @@ services:
       - _APP_OPENSSL_KEY_V1
       - _APP_DOMAIN
       - _APP_DOMAIN_TARGET
+      - _APP_DOMAIN_FUNCTIONS
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
       - _APP_DB_HOST
       - _APP_DB_PORT
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
-      - _APP_CONNECTIONS_MAX
-      - _APP_POOL_CLIENTS
-      - _APP_REDIS_HOST
-      - _APP_REDIS_PORT
-      - _APP_REDIS_USER
-      - _APP_REDIS_PASS
-      - _APP_CONNECTIONS_DB_PROJECT
-      - _APP_CONNECTIONS_DB_CONSOLE
-      - _APP_CONNECTIONS_CACHE
-      - _APP_CONNECTIONS_QUEUE
-      - _APP_CONNECTIONS_PUBSUB
       - _APP_SMTP_HOST
       - _APP_SMTP_PORT
       - _APP_SMTP_SECURE
       - _APP_SMTP_USERNAME
       - _APP_SMTP_PASSWORD
-      - _APP_USERS_STATS_RECIPIENTS
       - _APP_USAGE_STATS
+      - _APP_INFLUXDB_HOST
+      - _APP_INFLUXDB_PORT
       - _APP_STORAGE_LIMIT
       - _APP_STORAGE_PREVIEW_LIMIT
       - _APP_STORAGE_ANTIVIRUS
       - _APP_STORAGE_ANTIVIRUS_HOST
       - _APP_STORAGE_ANTIVIRUS_PORT
-      - _APP_CONNECTIONS_STORAGE
+      - _APP_STORAGE_DEVICE
+      - _APP_STORAGE_S3_ACCESS_KEY
+      - _APP_STORAGE_S3_SECRET
+      - _APP_STORAGE_S3_REGION
+      - _APP_STORAGE_S3_BUCKET
+      - _APP_STORAGE_DO_SPACES_ACCESS_KEY
+      - _APP_STORAGE_DO_SPACES_SECRET
+      - _APP_STORAGE_DO_SPACES_REGION
+      - _APP_STORAGE_DO_SPACES_BUCKET
+      - _APP_STORAGE_BACKBLAZE_ACCESS_KEY
+      - _APP_STORAGE_BACKBLAZE_SECRET
+      - _APP_STORAGE_BACKBLAZE_REGION
+      - _APP_STORAGE_BACKBLAZE_BUCKET
+      - _APP_STORAGE_LINODE_ACCESS_KEY
+      - _APP_STORAGE_LINODE_SECRET
+      - _APP_STORAGE_LINODE_REGION
+      - _APP_STORAGE_LINODE_BUCKET
+      - _APP_STORAGE_WASABI_ACCESS_KEY
+      - _APP_STORAGE_WASABI_SECRET
+      - _APP_STORAGE_WASABI_REGION
+      - _APP_STORAGE_WASABI_BUCKET
       - _APP_FUNCTIONS_SIZE_LIMIT
       - _APP_FUNCTIONS_TIMEOUT
       - _APP_FUNCTIONS_BUILD_TIMEOUT
@@ -153,6 +163,8 @@ services:
       - _APP_EXECUTOR_HOST
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
+      - _APP_STATSD_HOST
+      - _APP_STATSD_PORT
       - _APP_MAINTENANCE_INTERVAL
       - _APP_MAINTENANCE_RETENTION_EXECUTION
       - _APP_MAINTENANCE_RETENTION_CACHE
@@ -161,14 +173,18 @@ services:
       - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY
       - _APP_SMS_PROVIDER
       - _APP_SMS_FROM
-      - _APP_REGION
       - _APP_GRAPHQL_MAX_BATCH_SIZE
       - _APP_GRAPHQL_MAX_COMPLEXITY
       - _APP_GRAPHQL_MAX_DEPTH
-      - _APP_CONSOLE_GITHUB_APP_ID
-      - _APP_CONSOLE_GITHUB_SECRET
+      - _APP_VCS_GITHUB_APP_NAME
+      - _APP_VCS_GITHUB_PRIVATE_KEY
+      - _APP_VCS_GITHUB_APP_ID
+      - _APP_VCS_GITHUB_WEBHOOK_SECRET
+      - _APP_VCS_GITHUB_CLIENT_SECRET
+      - _APP_VCS_GITHUB_CLIENT_ID
       - _APP_MIGRATIONS_FIREBASE_CLIENT_ID
       - _APP_MIGRATIONS_FIREBASE_CLIENT_SECRET
+      - _APP_ASSISTANT_OPENAI_API_KEY
 
   appwrite-realtime:
     entrypoint: realtime
@@ -204,25 +220,17 @@ services:
     environment:
       - _APP_ENV
       - _APP_WORKER_PER_CORE
-      - _APP_SERVER_MULTIPROCESS=enabled
       - _APP_OPTIONS_ABUSE
       - _APP_OPENSSL_KEY_V1
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
       - _APP_DB_HOST
       - _APP_DB_PORT
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
-      - _APP_CONNECTIONS_MAX
-      - _APP_POOL_CLIENTS
-      - _APP_REDIS_HOST
-      - _APP_REDIS_PORT
-      - _APP_REDIS_USER
-      - _APP_REDIS_PASS
-      - _APP_CONNECTIONS_DB_CONSOLE
-      - _APP_CONNECTIONS_DB_PROJECT
-      - _APP_CONNECTIONS_CACHE
-      - _APP_CONNECTIONS_PUBSUB
-      - _APP_CONNECTIONS_QUEUE
       - _APP_USAGE_STATS
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
@@ -244,19 +252,15 @@ services:
       - _APP_ENV
       - _APP_WORKER_PER_CORE
       - _APP_OPENSSL_KEY_V1
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
       - _APP_DB_HOST
       - _APP_DB_PORT
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
-      - _APP_REDIS_HOST
-      - _APP_REDIS_PORT
-      - _APP_REDIS_USER
-      - _APP_REDIS_PASS
-      - _APP_CONNECTIONS_DB_CONSOLE
-      - _APP_CONNECTIONS_DB_PROJECT
-      - _APP_CONNECTIONS_CACHE
-      - _APP_CONNECTIONS_QUEUE
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
 
@@ -277,15 +281,12 @@ services:
     environment:
       - _APP_ENV
       - _APP_WORKER_PER_CORE
-      - _APP_CONNECTIONS_MAX
-      - _APP_POOL_CLIENTS
       - _APP_OPENSSL_KEY_V1
       - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS
       - _APP_REDIS_HOST
       - _APP_REDIS_PORT
       - _APP_REDIS_USER
       - _APP_REDIS_PASS
-      - _APP_CONNECTIONS_QUEUE
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
 
@@ -310,23 +311,37 @@ services:
     environment:
       - _APP_ENV
       - _APP_WORKER_PER_CORE
-      - _APP_CONNECTIONS_MAX
-      - _APP_POOL_CLIENTS
       - _APP_OPENSSL_KEY_V1
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
       - _APP_DB_HOST
       - _APP_DB_PORT
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
-      - _APP_REDIS_HOST
-      - _APP_REDIS_PORT
-      - _APP_REDIS_USER
-      - _APP_REDIS_PASS
-      - _APP_CONNECTIONS_DB_CONSOLE
-      - _APP_CONNECTIONS_DB_PROJECT
-      - _APP_CONNECTIONS_CACHE
-      - _APP_CONNECTIONS_QUEUE
-      - _APP_CONNECTIONS_STORAGE
+      - _APP_STORAGE_DEVICE
+      - _APP_STORAGE_S3_ACCESS_KEY
+      - _APP_STORAGE_S3_SECRET
+      - _APP_STORAGE_S3_REGION
+      - _APP_STORAGE_S3_BUCKET
+      - _APP_STORAGE_DO_SPACES_ACCESS_KEY
+      - _APP_STORAGE_DO_SPACES_SECRET
+      - _APP_STORAGE_DO_SPACES_REGION
+      - _APP_STORAGE_DO_SPACES_BUCKET
+      - _APP_STORAGE_BACKBLAZE_ACCESS_KEY
+      - _APP_STORAGE_BACKBLAZE_SECRET
+      - _APP_STORAGE_BACKBLAZE_REGION
+      - _APP_STORAGE_BACKBLAZE_BUCKET
+      - _APP_STORAGE_LINODE_ACCESS_KEY
+      - _APP_STORAGE_LINODE_SECRET
+      - _APP_STORAGE_LINODE_REGION
+      - _APP_STORAGE_LINODE_BUCKET
+      - _APP_STORAGE_WASABI_ACCESS_KEY
+      - _APP_STORAGE_WASABI_SECRET
+      - _APP_STORAGE_WASABI_REGION
+      - _APP_STORAGE_WASABI_BUCKET
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
       - _APP_EXECUTOR_SECRET
@@ -348,22 +363,16 @@ services:
     environment:
       - _APP_ENV
       - _APP_WORKER_PER_CORE
-      - _APP_CONNECTIONS_MAX
-      - _APP_POOL_CLIENTS
       - _APP_OPENSSL_KEY_V1
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
       - _APP_DB_HOST
       - _APP_DB_PORT
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
-      - _APP_REDIS_HOST
-      - _APP_REDIS_PORT
-      - _APP_REDIS_USER
-      - _APP_REDIS_PASS
-      - _APP_CONNECTIONS_DB_CONSOLE
-      - _APP_CONNECTIONS_DB_PROJECT
-      - _APP_CONNECTIONS_CACHE
-      - _APP_CONNECTIONS_QUEUE
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
 
@@ -374,7 +383,9 @@ services:
     image: appwrite-dev
     networks:
       - appwrite
-    volumes: 
+    volumes:
+      - appwrite-functions:/storage/functions:rw
+      - appwrite-builds:/storage/builds:rw
       - ./app:/usr/src/code/app
       - ./src:/usr/src/code/src
     depends_on:
@@ -383,27 +394,29 @@ services:
     environment:
       - _APP_ENV
       - _APP_WORKER_PER_CORE
-      - _APP_CONNECTIONS_MAX
-      - _APP_POOL_CLIENTS
       - _APP_OPENSSL_KEY_V1
       - _APP_EXECUTOR_SECRET
       - _APP_EXECUTOR_HOST
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
       - _APP_DB_HOST
       - _APP_DB_PORT
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
-      - _APP_REDIS_HOST
-      - _APP_REDIS_PORT
-      - _APP_REDIS_USER
-      - _APP_REDIS_PASS
-      - _APP_CONNECTIONS_DB_CONSOLE
-      - _APP_CONNECTIONS_DB_PROJECT
-      - _APP_CONNECTIONS_CACHE
-      - _APP_CONNECTIONS_QUEUE
-      - _APP_CONNECTIONS_STORAGE
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
+      - _APP_VCS_GITHUB_APP_NAME
+      - _APP_VCS_GITHUB_PRIVATE_KEY
+      - _APP_VCS_GITHUB_APP_ID
+      - _APP_FUNCTIONS_TIMEOUT
+      - _APP_FUNCTIONS_BUILD_TIMEOUT
+      - _APP_FUNCTIONS_CPUS
+      - _APP_FUNCTIONS_MEMORY
+      - _APP_OPTIONS_FORCE_HTTPS
+      - _APP_DOMAIN
 
   appwrite-worker-certificates:
     entrypoint: worker-certificates
@@ -423,25 +436,20 @@ services:
     environment:
       - _APP_ENV
       - _APP_WORKER_PER_CORE
-      - _APP_CONNECTIONS_MAX
-      - _APP_POOL_CLIENTS
       - _APP_OPENSSL_KEY_V1
       - _APP_DOMAIN
       - _APP_DOMAIN_TARGET
+      - _APP_DOMAIN_FUNCTIONS
       - _APP_SYSTEM_SECURITY_EMAIL_ADDRESS
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
       - _APP_DB_HOST
       - _APP_DB_PORT
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
-      - _APP_REDIS_HOST
-      - _APP_REDIS_PORT
-      - _APP_REDIS_USER
-      - _APP_REDIS_PASS
-      - _APP_CONNECTIONS_DB_CONSOLE
-      - _APP_CONNECTIONS_DB_PROJECT
-      - _APP_CONNECTIONS_CACHE
-      - _APP_CONNECTIONS_QUEUE
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
 
@@ -462,23 +470,20 @@ services:
     environment:
       - _APP_ENV
       - _APP_WORKER_PER_CORE
-      - _APP_CONNECTIONS_MAX
-      - _APP_POOL_CLIENTS
       - _APP_OPENSSL_KEY_V1
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
       - _APP_DB_HOST
       - _APP_DB_PORT
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
-      - _APP_REDIS_HOST
-      - _APP_REDIS_PORT
-      - _APP_REDIS_USER
-      - _APP_REDIS_PASS
-      - _APP_CONNECTIONS_DB_CONSOLE
-      - _APP_CONNECTIONS_DB_PROJECT
-      - _APP_CONNECTIONS_CACHE
-      - _APP_CONNECTIONS_QUEUE
       - _APP_FUNCTIONS_TIMEOUT
+      - _APP_FUNCTIONS_BUILD_TIMEOUT
+      - _APP_FUNCTIONS_CPUS
+      - _APP_FUNCTIONS_MEMORY
       - _APP_EXECUTOR_SECRET
       - _APP_EXECUTOR_HOST
       - _APP_USAGE_STATS
@@ -504,8 +509,6 @@ services:
     environment:
       - _APP_ENV
       - _APP_WORKER_PER_CORE
-      - _APP_CONNECTIONS_MAX
-      - _APP_POOL_CLIENTS
       - _APP_OPENSSL_KEY_V1
       - _APP_SYSTEM_EMAIL_NAME
       - _APP_SYSTEM_EMAIL_ADDRESS
@@ -513,7 +516,6 @@ services:
       - _APP_REDIS_PORT
       - _APP_REDIS_USER
       - _APP_REDIS_PASS
-      - _APP_CONNECTIONS_QUEUE
       - _APP_SMTP_HOST
       - _APP_SMTP_PORT
       - _APP_SMTP_SECURE
@@ -538,52 +540,15 @@ services:
     environment:
       - _APP_ENV
       - _APP_WORKER_PER_CORE
-      - _APP_CONNECTIONS_MAX
-      - _APP_POOL_CLIENTS
       - _APP_REDIS_HOST
       - _APP_REDIS_PORT
       - _APP_REDIS_USER
       - _APP_REDIS_PASS
-      - _APP_CONNECTIONS_QUEUE
       - _APP_SMS_PROVIDER
       - _APP_SMS_FROM
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
 
-  appwrite-hamster:
-    entrypoint: hamster
-    <<: *x-logging
-    container_name: appwrite-hamster
-    image: appwrite-dev
-    networks:
-      - appwrite
-    volumes:
-      - ./app:/usr/src/code/app
-      - ./src:/usr/src/code/src
-    depends_on:
-      - redis
-    environment:
-      - _APP_ENV
-      - _APP_WORKER_PER_CORE
-      - _APP_CONNECTIONS_MAX
-      - _APP_POOL_CLIENTS
-      - _APP_OPENSSL_KEY_V1
-      - _APP_DB_HOST
-      - _APP_DB_PORT
-      - _APP_DB_SCHEMA
-      - _APP_DB_USER
-      - _APP_DB_PASS
-      - _APP_REDIS_HOST
-      - _APP_REDIS_PORT
-      - _APP_REDIS_USER
-      - _APP_REDIS_PASS
-      - _APP_CONNECTIONS_DB_CONSOLE
-      - _APP_CONNECTIONS_DB_PROJECT
-      - _APP_CONNECTIONS_CACHE
-      - _APP_HAMSTER_INTERVAL
-      - _APP_HAMSTER_TIME
-      - _APP_MIXPANEL_TOKEN
-
   appwrite-worker-migrations:
     entrypoint: worker-migrations
     <<: *x-logging
@@ -596,11 +561,11 @@ services:
       - ./app:/usr/src/code/app
       - ./src:/usr/src/code/src
       - ./tests:/usr/src/code/tests
-      - ./vendor:/usr/src/code/tests
     depends_on:
       - mariadb
     environment:
       - _APP_ENV
+      - _APP_WORKER_PER_CORE
       - _APP_OPENSSL_KEY_V1
       - _APP_DOMAIN
       - _APP_DOMAIN_TARGET
@@ -618,11 +583,6 @@ services:
       - _APP_LOGGING_CONFIG
       - _APP_MIGRATIONS_FIREBASE_CLIENT_ID
       - _APP_MIGRATIONS_FIREBASE_CLIENT_SECRET
-      - _APP_CONNECTIONS_DB_PROJECT
-      - _APP_CONNECTIONS_DB_CONSOLE
-      - _APP_CONNECTIONS_CACHE
-      - _APP_CONNECTIONS_QUEUE
-      - _APP_CONNECTIONS_PUBSUB
 
   appwrite-maintenance:
     entrypoint: maintenance
@@ -639,62 +599,57 @@ services:
     environment:
       - _APP_ENV
       - _APP_WORKER_PER_CORE
-      - _APP_CONNECTIONS_MAX
-      - _APP_POOL_CLIENTS
       - _APP_DOMAIN
       - _APP_DOMAIN_TARGET
+      - _APP_DOMAIN_FUNCTIONS
       - _APP_OPENSSL_KEY_V1
+      - _APP_REDIS_HOST
+      - _APP_REDIS_PORT
+      - _APP_REDIS_USER
+      - _APP_REDIS_PASS
       - _APP_DB_HOST
       - _APP_DB_PORT
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
-      - _APP_REDIS_HOST
-      - _APP_REDIS_PORT
-      - _APP_REDIS_USER
-      - _APP_REDIS_PASS
-      - _APP_CONNECTIONS_DB_CONSOLE
-      - _APP_CONNECTIONS_DB_PROJECT
-      - _APP_CONNECTIONS_CACHE
       - _APP_MAINTENANCE_INTERVAL
       - _APP_MAINTENANCE_RETENTION_EXECUTION
       - _APP_MAINTENANCE_RETENTION_CACHE
       - _APP_MAINTENANCE_RETENTION_ABUSE
       - _APP_MAINTENANCE_RETENTION_AUDIT
+      - _APP_MAINTENANCE_RETENTION_USAGE_HOURLY
       - _APP_MAINTENANCE_RETENTION_SCHEDULES
 
-  appwrite-worker-usage:
-    entrypoint: worker-usage
+  appwrite-usage:
+    entrypoint: usage
     <<: *x-logging
-    container_name: appwrite-worker-usage
+    container_name: appwrite-usage
     image: appwrite-dev
     networks:
       - appwrite
     volumes:
       - ./app:/usr/src/code/app
       - ./src:/usr/src/code/src
+      - ./dev:/usr/local/dev
     depends_on:
-      - redis
+      - influxdb
       - mariadb
     environment:
       - _APP_ENV
       - _APP_WORKER_PER_CORE
-      - _APP_CONNECTIONS_MAX
-      - _APP_POOL_CLIENTS
       - _APP_OPENSSL_KEY_V1
       - _APP_DB_HOST
       - _APP_DB_PORT
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
+      - _APP_INFLUXDB_HOST
+      - _APP_INFLUXDB_PORT
+      - _APP_USAGE_AGGREGATION_INTERVAL
       - _APP_REDIS_HOST
       - _APP_REDIS_PORT
       - _APP_REDIS_USER
       - _APP_REDIS_PASS
-      - _APP_CONNECTIONS_DB_CONSOLE
-      - _APP_CONNECTIONS_DB_PROJECT
-      - _APP_CONNECTIONS_CACHE
-      - _APP_CONNECTIONS_QUEUE
       - _APP_USAGE_STATS
       - _APP_LOGGING_PROVIDER
       - _APP_LOGGING_CONFIG
@@ -715,8 +670,6 @@ services:
     environment:
       - _APP_ENV
       - _APP_WORKER_PER_CORE
-      - _APP_CONNECTIONS_MAX
-      - _APP_POOL_CLIENTS
       - _APP_REDIS_HOST
       - _APP_REDIS_PORT
       - _APP_REDIS_USER
@@ -726,11 +679,6 @@ services:
       - _APP_DB_SCHEMA
       - _APP_DB_USER
       - _APP_DB_PASS
-      - _APP_CONNECTIONS_DB_PROJECT
-      - _APP_CONNECTIONS_DB_CONSOLE
-      - _APP_CONNECTIONS_CACHE
-      - _APP_CONNECTIONS_QUEUE
-      - _APP_REGION
 
   appwrite-assistant:
     container_name: appwrite-assistant
@@ -738,14 +686,14 @@ services:
     networks:
       - appwrite
     environment:
-      - OPENAI_API_KEY
+      - _APP_ASSISTANT_OPENAI_API_KEY
 
   openruntimes-executor:
     container_name: openruntimes-executor
-    hostname: exc1
+    hostname: executor
     <<: *x-logging
     stop_signal: SIGINT
-    image: openruntimes/executor:0.1.6
+    image: openruntimes/executor:0.3.5
     networks:
       - appwrite
       - runtimes
@@ -753,9 +701,10 @@ services:
       - /var/run/docker.sock:/var/run/docker.sock
       - appwrite-builds:/storage/builds:rw
       - appwrite-functions:/storage/functions:rw
+      # Host mount nessessary to share files between executor and runtimes.
+      # It's not possible to share mount file between 2 containers without host mount (copying is too slow)
       - /tmp:/tmp:rw
     environment:
-      - OPR_EXECUTOR_CONNECTION_STORAGE=$_APP_CONNECTIONS_STORAGE
       - OPR_EXECUTOR_INACTIVE_TRESHOLD=$_APP_FUNCTIONS_INACTIVE_THRESHOLD
       - OPR_EXECUTOR_MAINTENANCE_INTERVAL=$_APP_FUNCTIONS_MAINTENANCE_INTERVAL
       - OPR_EXECUTOR_NETWORK=$_APP_FUNCTIONS_RUNTIMES_NETWORK
@@ -766,6 +715,49 @@ services:
       - OPR_EXECUTOR_SECRET=$_APP_EXECUTOR_SECRET
       - OPR_EXECUTOR_LOGGING_PROVIDER=$_APP_LOGGING_PROVIDER
       - OPR_EXECUTOR_LOGGING_CONFIG=$_APP_LOGGING_CONFIG
+      - OPR_EXECUTOR_STORAGE_DEVICE=$_APP_STORAGE_DEVICE
+      - OPR_EXECUTOR_STORAGE_S3_ACCESS_KEY=$_APP_STORAGE_S3_ACCESS_KEY
+      - OPR_EXECUTOR_STORAGE_S3_SECRET=$_APP_STORAGE_S3_SECRET
+      - OPR_EXECUTOR_STORAGE_S3_REGION=$_APP_STORAGE_S3_REGION
+      - OPR_EXECUTOR_STORAGE_S3_BUCKET=$_APP_STORAGE_S3_BUCKET
+      - OPR_EXECUTOR_STORAGE_DO_SPACES_ACCESS_KEY=$_APP_STORAGE_DO_SPACES_ACCESS_KEY
+      - OPR_EXECUTOR_STORAGE_DO_SPACES_SECRET=$_APP_STORAGE_DO_SPACES_SECRET
+      - OPR_EXECUTOR_STORAGE_DO_SPACES_REGION=$_APP_STORAGE_DO_SPACES_REGION
+      - OPR_EXECUTOR_STORAGE_DO_SPACES_BUCKET=$_APP_STORAGE_DO_SPACES_BUCKET
+      - OPR_EXECUTOR_STORAGE_BACKBLAZE_ACCESS_KEY=$_APP_STORAGE_BACKBLAZE_ACCESS_KEY
+      - OPR_EXECUTOR_STORAGE_BACKBLAZE_SECRET=$_APP_STORAGE_BACKBLAZE_SECRET
+      - OPR_EXECUTOR_STORAGE_BACKBLAZE_REGION=$_APP_STORAGE_BACKBLAZE_REGION
+      - OPR_EXECUTOR_STORAGE_BACKBLAZE_BUCKET=$_APP_STORAGE_BACKBLAZE_BUCKET
+      - OPR_EXECUTOR_STORAGE_LINODE_ACCESS_KEY=$_APP_STORAGE_LINODE_ACCESS_KEY
+      - OPR_EXECUTOR_STORAGE_LINODE_SECRET=$_APP_STORAGE_LINODE_SECRET
+      - OPR_EXECUTOR_STORAGE_LINODE_REGION=$_APP_STORAGE_LINODE_REGION
+      - OPR_EXECUTOR_STORAGE_LINODE_BUCKET=$_APP_STORAGE_LINODE_BUCKET
+      - OPR_EXECUTOR_STORAGE_WASABI_ACCESS_KEY=$_APP_STORAGE_WASABI_ACCESS_KEY
+      - OPR_EXECUTOR_STORAGE_WASABI_SECRET=$_APP_STORAGE_WASABI_SECRET
+      - OPR_EXECUTOR_STORAGE_WASABI_REGION=$_APP_STORAGE_WASABI_REGION
+      - OPR_EXECUTOR_STORAGE_WASABI_BUCKET=$_APP_STORAGE_WASABI_BUCKET
+
+  openruntimes-proxy:
+    container_name: openruntimes-proxy
+    hostname: proxy
+    <<: *x-logging
+    stop_signal: SIGINT
+    image: openruntimes/proxy:0.3.0
+    networks:
+      - appwrite
+      - runtimes
+    environment:
+      - OPR_PROXY_WORKER_PER_CORE=$_APP_WORKER_PER_CORE
+      - OPR_PROXY_ENV=$_APP_ENV
+      - OPR_PROXY_EXECUTOR_SECRET=$_APP_EXECUTOR_SECRET
+      - OPR_PROXY_SECRET=$_APP_EXECUTOR_SECRET
+      - OPR_PROXY_LOGGING_PROVIDER=$_APP_LOGGING_PROVIDER
+      - OPR_PROXY_LOGGING_CONFIG=$_APP_LOGGING_CONFIG
+      - OPR_PROXY_ALGORITHM=random
+      - OPR_PROXY_EXECUTORS=executor
+      - OPR_PROXY_HEALTHCHECK_INTERVAL=10000
+      - OPR_PROXY_MAX_TIMEOUT=600
+      - OPR_PROXY_HEALTHCHECK=enabled
 
   mariadb:
     image: mariadb:10.7 # fix issues when upgrading using: mysql_upgrade -u root -p
@@ -782,11 +774,13 @@ services:
       - MYSQL_DATABASE=${_APP_DB_SCHEMA}
       - MYSQL_USER=${_APP_DB_USER}
       - MYSQL_PASSWORD=${_APP_DB_PASS}
-    command: 'mysqld --innodb-flush-method=fsync --max_connections=${_APP_CONNECTIONS_MAX}'
+    command: 'mysqld --innodb-flush-method=fsync' # add ' --query_cache_size=0' for DB tests
+    # command: mv /var/lib/mysql/ib_logfile0 /var/lib/mysql/ib_logfile0.bu && mv /var/lib/mysql/ib_logfile1 /var/lib/mysql/ib_logfile1.bu
 
   # smtp:
   #   image: appwrite/smtp:1.2.0
   #   container_name: appwrite-smtp
+  #   restart: unless-stopped
   #   networks:
   #     - appwrite
   #   environment:
@@ -818,6 +812,26 @@ services:
   #     - appwrite
   #   volumes:
   #     - appwrite-uploads:/storage/uploads
+
+  influxdb:
+    image: appwrite/influxdb:1.5.0
+    container_name: appwrite-influxdb
+    <<: *x-logging
+    networks:
+      - appwrite
+    volumes:
+      - appwrite-influxdb:/var/lib/influxdb:rw
+
+  telegraf:
+    image: appwrite/telegraf:1.4.0
+    container_name: appwrite-telegraf
+    <<: *x-logging
+    networks:
+      - appwrite
+    environment:
+      - _APP_INFLUXDB_HOST
+      - _APP_INFLUXDB_PORT
+
   # Dev Tools Start ------------------------------------------------------------------------------------------
   #
   # The Appwrite Team uses the following tools to help debug, monitor and diagnose the Appwrite stack
@@ -828,6 +842,7 @@ services:
   # RequestCatcher - An HTTP server. Catches all system https calls and displays them using a simple HTTP API. Used to debug & tests webhooks and HTTP tasks
   # RedisCommander - A nice UI for exploring Redis data
   # Resque - A nice UI for exploring Redis pub/sub, view the different queues workloads, pending and failed tasks
+  # Chronograf - A nice UI for exploring InfluxDB data
   # Webgrind - A nice UI for exploring and debugging code-level stuff
 
   maildev: # used mainly for dev tests
@@ -852,26 +867,15 @@ services:
     image: adminer
     container_name: appwrite-adminer
     <<: *x-logging
+    restart: always
     ports:
       - 9506:8080
     networks:
       - appwrite
-  
-  # appwrite-volume-sync:
-  #   entrypoint: volume-sync
-  #   <<: *x-logging
-  #   container_name: appwrite-volume-sync
-  #   image: appwrite-dev
-  #   command:
-  #     - --source=/data/src/ --destination=/data/dest/ --interval=10
-  #   networks:
-  #     - appwrite
-  #   # volumes: # Mount the rsync source and destination directories  
-  #   #   - /nfs/config:/data/src 
-  #   #   - /storage/config:/data/dest
 
   # redis-commander:
   #   image: rediscommander/redis-commander:latest
+  #   restart: unless-stopped
   #   networks:
   #   - appwrite
   #   environment:
@@ -881,6 +885,7 @@ services:
 
   # resque:
   #   image: appwrite/resque-web:1.1.0
+  #   restart: unless-stopped
   #   networks:
   #     - appwrite
   #   ports:
@@ -890,6 +895,26 @@ services:
   #     - RESQUE_WEB_PORT=6379
   #     - RESQUE_WEB_HTTP_BASIC_AUTH_USER=user
   #     - RESQUE_WEB_HTTP_BASIC_AUTH_PASSWORD=password
+
+  # chronograf:
+  #   image: chronograf:1.6
+  #   container_name: appwrite-chronograf
+  #   restart: unless-stopped
+  #   networks:
+  #   - appwrite
+  #   volumes:
+  #   - appwrite-chronograf:/var/lib/chronograf
+  #   ports:
+  #   - "8888:8888"
+  #   environment:
+  #   - INFLUXDB_URL=http://influxdb:8086
+  #   - KAPACITOR_URL=http://kapacitor:9092
+  #   - AUTH_DURATION=48h
+  #   - TOKEN_SECRET=duperduper5674829!jwt
+  #   - GH_CLIENT_ID=d86f7145a41eacfc52cc
+  #   - GH_CLIENT_SECRET=9e0081062367a2134e7f2ea95ba1a32d08b6c8ab
+  #   - GH_ORGS=appwrite
+
   # webgrind:
   #   image: 'jokkedk/webgrind:latest'
   #   volumes:
@@ -927,4 +952,6 @@ volumes:
   appwrite-certificates:
   appwrite-functions:
   appwrite-builds:
+  appwrite-influxdb:
   appwrite-config:
+  # appwrite-chronograf:

docs/references/functions/create-deployment.md

@@ -2,4 +2,4 @@ Create a new function code deployment. Use this endpoint to upload a new version
 
 This endpoint accepts a tar.gz file compressed with your code. Make sure to include any dependencies your code has within the compressed file. You can learn more about code packaging in the [Appwrite Cloud Functions tutorial](/docs/functions).
 
-Use the "command" param to set the entry point used to execute your code.
+Use the "command" param to set the entrypoint used to execute your code.

docs/references/functions/create-variable.md

@@ -1 +1 @@
-Create a new function variable. These variables can be accessed within function in the `env` object under the request variable.
+Create a new function environment variable. These variables can be accessed in the function at runtime as environment variables.

docs/references/project/create-variable.md

@@ -0,0 +1 @@
+Create a new project variable. This variable will be accessible in all Appwrite Functions at runtime.

docs/references/project/delete-variable.md

@@ -0,0 +1 @@
+Delete a project variable by its unique ID. 

docs/references/project/get-variable.md

@@ -0,0 +1 @@
+Get a project variable by its unique ID.

docs/references/project/list-variables.md

@@ -0,0 +1 @@
+Get a list of all project variables. These variables will be accessible in all Appwrite Functions at runtime.

docs/references/project/update-variable.md

@@ -0,0 +1 @@
+Update project variable by its unique ID. This variable will be accessible in all Appwrite Functions at runtime.

docs/references/proxy/create-rule.md

@@ -0,0 +1 @@
+Create a new proxy rule.

docs/references/proxy/delete-rule.md

@@ -0,0 +1 @@
+Delete a proxy rule by its unique ID.

docs/references/proxy/get-rule.md

@@ -0,0 +1 @@
+Get a proxy rule by its unique ID.

docs/references/proxy/list-rules.md

@@ -0,0 +1 @@
+Get a list of all the proxy rules. You can use the query params to filter your results.

docs/services/proxy.md

@@ -0,0 +1,3 @@
+The Proxy service allows you to configure behavior for your attached domains. You can use proxy service to create rules that define what is returned on specific domains and subdomains.
+
+Proxy Rules can be configured to serve Appwrite API, which allows you to comply with first-party cookies, making your website more secure. It can also be configured to serve Appwrite Function, which lets you accept incoming webhooks, build custom API endpoints, and serve static files.

docs/services/vcs.md

@@ -0,0 +1,3 @@
+The VCS (Version Control System) service in Appwrite provides a way to interact with VCS providers like Git, GitHub etc. and manage your code repositories. You can use it to install the VCS app or to create, list, and delete repositories. You can also use the VCS service to clone, push, and pull code from your repositories.
+
+The VCS service helps you to either link an existing code repository to your Appwrite Function or to create a new repository from scratch using one of the available templates. If you link a repository to an Appwrite Function, it automatically creates a new deployment when you push changes.

docs/tutorials/add-oauth2-provider.md

@@ -42,7 +42,7 @@ app/config/providers.php
 
 Make sure to fill in all data needed and that your provider array key name:
 
-- is in [`camelCase`](https://en.wikipedia.org/wiki/Camel_case) format
+- is in [`camelCase`](https://en.wikipedia.org/wiki/Camel_case) format for sentence, but lowercase for names. `github` must be all lowercased, but `paypalSandbox` should have uppercase S
 - has no spaces or special characters
 
 > Please make sure to keep the list of providers in `providers.php` in the alphabetical order A-Z.

phpcs.xml

@@ -7,6 +7,8 @@
     <ini name="memory_limit" value="4096M"/>
     <!-- Exclude SDK's for performance reasons -->
     <exclude-pattern>./app/sdks</exclude-pattern>
+    <!-- Exclude functions as they are in different languages -->
+    <exclude-pattern>./tests/resources/functions</exclude-pattern>
     <!-- Exclude console -->
     <exclude-pattern>./app/console</exclude-pattern>
     <!-- Ignore max line width -->

src/Appwrite/Auth/OAuth2.php

@@ -208,7 +208,7 @@ abstract class OAuth2
 
         \curl_close($ch);
 
-        if ($code != 200) {
+        if ($code >= 400) {
             throw new Exception($response, $code);
         }
 

src/Appwrite/Auth/OAuth2/Exception.php

@@ -16,13 +16,13 @@ class Exception extends AppwriteException
         $this->message = $response;
         $decoded = json_decode($response, true);
         if (\is_array($decoded)) {
-            if (\is_array($decoded['error'])) {
+            if (\is_array($decoded['error'] ?? '')) {
                 $this->error = $decoded['error']['status'];
                 $this->errorDescription = $decoded['error']['message'];
                 $this->message =  $this->error . ': ' . $this->errorDescription;
             } else {
-                $this->error = $decoded['error'];
-                $this->errorDescription = $decoded['error_description'];
+                $this->error = $decoded['error'] ?? $decoded['message'] ?? 'Unknown error';
+                $this->errorDescription = $decoded['error_description'] ?? 'No description';
                 $this->message =  $this->error . ': ' . $this->errorDescription;
             }
         }

src/Appwrite/Auth/OAuth2/Firebase.php

@@ -27,6 +27,38 @@ class Firebase extends OAuth2
         'https://www.googleapis.com/auth/userinfo.profile'
     ];
 
+    /**
+     * @var array
+     */
+    protected array $iamPermissions = [
+        // Database
+        'datastore.databases.get',
+        'datastore.databases.list',
+        'datastore.entities.get',
+        'datastore.entities.list',
+        'datastore.indexes.get',
+        'datastore.indexes.list',
+        // Generic Firebase permissions
+        'firebase.projects.get',
+
+        // Auth
+        'firebaseauth.configs.get',
+        'firebaseauth.configs.getHashConfig',
+        'firebaseauth.configs.getSecret',
+        'firebaseauth.users.get',
+        'identitytoolkit.tenants.get',
+        'identitytoolkit.tenants.list',
+
+        // IAM Assignment
+        'iam.serviceAccounts.list',
+
+        // Storage
+        'storage.buckets.get',
+        'storage.buckets.list',
+        'storage.objects.get',
+        'storage.objects.list'
+    ];
+
     /**
      * @return string
      */
@@ -198,7 +230,7 @@ class Firebase extends OAuth2
     /*
         Be careful with the setIAMPolicy method, it will overwrite all existing policies
     **/
-    public function assignIAMRoles(string $accessToken, string $email, string $projectId)
+    public function assignIAMRole(string $accessToken, string $email, string $projectId, array $role)
     {
         // Get IAM Roles
         $iamRoles = $this->request('POST', 'https://cloudresourcemanager.googleapis.com/v1/projects/' . $projectId . ':getIamPolicy', [
@@ -209,14 +241,7 @@ class Firebase extends OAuth2
         $iamRoles = \json_decode($iamRoles, true);
 
         $iamRoles['bindings'][] = [
-            'role' => 'roles/identitytoolkit.admin',
-            'members' => [
-                'serviceAccount:' . $email
-            ]
-        ];
-
-        $iamRoles['bindings'][] = [
-            'role' => 'roles/firebase.admin',
+            'role' => $role['name'],
             'members' => [
                 'serviceAccount:' . $email
             ]
@@ -226,14 +251,69 @@ class Firebase extends OAuth2
         $this->request('POST', 'https://cloudresourcemanager.googleapis.com/v1/projects/' . $projectId . ':setIamPolicy', [
             'Authorization: Bearer ' . \urlencode($accessToken),
             'Content-Type: application/json'
-        ], json_encode([
+        ], \json_encode([
             'policy' => $iamRoles
         ]));
     }
 
+    private function generateRandomString($length = 10): string
+    {
+        $characters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ';
+        $charactersLength = strlen($characters);
+        $randomString = '';
+        for ($i = 0; $i < $length; $i++) {
+            $randomString .= $characters[random_int(0, $charactersLength - 1)];
+        }
+        return $randomString;
+    }
+
+    private function createCustomRole(string $accessToken, string $projectId): array
+    {
+        // Check if role already exists
+        try {
+            $role = $this->request('GET', 'https://iam.googleapis.com/v1/projects/' . $projectId . '/roles/appwriteMigrations', [
+                'Content-Type: application/json',
+                'Authorization: Bearer ' . \urlencode($accessToken),
+            ]);
+
+            $role = \json_decode($role, true);
+
+            return $role;
+        } catch (\Exception $e) {
+            if ($e->getCode() !== 404) {
+                throw $e;
+            }
+        }
+
+        // Create role if doesn't exist or isn't correct
+        $role = $this->request(
+            'POST',
+            'https://iam.googleapis.com/v1/projects/' . $projectId . '/roles/',
+            [
+                'Content-Type: application/json',
+                'Authorization: Bearer ' . \urlencode($accessToken),
+            ],
+            \json_encode(
+                [
+                    'roleId' => 'appwriteMigrations',
+                    'role' => [
+                        'title' => 'Appwrite Migrations',
+                        'description' => 'A helper role for Appwrite Migrations',
+                        'includedPermissions' => $this->iamPermissions,
+                        'stage' => 'GA'
+                    ]
+                ]
+            )
+        );
+
+        return json_decode($role, true);
+    }
+
     public function createServiceAccount(string $accessToken, string $projectId): array
     {
         // Create Service Account
+        $uid = $this->generateRandomString();
+
         $response = $this->request(
             'POST',
             'https://iam.googleapis.com/v1/projects/' . $projectId . '/serviceAccounts',
@@ -241,17 +321,22 @@ class Firebase extends OAuth2
                 'Authorization: Bearer ' . \urlencode($accessToken),
                 'Content-Type: application/json'
             ],
-            json_encode([
-                'accountId' => 'appwrite-migrations',
+            \json_encode([
+                'accountId' => 'appwrite-' . $uid,
                 'serviceAccount' => [
-                    'displayName' => 'Appwrite Migrations'
+                    'displayName' => 'Appwrite Migrations ' . $uid
                 ]
             ])
         );
 
         $response = json_decode($response, true);
 
-        $this->assignIAMRoles($accessToken, $response['email'], $projectId);
+        // Create and assign IAM Roles
+        $role = $this->createCustomRole($accessToken, $projectId);
+
+        \sleep(1); // Wait for IAM to propagate changes.
+
+        $this->assignIAMRole($accessToken, $response['email'], $projectId, $role);
 
         // Create Service Account Key
         $responseKey = $this->request(
@@ -267,4 +352,38 @@ class Firebase extends OAuth2
 
         return json_decode(base64_decode($responseKey['privateKeyData']), true);
     }
+
+    public function cleanupServiceAccounts(string $accessToken, string $projectId)
+    {
+        // List Service Accounts
+        $response = $this->request(
+            'GET',
+            'https://iam.googleapis.com/v1/projects/' . $projectId . '/serviceAccounts',
+            [
+                'Authorization: Bearer ' . \urlencode($accessToken),
+                'Content-Type: application/json'
+            ]
+        );
+
+        $response = json_decode($response, true);
+
+        if (empty($response['accounts'])) {
+            return false;
+        }
+
+        foreach ($response['accounts'] as $account) {
+            if (strpos($account['email'], 'appwrite-') !== false) {
+                $this->request(
+                    'DELETE',
+                    'https://iam.googleapis.com/v1/projects/' . $projectId . '/serviceAccounts/' . $account['email'],
+                    [
+                        'Authorization: Bearer ' . \urlencode($accessToken),
+                        'Content-Type: application/json'
+                    ]
+                );
+            }
+        }
+
+        return true;
+    }
 }

src/Appwrite/Auth/OAuth2/Github.php

@@ -208,4 +208,15 @@ class Github extends OAuth2
 
         return $this->user;
     }
+
+    public function createRepository(string $accessToken, string $repositoryName, bool $private): array
+    {
+        $repository = $this->request('POST', 'https://api.github.com/user/repos', ['Authorization: token ' . \urlencode($accessToken)], \json_encode([
+            'name' => $repositoryName,
+            'private' => $private
+        ]));
+
+        $repository = \json_decode($repository, true);
+        return $repository;
+    }
 }

src/Appwrite/Detector/Detector.php

@@ -77,10 +77,14 @@ class Detector
      */
     public function getDevice(): array
     {
+        $deviceName = $this->getDetector()->getDeviceName();
+        $deviceBrand = $this->getDetector()->getBrandName();
+        $deviceModel = $this->getDetector()->getModel();
+
         return [
-            'deviceName' => $this->getDetector()->getDeviceName(),
-            'deviceBrand' => $this->getDetector()->getBrandName(),
-            'deviceModel' => $this->getDetector()->getModel(),
+            'deviceName' => empty($deviceName) ? null : $deviceName,
+            'deviceBrand' => empty($deviceBrand) ? null : $deviceBrand,
+            'deviceModel' => empty($deviceModel) ? null : $deviceModel,
         ];
     }
 

src/Appwrite/Event/Build.php

@@ -10,12 +10,26 @@ class Build extends Event
     protected string $type = '';
     protected ?Document $resource = null;
     protected ?Document $deployment = null;
+    protected ?Document $template = null;
 
     public function __construct()
     {
         parent::__construct(Event::BUILDS_QUEUE_NAME, Event::BUILDS_CLASS_NAME);
     }
 
+    /**
+     * Sets template for the build event.
+     *
+     * @param Document $template
+     * @return self
+     */
+    public function setTemplate(Document $template): self
+    {
+        $this->template = $template;
+
+        return $this;
+    }
+
     /**
      * Sets resource document for the build event.
      *
@@ -97,7 +111,8 @@ class Build extends Event
             'project' => $this->project,
             'resource' => $this->resource,
             'deployment' => $this->deployment,
-            'type' => $this->type
+            'type' => $this->type,
+            'template' => $this->template
         ]);
     }
 }

src/Appwrite/Event/Delete.php

@@ -13,7 +13,6 @@ class Delete extends Event
     protected ?string $datetime = null;
     protected ?string $hourlyUsageRetentionDatetime = null;
 
-
     public function __construct()
     {
         parent::__construct(Event::DELETE_QUEUE_NAME, Event::DELETE_CLASS_NAME);
@@ -127,7 +126,7 @@ class Delete extends Event
             'document' => $this->document,
             'resource' => $this->resource,
             'datetime' => $this->datetime,
-            'hourlyUsageRetentionDatetime' => $this->hourlyUsageRetentionDatetime,
+            'hourlyUsageRetentionDatetime' => $this->hourlyUsageRetentionDatetime
         ]);
     }
 }

src/Appwrite/Event/Func.php

@@ -10,7 +10,10 @@ class Func extends Event
 {
     protected string $jwt = '';
     protected string $type = '';
-    protected string $data = '';
+    protected string $body = '';
+    protected string $path = '';
+    protected string $method = '';
+    protected array $headers = [];
     protected ?Document $function = null;
     protected ?Document $execution = null;
 
@@ -89,14 +92,53 @@ class Func extends Event
     }
 
     /**
-     * Sets custom data for the function event.
+     * Sets custom body for the function event.
      *
-     * @param string $data
+     * @param string $body
      * @return self
      */
-    public function setData(string $data): self
+    public function setBody(string $body): self
     {
-        $this->data = $data;
+        $this->body = $body;
+
+        return $this;
+    }
+
+    /**
+     * Sets custom method for the function event.
+     *
+     * @param string $method
+     * @return self
+     */
+    public function setMethod(string $method): self
+    {
+        $this->method = $method;
+
+        return $this;
+    }
+
+    /**
+     * Sets custom path for the function event.
+     *
+     * @param string $path
+     * @return self
+     */
+    public function setPath(string $path): self
+    {
+        $this->path = $path;
+
+        return $this;
+    }
+
+    /**
+     * Sets custom headers for the function event.
+     *
+     * @param string $headers
+     * @return self
+     */
+    public function setHeaders(array $headers): self
+    {
+        $this->headers = $headers;
 
         return $this;
     }
@@ -159,7 +201,10 @@ class Func extends Event
             'jwt' => $this->jwt,
             'payload' => $this->payload,
             'events' => $events,
-            'data' => $this->data,
+            'body' => $this->body,
+            'path' => $this->path,
+            'headers' => $this->headers,
+            'method' => $this->method,
         ]);
     }
 

src/Appwrite/Event/Mail.php

@@ -8,11 +8,11 @@ use Utopia\Database\Document;
 class Mail extends Event
 {
     protected string $recipient = '';
-    protected string $from = '';
     protected string $name = '';
     protected string $subject = '';
     protected string $body = '';
     protected array $smtp = [];
+    protected array $variables = [];
 
     public function __construct()
     {
@@ -65,29 +65,6 @@ class Mail extends Event
         return $this->recipient;
     }
 
-    /**
-     * Sets from for the mail event.
-     *
-     * @param string $from
-     * @return self
-     */
-    public function setFrom(string $from): self
-    {
-        $this->from = $from;
-
-        return $this;
-    }
-
-    /**
-     * Returns from for mail event.
-     *
-     * @return string
-     */
-    public function getFrom(): string
-    {
-        return $this->from;
-    }
-
     /**
      * Sets body for the mail event.
      *
@@ -166,7 +143,7 @@ class Mail extends Event
      */
     public function setSmtpUsername(string $username): self
     {
-        $this->smtp['username'];
+        $this->smtp['username'] = $username;
         return $this;
     }
 
@@ -178,7 +155,19 @@ class Mail extends Event
      */
     public function setSmtpPassword(string $password): self
     {
-        $this->smtp['password'];
+        $this->smtp['password'] = $password;
+        return $this;
+    }
+
+    /**
+     * Set SMTP secure
+     *
+     * @param string $password
+     * @return self
+     */
+    public function setSmtpSecure(string $secure): self
+    {
+        $this->smtp['secure'] = $secure;
         return $this;
     }
 
@@ -194,6 +183,18 @@ class Mail extends Event
         return $this;
     }
 
+    /**
+     * Set SMTP sender name
+     *
+     * @param string $senderName
+     * @return self
+     */
+    public function setSmtpSenderName(string $senderName): self
+    {
+        $this->smtp['senderName'] = $senderName;
+        return $this;
+    }
+
     /**
      * Set SMTP reply to
      *
@@ -246,6 +247,16 @@ class Mail extends Event
         return $this->smtp['password'] ?? '';
     }
 
+    /**
+     * Get SMTP secure
+     *
+     * @return string
+     */
+    public function getSmtpSecure(): string
+    {
+        return $this->smtp['secure'] ?? '';
+    }
+
     /**
      * Get SMTP sender email
      *
@@ -256,6 +267,16 @@ class Mail extends Event
         return $this->smtp['senderEmail'] ?? '';
     }
 
+    /**
+     * Get SMTP sender name
+     *
+     * @return string
+     */
+    public function getSmtpSenderName(): string
+    {
+        return $this->smtp['senderName'] ?? '';
+    }
+
     /**
      * Get SMTP reply to
      *
@@ -266,6 +287,28 @@ class Mail extends Event
         return $this->smtp['replyTo'] ?? '';
     }
 
+    /**
+     * Get Email Variables
+     *
+     * @return array
+     */
+    public function getVariables(): array
+    {
+        return $this->variables;
+    }
+
+    /**
+     * Set Email Variables
+     *
+     * @param array $variables
+     * @return self
+     */
+    public function setVariables(array $variables): self
+    {
+        $this->variables = $variables;
+        return $this;
+    }
+
     /**
      * Executes the event and sends it to the mails worker.
      *
@@ -275,12 +318,12 @@ class Mail extends Event
     public function trigger(): string|bool
     {
         return Resque::enqueue($this->queue, $this->class, [
-            'from' => $this->from,
             'recipient' => $this->recipient,
             'name' => $this->name,
             'subject' => $this->subject,
             'body' => $this->body,
             'smtp' => $this->smtp,
+            'variables' => $this->variables,
             'events' => Event::generateEvents($this->getEvent(), $this->getParams())
         ]);
     }

src/Appwrite/Extend/Exception.php

@@ -54,6 +54,7 @@ class Exception extends \Exception
     public const GENERAL_PROTOCOL_UNSUPPORTED      = 'general_protocol_unsupported';
     public const GENERAL_CODES_DISABLED            = 'general_codes_disabled';
     public const GENERAL_USAGE_DISABLED            = 'general_usage_disabled';
+    public const GENERAL_NOT_IMPLEMENTED           = 'general_not_implemented';
 
     /** Users */
     public const USER_COUNT_EXCEEDED               = 'user_count_exceeded';
@@ -118,9 +119,17 @@ class Exception extends \Exception
     public const STORAGE_INVALID_RANGE             = 'storage_invalid_range';
     public const STORAGE_INVALID_APPWRITE_ID       = 'storage_invalid_appwrite_id';
 
+    /** VCS */
+    public const INSTALLATION_NOT_FOUND              = 'installation_not_found';
+    public const PROVIDER_REPOSITORY_NOT_FOUND       = 'provider_repository_not_found';
+    public const REPOSITORY_NOT_FOUND                = 'repository_not_found';
+    public const PROVIDER_CONTRIBUTION_CONFLICT      = 'provider_contribution_conflict';
+    public const GENERAL_PROVIDER_FAILURE            = 'general_provider_failure';
+
     /** Functions */
     public const FUNCTION_NOT_FOUND                = 'function_not_found';
     public const FUNCTION_RUNTIME_UNSUPPORTED      = 'function_runtime_unsupported';
+    public const FUNCTION_ENTRYPOINT_MISSING      = 'function_entrypoint_missing';
 
     /** Deployments */
     public const DEPLOYMENT_NOT_FOUND              = 'deployment_not_found';
@@ -186,6 +195,16 @@ class Exception extends \Exception
     /** Webhooks */
     public const WEBHOOK_NOT_FOUND                 = 'webhook_not_found';
 
+    /** Router */
+    public const ROUTER_HOST_NOT_FOUND             = 'router_host_not_found';
+    public const ROUTER_DOMAIN_NOT_CONFIGURED      = 'router_domain_not_configured';
+
+    /** Proxy */
+    public const RULE_RESOURCE_NOT_FOUND            = 'rule_resource_not_found';
+    public const RULE_NOT_FOUND                     = 'rule_not_found';
+    public const RULE_ALREADY_EXISTS                = 'rule_already_exists';
+    public const RULE_VERIFICATION_FAILED           = 'rule_verification_failed';
+
     /** Keys */
     public const KEY_NOT_FOUND                     = 'key_not_found';
 
@@ -196,13 +215,6 @@ class Exception extends \Exception
     /** Platform */
     public const PLATFORM_NOT_FOUND                = 'platform_not_found';
 
-    /** Domain */
-    public const DOMAIN_NOT_FOUND                  = 'domain_not_found';
-    public const DOMAIN_ALREADY_EXISTS             = 'domain_already_exists';
-    public const DOMAIN_FORBIDDEN                  = 'domain_forbidden';
-    public const DOMAIN_VERIFICATION_FAILED        = 'domain_verification_failed';
-    public const DOMAIN_TARGET_INVALID             = 'domain_target_invalid';
-
     /** GraphqQL */
     public const GRAPHQL_NO_QUERY                  = 'graphql_no_query';
     public const GRAPHQL_TOO_MANY_QUERIES          = 'graphql_too_many_queries';

src/Appwrite/GraphQL/Resolvers.php

@@ -269,7 +269,7 @@ class Resolvers
         try {
             $route = $utopia->match($request, fresh: true);
 
-            $utopia->execute($route, $request);
+            $utopia->execute($route, $request, $response);
         } catch (\Throwable $e) {
             if ($beforeReject) {
                 $e = $beforeReject($e);

src/Appwrite/GraphQL/Types/Mapper.php

@@ -256,10 +256,12 @@ class Mapper
             case 'Appwrite\Utopia\Database\Validator\Queries\Indexes':
             case 'Appwrite\Utopia\Database\Validator\Queries\Databases':
             case 'Appwrite\Utopia\Database\Validator\Queries\Deployments':
+            case 'Appwrite\Utopia\Database\Validator\Queries\Installations':
             case 'Utopia\Database\Validator\Queries\Documents':
             case 'Appwrite\Utopia\Database\Validator\Queries\Executions':
             case 'Appwrite\Utopia\Database\Validator\Queries\Files':
             case 'Appwrite\Utopia\Database\Validator\Queries\Functions':
+            case 'Appwrite\Utopia\Database\Validator\Queries\Rules':
             case 'Appwrite\Utopia\Database\Validator\Queries\Memberships':
             case 'Utopia\Database\Validator\Permissions':
             case 'Appwrite\Utopia\Database\Validator\Queries\Projects':

src/Appwrite/Messaging/Adapter/Realtime.php

@@ -260,6 +260,11 @@ class Realtime extends Adapter
                 $channels[] = 'account.' . $parts[1];
                 $roles = [Role::user(ID::custom($parts[1]))->toString()];
                 break;
+            case 'rules':
+                $channels[] = 'console';
+                $projectId = 'console';
+                $roles = [Role::team($project->getAttribute('teamId'))->toString()];
+                break;
             case 'teams':
                 if ($parts[2] === 'memberships') {
                     $permissionsChanged = $parts[4] ?? false;
@@ -325,6 +330,11 @@ class Realtime extends Adapter
                     $roles = [Role::team($project->getAttribute('teamId'))->toString()];
                 }
 
+                break;
+            case 'migrations':
+                $channels[] = 'console';
+                $projectId = 'console';
+                $roles = [Role::team($project->getAttribute('teamId'))->toString()];
                 break;
         }
 

src/Appwrite/Migration/Migration.php

@@ -77,7 +77,11 @@ abstract class Migration
         Authorization::disable();
         Authorization::setDefaultStatus(false);
 
-        $this->collections = array_merge([
+        $this->collections = Config::getParam('collections', []);
+
+        $projectCollections = $this->collections['projects'];
+
+        $this->collections['projects'] = array_merge([
             '_metadata' => [
                 '$id' => ID::custom('_metadata'),
                 '$collection' => Database::METADATA
@@ -90,7 +94,7 @@ abstract class Migration
                 '$id' => ID::custom('abuse'),
                 '$collection' => Database::METADATA
             ]
-        ], Config::getParam('collections', []));
+        ], $projectCollections);
     }
 
     /**
@@ -131,7 +135,14 @@ abstract class Migration
      */
     public function forEachDocument(callable $callback): void
     {
-        foreach ($this->collections as $collection) {
+        $internalProjectId = $this->project->getInternalId();
+
+        $collections = match ($internalProjectId) {
+            'console' => $this->collections['console'],
+            default => $this->collections['projects'],
+        };
+
+        foreach ($collections as $collection) {
             if ($collection['$collection'] !== Database::METADATA) {
                 continue;
             }
@@ -148,12 +159,12 @@ abstract class Migration
                         $old = $document->getArrayCopy();
                         $new = call_user_func($callback, $document);
 
-                        if (is_null($new) || !self::hasDifference($new->getArrayCopy(), $old)) {
+                        if (is_null($new) || $new->getArrayCopy() == $old) {
                             return;
                         }
 
                         try {
-                            $new = $this->projectDB->updateDocument($document->getCollection(), $document->getId(), $document);
+                            $this->projectDB->updateDocument($document->getCollection(), $document->getId(), $document);
                         } catch (\Throwable $th) {
                             Console::error('Failed to update document: ' . $th->getMessage());
                             return;
@@ -199,32 +210,6 @@ abstract class Migration
         } while (!is_null($nextDocument));
     }
 
-    /**
-     * Checks 2 arrays for differences.
-     *
-     * @param array $array1
-     * @param array $array2
-     * @return bool
-     */
-    public static function hasDifference(array $array1, array $array2): bool
-    {
-        foreach ($array1 as $key => $value) {
-            if (is_array($value)) {
-                if (!isset($array2[$key]) || !is_array($array2[$key])) {
-                    return true;
-                } else {
-                    if (self::hasDifference($value, $array2[$key])) {
-                        return true;
-                    }
-                }
-            } elseif (!array_key_exists($key, $array2) || $array2[$key] !== $value) {
-                return true;
-            }
-        }
-
-        return false;
-    }
-
     /**
      * Creates colletion from the config collection.
      *
@@ -237,10 +222,15 @@ abstract class Migration
     {
         $name ??= $id;
 
+        $collectionType = match ($this->project->getInternalId()) {
+            'console' => 'console',
+            default => 'projects',
+        };
+
         if (!$this->projectDB->exists(App::getEnv('_APP_DB_SCHEMA', 'appwrite'), $name)) {
             $attributes = [];
             $indexes = [];
-            $collection = $this->collections[$id];
+            $collection = $this->collections[$collectionType][$id];
 
             foreach ($collection['attributes'] as $attribute) {
                 $attributes[] = new Document([
@@ -286,10 +276,22 @@ abstract class Migration
     public function createAttributeFromCollection(Database $database, string $collectionId, string $attributeId, string $from = null): void
     {
         $from ??= $collectionId;
-        $collection = Config::getParam('collections', [])[$from] ?? null;
-        if (is_null($collection)) {
-            throw new Exception("Collection {$collectionId} not found");
+
+        $collectionType = match ($this->project->getInternalId()) {
+            'console' => 'console',
+            default => 'projects',
+        };
+
+        if ($from === 'files') {
+            $collectionType = 'buckets';
         }
+
+        $collection = $this->collections[$collectionType][$from] ?? null;
+
+        if (is_null($collection)) {
+            throw new Exception("Collection {$from} not found");
+        }
+
         $attributes = $collection['attributes'];
 
         $attributeKey = array_search($attributeId, array_column($attributes, '$id'));
@@ -332,17 +334,24 @@ abstract class Migration
     public function createIndexFromCollection(Database $database, string $collectionId, string $indexId, string $from = null): void
     {
         $from ??= $collectionId;
-        $collection = Config::getParam('collections', [])[$collectionId] ?? null;
+
+        $collectionType = match ($this->project->getInternalId()) {
+            'console' => 'console',
+            default => 'projects',
+        };
+
+        $collection = $this->collections[$collectionType][$from] ?? null;
 
         if (is_null($collection)) {
             throw new Exception("Collection {$collectionId} not found");
         }
+
         $indexes = $collection['indexes'];
 
         $indexKey = array_search($indexId, array_column($indexes, '$id'));
 
         if ($indexKey === false) {
-            throw new Exception("Attribute {$indexId} not found");
+            throw new Exception("Index {$indexId} not found");
         }
 
         $index = $indexes[$indexKey];

src/Appwrite/Migration/Version/V17.php

@@ -70,6 +70,18 @@ class V17 extends Migration
             $this->projectDB->setNamespace("_{$this->project->getInternalId()}");
 
             switch ($id) {
+                case 'builds':
+                    try {
+                        /**
+                         * Create 'size' attribute
+                         */
+                        $this->createAttributeFromCollection($this->projectDB, $id, 'size');
+                        $this->projectDB->deleteCachedCollection($id);
+                    } catch (\Throwable $th) {
+                        Console::warning("'size' from {$id}: {$th->getMessage()}");
+                    }
+
+                    break;
                 case 'files':
                     try {
                         /**

src/Appwrite/Migration/Version/V19.php

@@ -2,12 +2,14 @@
 
 namespace Appwrite\Migration\Version;
 
-use Appwrite\Auth\Auth;
-use Utopia\Config\Config;
 use Appwrite\Migration\Migration;
+use Utopia\App;
 use Utopia\CLI\Console;
+use Utopia\Config\Config;
 use Utopia\Database\Database;
+use Utopia\Database\DateTime;
 use Utopia\Database\Document;
+use Utopia\Database\Exception;
 
 class V19 extends Migration
 {
@@ -28,12 +30,6 @@ class V19 extends Migration
         Console::log('Migrating Project: ' . $this->project->getAttribute('name') . ' (' . $this->project->getId() . ')');
         $this->projectDB->setNamespace("_{$this->project->getInternalId()}");
 
-        $this->alterPermissionIndex('_metadata');
-        $this->alterUidType('_metadata');
-
-        Console::info('Migrating Databases');
-        $this->migrateDatabases();
-
         Console::info('Migrating Collections');
         $this->migrateCollections();
 
@@ -42,29 +38,29 @@ class V19 extends Migration
 
         Console::info('Migrating Documents');
         $this->forEachDocument([$this, 'fixDocument']);
+
+        Console::log('Cleaning Up Collections');
+        $this->cleanCollections();
     }
 
     /**
-     * Migrate all Databases.
+     * Migrating all Bucket tables.
      *
      * @return void
      * @throws \Exception
+     * @throws \PDOException
      */
-    private function migrateDatabases(): void
+    protected function migrateBuckets(): void
     {
-        foreach ($this->documentsIterator('databases') as $database) {
-            Console::log("Migrating Collections of {$database->getId()} ({$database->getAttribute('name')})");
+        foreach ($this->documentsIterator('buckets') as $bucket) {
+            $id = "bucket_{$bucket->getInternalId()}";
+            Console::log("Migrating Bucket {$id} {$bucket->getId()} ({$bucket->getAttribute('name')})");
 
-            $databaseTable = "database_{$database->getInternalId()}";
-
-            $this->alterPermissionIndex($databaseTable);
-            $this->alterUidType($databaseTable);
-
-            foreach ($this->documentsIterator($databaseTable) as $collection) {
-                $collectionTable = "{$databaseTable}_collection_{$collection->getInternalId()}";
-                Console::log("Migrating Collections of {$collectionTable} {$collection->getId()} ({$collection->getAttribute('name')})");
-                $this->alterPermissionIndex($collectionTable);
-                $this->alterUidType($collectionTable);
+            try {
+                $this->createAttributeFromCollection($this->projectDB, $id, 'bucketInternalId', 'files');
+                $this->projectDB->deleteCachedCollection($id);
+            } catch (\Throwable $th) {
+                Console::warning("'bucketInternalId' from {$id}: {$th->getMessage()}");
             }
         }
     }
@@ -73,36 +69,457 @@ class V19 extends Migration
      * Migrate all Collections.
      *
      * @return void
+     * @throws \Throwable
+     * @throws Exception
      */
     private function migrateCollections(): void
     {
-        foreach ($this->collections as $collection) {
+        $internalProjectId = $this->project->getInternalId();
+        $collectionType = match ($internalProjectId) {
+            'console' => 'console',
+            default => 'projects',
+        };
+        $collections = $this->collections[$collectionType];
+        foreach ($collections as $collection) {
             $id = $collection['$id'];
 
+            if ($id === 'schedules' && $internalProjectId === 'console') {
+                continue;
+            }
+
             Console::log("Migrating Collection \"{$id}\"");
 
-            $this->projectDB->setNamespace("_{$this->project->getInternalId()}");
+            $this->projectDB->setNamespace("_$internalProjectId");
 
             switch ($id) {
-                case 'projects':
+                case '_metadata':
+                    $this->createCollection('identities');
+                    $this->createCollection('migrations');
+                    // Holding off on this until a future release
+                    // $this->createCollection('statsLogger');
+                    break;
+                case 'attributes':
+                case 'indexes':
                     try {
-                        /**
-                         * Create 'passwordHistory' attribute
-                         */
-                        $this->createAttributeFromCollection($this->projectDB, $id, 'smtp');
-                        $this->createAttributeFromCollection($this->projectDB, $id, 'templates');
-                        $this->projectDB->deleteCachedCollection($id);
+                        $this->projectDB->updateAttribute($id, 'databaseInternalId', required: true);
                     } catch (\Throwable $th) {
-                        Console::warning("'SMTP and Templates' from {$id}: {$th->getMessage()}");
+                        Console::warning("'databaseInternalId' from {$id}: {$th->getMessage()}");
                     }
+
+                    try {
+                        $this->projectDB->updateAttribute($id, 'collectionInternalId', required: true);
+                    } catch (\Throwable $th) {
+                        Console::warning("'collectionInternalId' from {$id}: {$th->getMessage()}");
+                    }
+
+                    try {
+                        $this->createAttributeFromCollection($this->projectDB, $id, 'error');
+                    } catch (\Throwable $th) {
+                        Console::warning("'error' from {$id}: {$th->getMessage()}");
+                    }
+
+                    $this->projectDB->deleteCachedCollection($id);
+
+                    break;
+                case 'buckets':
+                    // Recreate indexes so they're the right size
+                    $indexesToDelete = [
+                        '_key_name',
+                    ];
+                    foreach ($indexesToDelete as $index) {
+                        try {
+                            $this->projectDB->deleteIndex($id, $index);
+                        } catch (\Throwable $th) {
+                            Console::warning("'$index' from {$id}: {$th->getMessage()}");
+                        }
+                    }
+
+                    $indexesToCreate = [
+                        ...$indexesToDelete
+                    ];
+
+                    foreach ($indexesToCreate as $index) {
+                        try {
+                            $this->createIndexFromCollection($this->projectDB, $id, $index);
+                        } catch (\Throwable $th) {
+                            Console::warning("'$index' from {$id}: {$th->getMessage()}");
+                        }
+                    }
+
+                    $this->projectDB->deleteCachedCollection($id);
+
+                    break;
+                case 'builds':
+                    try {
+                        $this->createAttributeFromCollection($this->projectDB, $id, 'deploymentInternalId');
+                    } catch (\Throwable $th) {
+                        Console::warning("'deploymentInternalId' from {$id}: {$th->getMessage()}");
+                    }
+
+                    try {
+                        $this->createAttributeFromCollection($this->projectDB, $id, 'logs');
+                    } catch (\Throwable $th) {
+                        Console::warning("'logs' from {$id}: {$th->getMessage()}");
+                    }
+
+                    $this->projectDB->deleteCachedCollection($id);
+
+                    break;
+                case 'certificates':
+                    try {
+                        $this->projectDB->renameAttribute($id, 'log', 'logs');
+                    } catch (\Throwable $th) {
+                        Console::warning("'errors' from {$id}: {$th->getMessage()}");
+                    }
+
+                    try {
+                        $this->projectDB->updateAttribute($id, 'logs', size: 1000000);
+                    } catch (\Throwable $th) {
+                        Console::warning("'errors' from {$id}: {$th->getMessage()}");
+                    }
+
+                    $this->projectDB->deleteCachedCollection($id);
+
+                    break;
+                case 'databases':
+                    try {
+                        $this->createAttributeFromCollection($this->projectDB, $id, 'enabled');
+                    } catch (\Throwable $th) {
+                        Console::warning("'enabled' from {$id}: {$th->getMessage()}");
+                    }
+
+                    $this->projectDB->deleteCachedCollection($id);
+
+                    break;
+                case 'deployments':
+                    $attributesToCreate = [
+                        'resourceInternalId',
+                        'buildInternalId',
+                        'type',
+                    ];
+                    foreach ($attributesToCreate as $attribute) {
+                        try {
+                            $this->createAttributeFromCollection($this->projectDB, $id, $attribute);
+                        } catch (\Throwable $th) {
+                            Console::warning("$attribute from {$id}: {$th->getMessage()}");
+                        }
+                    }
+
+                    // Recreate indexes so they're the right size
+                    $indexesToDelete = [
+                        '_key_entrypoint',
+                        '_key_resource',
+                        '_key_resource_type',
+                        '_key_buildId',
+                    ];
+                    foreach ($indexesToDelete as $index) {
+                        try {
+                            $this->projectDB->deleteIndex($id, $index);
+                        } catch (\Throwable $th) {
+                            Console::warning("'$index' from {$id}: {$th->getMessage()}");
+                        }
+                    }
+
+                    $indexesToCreate = [
+                        '_key_resource',
+                        '_key_resource_type',
+                        '_key_buildId',
+                    ];
+                    foreach ($indexesToCreate as $index) {
+                        try {
+                            $this->createIndexFromCollection($this->projectDB, $id, $index);
+                        } catch (\Throwable $th) {
+                            Console::warning("'$index' from {$id}: {$th->getMessage()}");
+                        }
+                    }
+
+                    $this->projectDB->deleteCachedCollection($id);
+
+                    break;
+                case 'executions':
+                    try {
+                        $this->createAttributeFromCollection($this->projectDB, $id, 'functionInternalId');
+                    } catch (\Throwable $th) {
+                        Console::warning("'functionInternalId' from {$id}: {$th->getMessage()}");
+                    }
+
+                    try {
+                        $this->createAttributeFromCollection($this->projectDB, $id, 'deploymentInternalId');
+                    } catch (\Throwable $th) {
+                        Console::warning("'deploymentInternalId' from {$id}: {$th->getMessage()}");
+                    }
+
+                    try {
+                        $this->projectDB->renameAttribute($id, 'stderr', 'errors');
+                    } catch (\Throwable $th) {
+                        Console::warning("'errors' from {$id}: {$th->getMessage()}");
+                    }
+
+                    try {
+                        $this->projectDB->renameAttribute($id, 'stdout', 'logs');
+                    } catch (\Throwable $th) {
+                        Console::warning("'logs' from {$id}: {$th->getMessage()}");
+                    }
+
+                    try {
+                        $this->projectDB->renameAttribute($id, 'statusCode', 'responseStatusCode');
+                    } catch (\Throwable $th) {
+                        Console::warning("'responseStatusCode' from {$id}: {$th->getMessage()}");
+                    }
+
+                    $this->projectDB->deleteCachedCollection($id);
+
+                    break;
+                case 'files':
+                    // Recreate indexes so they're the right size
+                    $indexesToDelete = [
+                        '_key_name',
+                        '_key_signature',
+                        '_key_mimeType',
+                    ];
+                    foreach ($indexesToDelete as $index) {
+                        try {
+                            $this->projectDB->deleteIndex($id, $index);
+                        } catch (\Throwable $th) {
+                            Console::warning("'$index' from {$id}: {$th->getMessage()}");
+                        }
+                    }
+
+                    $indexesToCreate = $indexesToDelete;
+                    foreach ($indexesToCreate as $index) {
+                        try {
+                            $this->createIndexFromCollection($this->projectDB, $id, $index);
+                        } catch (\Throwable $th) {
+                            Console::warning("'$index' from {$id}: {$th->getMessage()}");
+                        }
+                    }
+
+                    $this->projectDB->deleteCachedCollection($id);
+
+                    break;
+                case 'functions':
+                    $attributesToCreate = [
+                        'live',
+                        'installationId',
+                        'installationInternalId',
+                        'providerRepositoryId',
+                        'repositoryId',
+                        'repositoryInternalId',
+                        'providerBranch',
+                        'providerRootDirectory',
+                        'providerSilentMode',
+                        'logging',
+                        'deploymentInternalId',
+                        'scheduleInternalId',
+                        'scheduleId',
+                        'version',
+                        'entrypoint',
+                        'commands',
+                    ];
+                    foreach ($attributesToCreate as $attribute) {
+                        try {
+                            $this->createAttributeFromCollection($this->projectDB, $id, $attribute);
+                        } catch (\Throwable $th) {
+                            Console::warning("'$attribute' from {$id}: {$th->getMessage()}");
+                        }
+                    }
+
+                    // Recreate indexes so they're the right size
+                    $indexesToDelete = [
+                        '_key_name',
+                        '_key_runtime',
+                        '_key_deployment',
+                    ];
+                    foreach ($indexesToDelete as $index) {
+                        try {
+                            $this->projectDB->deleteIndex($id, $index);
+                        } catch (\Throwable $th) {
+                            Console::warning("'$index' from {$id}: {$th->getMessage()}");
+                        }
+                    }
+
+                    $indexesToCreate = [
+                        ...$indexesToDelete,
+                        '_key_installationId',
+                        '_key_installationInternalId',
+                        '_key_providerRepositoryId',
+                        '_key_repositoryId',
+                        '_key_repositoryInternalId',
+                    ];
+                    foreach ($indexesToCreate as $index) {
+                        try {
+                            $this->createIndexFromCollection($this->projectDB, $id, $index);
+                        } catch (\Throwable $th) {
+                            Console::warning("'$index' from {$id}: {$th->getMessage()}");
+                        }
+                    }
+
+                    $this->projectDB->deleteCachedCollection($id);
+
+                    break;
+                case 'memberships':
+                    try {
+                        $this->projectDB->updateAttribute($id, 'teamInternalId', required: true);
+                    } catch (\Throwable $th) {
+                        Console::warning("'teamInternalId' from {$id}: {$th->getMessage()}");
+                    }
+
+                    $this->projectDB->deleteCachedCollection($id);
+
+                    // Intentional fall through to update memberships.userInternalId
+                case 'sessions':
+                case 'tokens':
+                    try {
+                        $this->projectDB->updateAttribute($id, 'userInternalId', required: true);
+                    } catch (\Throwable $th) {
+                        Console::warning("'userInternalId' from {$id}: {$th->getMessage()}");
+                    }
+
+                    $this->projectDB->deleteCachedCollection($id);
+
+                    break;
+                case 'domains':
+                case 'keys':
+                case 'platforms':
+                case 'webhooks':
+                    try {
+                        $this->projectDB->updateAttribute($id, 'projectInternalId', required: true);
+                    } catch (\Throwable $th) {
+                        Console::warning("'projectInternalId' from {$id}: {$th->getMessage()}");
+                    }
+
+                    $this->projectDB->deleteCachedCollection($id);
+
+                    break;
+                case 'projects':
+                    $attributesToCreate = [
+                        'database',
+                        'smtp',
+                        'templates',
+                    ];
+                    foreach ($attributesToCreate as $attribute) {
+                        try {
+                            $this->createAttributeFromCollection($this->projectDB, $id, $attribute);
+                        } catch (\Throwable $th) {
+                            Console::warning("'$attribute' from {$id}: {$th->getMessage()}");
+                            Console::warning($th->getTraceAsString());
+                        }
+                    }
+
+                    $this->projectDB->deleteCachedCollection($id);
+
+                    break;
+                case 'stats':
+                    try {
+                        $this->projectDB->updateAttribute($id, 'value', signed: true);
+                    } catch (\Throwable $th) {
+                        Console::warning("'value' from {$id}: {$th->getMessage()}");
+                    }
+
+                    // Holding off on these until a future release
+                    // try {
+                    //     $this->projectDB->deleteAttribute($id, 'type');
+                    //     $this->projectDB->deleteCachedCollection($id);
+                    // } catch (\Throwable $th) {
+                    //     Console::warning("'type' from {$id}: {$th->getMessage()}");
+                    // }
+
+                    // try {
+                    //     $this->projectDB->deleteIndex($id, '_key_metric_period_time');
+                    //     $this->projectDB->deleteCachedCollection($id);
+                    // } catch (\Throwable $th) {
+                    //     Console::warning("'_key_metric_period_time' from {$id}: {$th->getMessage()}");
+                    // }
+
+                    // try {
+                    //     $this->createIndexFromCollection($this->projectDB, $id, '_key_metric_period_time');
+                    //     $this->projectDB->deleteCachedCollection($id);
+                    // } catch (\Throwable $th) {
+                    //     Console::warning("'_key_metric_period_time' from {$id}: {$th->getMessage()}");
+                    // }
+
+                    $this->projectDB->deleteCachedCollection($id);
+
+                    break;
+                case 'users':
+                    try {
+                        $this->createAttributeFromCollection($this->projectDB, $id, 'labels');
+                    } catch (\Throwable $th) {
+                        Console::warning("'labels' from {$id}: {$th->getMessage()}");
+                    }
+
+                    try {
+                        $this->createAttributeFromCollection($this->projectDB, $id, 'accessedAt');
+                    } catch (\Throwable $th) {
+                        Console::warning("'accessedAt' from {$id}: {$th->getMessage()}");
+                    }
+
+                    try {
+                        $this->projectDB->updateAttribute($id, 'search', filters: ['userSearch']);
+                    } catch (\Throwable $th) {
+                        Console::warning("'search' from {$id}: {$th->getMessage()}");
+                    }
+
+                    try {
+                        $this->createIndexFromCollection($this->projectDB, $id, '_key_accessedAt');
+                    } catch (\Throwable $th) {
+                        Console::warning("'_key_accessedAt' from {$id}: {$th->getMessage()}");
+                    }
+
+                    $this->projectDB->deleteCachedCollection($id);
+
+                    break;
+                case 'variables':
+                    try {
+                        $this->projectDB->deleteIndex($id, '_key_function');
+                    } catch (\Throwable $th) {
+                        Console::warning("'_key_function' from {$id}: {$th->getMessage()}");
+                    }
+
+                    try {
+                        $this->projectDB->deleteIndex($id, '_key_uniqueKey');
+                    } catch (\Throwable $th) {
+                        Console::warning("'_key_uniqueKey' from {$id}: {$th->getMessage()}");
+                    }
+
+                    try {
+                        $this->createAttributeFromCollection($this->projectDB, $id, 'resourceType');
+                    } catch (\Throwable $th) {
+                        Console::warning("'resourceType' from {$id}: {$th->getMessage()}");
+                    }
+
+                    try {
+                        $this->projectDB->renameAttribute($id, 'functionInternalId', 'resourceInternalId');
+                    } catch (\Throwable $th) {
+                        Console::warning("'resourceInternalId' from {$id}: {$th->getMessage()}");
+                    }
+
+                    try {
+                        $this->projectDB->renameAttribute($id, 'functionId', 'resourceId');
+                    } catch (\Throwable $th) {
+                        Console::warning("'resourceId' from {$id}: {$th->getMessage()}");
+                    }
+
+                    $indexesToCreate = [
+                        '_key_resourceInternalId',
+                        '_key_resourceId_resourceType',
+                        '_key_resourceType',
+                        '_key_uniqueKey',
+                    ];
+                    foreach ($indexesToCreate as $index) {
+                        try {
+                            $this->createIndexFromCollection($this->projectDB, $id, $index);
+                        } catch (\Throwable $th) {
+                            Console::warning("'$index' from {$id}: {$th->getMessage()}");
+                        }
+                    }
+
+                    $this->projectDB->deleteCachedCollection($id);
+
                     break;
                 default:
                     break;
             }
-            if (!in_array($id, ['files', 'collections'])) {
-                $this->alterPermissionIndex($id);
-                $this->alterUidType($id);
-            }
 
             usleep(50000);
         }
@@ -117,62 +534,145 @@ class V19 extends Migration
     protected function fixDocument(Document $document): Document
     {
         switch ($document->getCollection()) {
+            case 'attributes':
+            case 'indexes':
+                $status = $document->getAttribute('status', '');
+                if ($status === 'failed') {
+                    $document->setAttribute('error', 'Unknown problem');
+                }
+                break;
+            case 'builds':
+                $deploymentId = $document->getAttribute('deploymentId');
+                $deployment = $this->projectDB->getDocument('deployments', $deploymentId);
+                $document->setAttribute('deploymentInternalId', $deployment->getInternalId());
+
+                $stdout = $document->getAttribute('stdout', '');
+                $stderr = $document->getAttribute('stderr', '');
+                $document->setAttribute('logs', $stdout . PHP_EOL . $stderr);
+                break;
+            case 'databases':
+                $document->setAttribute('enabled', true);
+                break;
+            case 'deployments':
+                $resourceId = $document->getAttribute('resourceId');
+                $function = $this->projectDB->getDocument('functions', $resourceId);
+                $document->setAttribute('resourceInternalId', $function->getInternalId());
+
+                $buildId = $document->getAttribute('buildId');
+                if (!empty($buildId)) {
+                    $build = $this->projectDB->getDocument('builds', $buildId);
+                    $document->setAttribute('buildInternalId', $build->getInternalId());
+                }
+
+                $document->setAttribute('type', 'manual');
+                break;
+            case 'executions':
+                $functionId = $document->getAttribute('functionId');
+                $function = $this->projectDB->getDocument('functions', $functionId);
+                $document->setAttribute('functionInternalId', $function->getInternalId());
+
+                $deploymentId = $document->getAttribute('deploymentId');
+                $deployment = $this->projectDB->getDocument('deployments', $deploymentId);
+                $document->setAttribute('deploymentInternalId', $deployment->getInternalId());
+                break;
+            case 'functions':
+                $document->setAttribute('live', true);
+                $document->setAttribute('logging', true);
+                $document->setAttribute('version', 'v2');
+                $deploymentId = $document->getAttribute('deployment');
+
+                if (!empty($deploymentId)) {
+                    $deployment = $this->projectDB->getDocument('deployments', $deploymentId);
+                    $document->setAttribute('deploymentInternalId', $deployment->getInternalId());
+                    $document->setAttribute('entrypoint', $deployment->getAttribute('entrypoint'));
+                }
+
+                $schedule = $this->consoleDB->createDocument('schedules', new Document([
+                    'region' => App::getEnv('_APP_REGION', 'default'), // Todo replace with projects region
+                    'resourceType' => 'function',
+                    'resourceId' => $document->getId(),
+                    'resourceInternalId' => $document->getInternalId(),
+                    'resourceUpdatedAt' => DateTime::now(),
+                    'projectId' => $this->project->getId(),
+                    'schedule'  => $document->getAttribute('schedule'),
+                    'active' => !empty($document->getAttribute('schedule')) && !empty($document->getAttribute('deployment')),
+                ]));
+
+                $document->setAttribute('scheduleId', $schedule->getId());
+                $document->setAttribute('scheduleInternalId', $schedule->getInternalId());
+                break;
             case 'projects':
-                /**
-                 * Bump version number.
-                 */
                 $document->setAttribute('version', '1.4.0');
 
+                $databases = Config::getParam('pools-database', []);
+                $database = $databases[0];
+
+                $document->setAttribute('database', $database);
                 $document->setAttribute('smtp', []);
                 $document->setAttribute('templates', []);
 
+                break;
+            case 'rules':
+                $status = 'created';
+                if ($document->getAttribute('verification', false)) {
+                    $status = 'verified';
+                }
+
+                $ruleDocument = new Document([
+                    'projectId' => $this->project->getId(),
+                    'projectInternalId' => $this->project->getInternalId(),
+                    'domain' => $document->getAttribute('domain'),
+                    'resourceType' => 'api',
+                    'resourceInternalId' => '',
+                    'resourceId' => '',
+                    'status' => $status,
+                    'certificateId' => $document->getAttribute('certificateId'),
+                ]);
+
+                try {
+                    $this->consoleDB->createDocument('rules', $ruleDocument);
+                } catch (\Throwable $th) {
+                    Console::warning("Error migrating domain {$document->getAttribute('domain')}: {$th->getMessage()}");
+                }
+
+                break;
+            default:
                 break;
         }
 
         return $document;
     }
 
-    protected function alterPermissionIndex($collectionName): void
+    private function cleanCollections(): void
     {
         try {
-            $table = "`{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_{$collectionName}_perms`";
-            $this->pdo->prepare("
-                ALTER TABLE {$table}
-                DROP INDEX `_permission`, 
-                ADD INDEX `_permission` (`_permission`, `_type`, `_document`);
-            ")->execute();
+            $this->projectDB->deleteAttribute('projects', 'domains');
         } catch (\Throwable $th) {
-            Console::warning($th->getMessage());
+            Console::warning("'domains' from projects: {$th->getMessage()}");
         }
-    }
 
-    protected function alterUidType($collectionName): void
-    {
+        $this->projectDB->deleteCachedCollection('projects');
+
         try {
-            $table = "`{$this->projectDB->getDefaultDatabase()}`.`_{$this->project->getInternalId()}_{$collectionName}`";
-            $this->pdo->prepare("
-            ALTER TABLE {$table}
-            CHANGE COLUMN `_uid` `_uid` VARCHAR(255) NOT NULL ;
-            ")->execute();
+            $this->projectDB->deleteAttribute('functions', 'schedule');
         } catch (\Throwable $th) {
-            Console::warning($th->getMessage());
+            Console::warning("'schedule' from functions: {$th->getMessage()}");
         }
-    }
 
-    /**
-     * Migrating all Bucket tables.
-     *
-     * @return void
-     * @throws \Exception
-     * @throws \PDOException
-     */
-    protected function migrateBuckets(): void
-    {
-        foreach ($this->documentsIterator('buckets') as $bucket) {
-            $id = "bucket_{$bucket->getInternalId()}";
-            Console::log("Migrating Bucket {$id} {$bucket->getId()} ({$bucket->getAttribute('name')})");
-            $this->alterPermissionIndex($id);
-            $this->alterUidType($id);
+        $this->projectDB->deleteCachedCollection('functions');
+
+        try {
+            $this->projectDB->deleteAttribute('builds', 'stderr');
+        } catch (\Throwable $th) {
+            Console::warning("'stderr' from builds: {$th->getMessage()}");
         }
+
+        try {
+            $this->projectDB->deleteAttribute('builds', 'stdout');
+        } catch (\Throwable $th) {
+            Console::warning("'stdout' from builds: {$th->getMessage()}");
+        }
+
+        $this->projectDB->deleteCachedCollection('builds');
     }
 }

src/Appwrite/Platform/Services/Tasks.php

@@ -30,6 +30,7 @@ class Tasks extends Service
         $this->type = self::TYPE_CLI;
         $this
             ->addAction(Version::getName(), new Version())
+            ->addAction(Usage::getName(), new Usage())
             ->addAction(Vars::getName(), new Vars())
             ->addAction(SSL::getName(), new SSL())
             ->addAction(Hamster::getName(), new Hamster())

src/Appwrite/Platform/Tasks/Install.php

@@ -6,7 +6,9 @@ use Appwrite\Auth\Auth;
 use Appwrite\Docker\Compose;
 use Appwrite\Docker\Env;
 use Appwrite\Utopia\View;
-use Utopia\Analytics\GoogleAnalytics;
+use Utopia\Analytics\Adapter;
+use Utopia\Analytics\Adapter\GoogleAnalytics;
+use Utopia\Analytics\Event;
 use Utopia\CLI\Console;
 use Utopia\Config\Config;
 use Utopia\Validator\Text;
@@ -33,22 +35,6 @@ class Install extends Action
 
     public function action(string $httpPort, string $httpsPort, string $organization, string $image, string $interactive): void
     {
-        /**
-         * 1. Start - DONE
-         * 2. Check for older setup and get older version - DONE
-         *  2.1 If older version is equal or bigger(?) than current version, **stop setup**
-         *  2.2. Get ENV vars - DONE
-         *   2.2.1 Fetch from older docker-compose.yml file
-         *   2.2.2 Fetch from older .env file (manually parse)
-         *  2.3 Use old ENV vars as default values
-         *  2.4 Ask for all required vars not given as CLI args and if in interactive mode
-         *      Otherwise, just use default vars. - DONE
-         * 3. Ask user to backup important volumes, env vars, and SQL tables
-         *      In th future we can try and automate this for smaller/medium size setups
-         * 4. Drop new docker-compose.yml setup (located inside the container, no network dependencies with appwrite.io) - DONE
-         * 5. Run docker compose up -d - DONE
-         * 6. Run data migration
-         */
         $config = Config::getParam('variables');
         $path = '/usr/src/code/appwrite';
         $defaultHTTPPort = '80';
@@ -70,7 +56,7 @@ class Install extends Action
         Console::success('Starting Appwrite installation...');
 
         // Create directory with write permissions
-        if (null !== $path && !\file_exists(\dirname($path))) {
+        if (!\file_exists(\dirname($path))) {
             if (!@\mkdir(\dirname($path), 0755, true)) {
                 Console::error('Can\'t create directory ' . \dirname($path));
                 Console::exit(1);
@@ -198,31 +184,28 @@ class Install extends Action
             }
         }
 
-        $templateForCompose = new View(__DIR__ . '/../views/install/compose.phtml');
-        $templateForEnv = new View(__DIR__ . '/../views/install/env.phtml');
+        $templateForCompose = new View(__DIR__ . '/../../../../app/views/install/compose.phtml');
+        $templateForEnv = new View(__DIR__ . '/../../../../app/views/install/env.phtml');
 
         $templateForCompose
             ->setParam('httpPort', $httpPort)
             ->setParam('httpsPort', $httpsPort)
             ->setParam('version', APP_VERSION_STABLE)
             ->setParam('organization', $organization)
-            ->setParam('image', $image)
-        ;
+            ->setParam('image', $image);
 
-        $templateForEnv
-            ->setParam('vars', $input)
-        ;
+        $templateForEnv->setParam('vars', $input);
 
         if (!file_put_contents($path . '/docker-compose.yml', $templateForCompose->render(false))) {
             $message = 'Failed to save Docker Compose file';
-            $analytics->createEvent('install/server', 'install', APP_VERSION_STABLE . ' - ' . $message);
+            $this->sendEvent($analytics, $message);
             Console::error($message);
             Console::exit(1);
         }
 
         if (!file_put_contents($path . '/.env', $templateForEnv->render(false))) {
             $message = 'Failed to save environment variables file';
-            $analytics->createEvent('install/server', 'install', APP_VERSION_STABLE . ' - ' . $message);
+            $this->sendEvent($analytics, $message);
             Console::error($message);
             Console::exit(1);
         }
@@ -243,14 +226,29 @@ class Install extends Action
 
         if ($exit !== 0) {
             $message = 'Failed to install Appwrite dockers';
-            $analytics->createEvent('install/server', 'install', APP_VERSION_STABLE . ' - ' . $message);
+            $this->sendEvent($analytics, $message);
             Console::error($message);
             Console::error($stderr);
             Console::exit($exit);
         } else {
             $message = 'Appwrite installed successfully';
-            $analytics->createEvent('install/server', 'install', APP_VERSION_STABLE . ' - ' . $message);
+            $this->sendEvent($analytics, $message);
             Console::success($message);
         }
     }
+
+    private function sendEvent(Adapter $analytics, string $message): void
+    {
+        $event = new Event();
+        $event->setName(APP_VERSION_STABLE);
+        $event->setValue($message);
+        $event->setUrl('http://localhost/');
+        $event->setProps([
+            'category' => 'install/server',
+            'action' => 'install',
+        ]);
+        $event->setType('install/server');
+
+        $analytics->createEvent($event);
+    }
 }

src/Appwrite/Platform/Tasks/Maintenance.php

@@ -2,7 +2,6 @@
 
 namespace Appwrite\Platform\Tasks;
 
-use Appwrite\Auth\Auth;
 use Appwrite\Event\Certificate;
 use Appwrite\Event\Delete;
 use Utopia\App;
@@ -109,7 +108,6 @@ class Maintenance extends Action
 
         function notifyDeleteCache($interval)
         {
-
             (new Delete())
                 ->setType(DELETE_TYPE_CACHE_BY_TIMESTAMP)
                 ->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval))
@@ -118,7 +116,6 @@ class Maintenance extends Action
 
         function notifyDeleteSchedules($interval)
         {
-
             (new Delete())
                 ->setType(DELETE_TYPE_SCHEDULES)
                 ->setDatetime(DateTime::addSeconds(new \DateTime(), -1 * $interval))

src/Appwrite/Platform/Tasks/Migrate.php

@@ -7,10 +7,10 @@ use Utopia\CLI\Console;
 use Appwrite\Migration\Migration;
 use Utopia\App;
 use Utopia\Cache\Cache;
-use Utopia\Cache\Adapter\Redis as RedisCache;
+use Utopia\Database\Database;
 use Utopia\Database\Document;
+use Utopia\Database\Query;
 use Utopia\Database\Validator\Authorization;
-use Utopia\Registry\Registry;
 use Utopia\Validator\Text;
 
 class Migrate extends Action
@@ -26,20 +26,22 @@ class Migrate extends Action
             ->desc('Migrate Appwrite to new version')
             /** @TODO APP_VERSION_STABLE needs to be defined */
             ->param('version', APP_VERSION_STABLE, new Text(8), 'Version to migrate to.', true)
-            ->inject('register')
-            ->callback(fn ($version, $register) => $this->action($version, $register));
+            ->inject('cache')
+            ->inject('dbForConsole')
+            ->inject('getProjectDB')
+            ->callback(fn ($version, $cache, $dbForConsole, $getProjectDB) => $this->action($version, $cache, $dbForConsole, $getProjectDB));
     }
 
-    private function clearProjectsCache(Redis $redis, Document $project)
+    private function clearProjectsCache(Cache $cache, Document $project)
     {
         try {
-            $redis->del($redis->keys("cache-_{$project->getInternalId()}:*"));
+            $cache->purge("cache-_{$project->getInternalId()}:*");
         } catch (\Throwable $th) {
             Console::error('Failed to clear project ("' . $project->getId() . '") cache with error: ' . $th->getMessage());
         }
     }
 
-    public function action(string $version, Registry $register)
+    public function action(string $version, Cache $cache, Database $dbForConsole, callable $getProjectDB)
     {
         Authorization::disable();
         if (!array_key_exists($version, Migration::$versions)) {
@@ -52,14 +54,6 @@ class Migrate extends Action
 
         Console::success('Starting Data Migration to version ' . $version);
 
-        $dbPool = $register->get('dbPool', true);
-        $redis = $register->get('cache', true);
-
-        $cache = new Cache(new RedisCache($redis));
-
-        $dbForConsole = $dbPool->getDB('console', $cache);
-        $dbForConsole->setNamespace('_project_console');
-
         $console = $app->getResource('console');
 
         $limit = 30;
@@ -79,6 +73,7 @@ class Migrate extends Action
         }
 
         $class = 'Appwrite\\Migration\\Version\\' . Migration::$versions[$version];
+        /** @var Migration $migration */
         $migration = new $class();
 
         while (!empty($projects)) {
@@ -90,11 +85,11 @@ class Migrate extends Action
                     continue;
                 }
 
-                $this->clearProjectsCache($redis, $project);
+                $this->clearProjectsCache($cache, $project);
 
                 try {
                     // TODO: Iterate through all project DBs
-                    $projectDB = $dbPool->getDB($project->getId(), $cache);
+                    $projectDB = $getProjectDB($project);
                     $migration
                         ->setProject($project, $projectDB, $dbForConsole)
                         ->execute();
@@ -103,11 +98,11 @@ class Migrate extends Action
                     throw $th;
                 }
 
-                $this->clearProjectsCache($redis, $project);
+                $this->clearProjectsCache($cache, $project);
             }
 
             $sum = \count($projects);
-            $projects = $dbForConsole->find('projects', limit: $limit, offset: $offset);
+            $projects = $dbForConsole->find('projects', [Query::limit($limit), Query::offset($offset)]);
 
             $offset = $offset + $limit;
             $count = $count + $sum;
@@ -115,7 +110,6 @@ class Migrate extends Action
             Console::log('Migrated ' . $count . '/' . $totalProjects . ' projects...');
         }
 
-        Swoole\Event::wait(); // Wait for Coroutines to finish
         Console::success('Data Migration Completed');
     }
 }

src/Appwrite/Platform/Tasks/SDKs.php

@@ -51,7 +51,7 @@ class SDKs extends Action
         $production = ($git) ? (Console::confirm('Type "Appwrite" to push code to production git repos') == 'Appwrite') : false;
         $message = ($git) ? Console::confirm('Please enter your commit message:') : '';
 
-        if (!in_array($version, ['0.6.x', '0.7.x', '0.8.x', '0.9.x', '0.10.x', '0.11.x', '0.12.x', '0.13.x', '0.14.x', '0.15.x', '1.0.x', '1.1.x', '1.2.x', '1.3.x', 'latest'])) {
+        if (!in_array($version, ['0.6.x', '0.7.x', '0.8.x', '0.9.x', '0.10.x', '0.11.x', '0.12.x', '0.13.x', '0.14.x', '0.15.x', '1.0.x', '1.1.x', '1.2.x', '1.3.x', '1.4.x', 'latest'])) {
             throw new Exception('Unknown version given');
         }
 
@@ -232,7 +232,7 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
                     ->setTwitter(APP_SOCIAL_TWITTER_HANDLE)
                     ->setDiscord(APP_SOCIAL_DISCORD_CHANNEL, APP_SOCIAL_DISCORD)
                     ->setDefaultHeaders([
-                        'X-Appwrite-Response-Format' => '1.0.0',
+                        'X-Appwrite-Response-Format' => '1.4.0',
                     ]);
 
                 try {

src/Appwrite/Platform/Tasks/Specs.php

@@ -40,8 +40,6 @@ class Specs extends Action
 
     public function action(string $version, string $mode, Registry $register): void
     {
-        //$db = $register->get('db');
-        //$redis = $register->get('cache');
         $appRoutes = App::getRoutes();
         $response = new Response(new HttpResponse());
         $mocks = ($mode === 'mocks');

src/Appwrite/Platform/Tasks/Usage.php

@@ -0,0 +1,60 @@
+<?php
+
+namespace Appwrite\Platform\Tasks;
+
+use Appwrite\Usage\Calculators\TimeSeries;
+use InfluxDB\Database as InfluxDatabase;
+use Utopia\App;
+use Utopia\CLI\Console;
+use Utopia\Database\Database as UtopiaDatabase;
+use Throwable;
+use Utopia\Platform\Action;
+use Utopia\Registry\Registry;
+
+class Usage extends Action
+{
+    public static function getName(): string
+    {
+        return 'usage';
+    }
+
+    public function __construct()
+    {
+        $this
+            ->desc('Schedules syncing data from influxdb to Appwrite console db')
+            ->inject('dbForConsole')
+            ->inject('influxdb')
+            ->inject('register')
+            ->inject('getProjectDB')
+            ->inject('logError')
+            ->callback(fn ($dbForConsole, $influxDB, $register, $getProjectDB, $logError) => $this->action($dbForConsole, $influxDB, $register, $getProjectDB, $logError));
+    }
+
+    protected function aggregateTimeseries(UtopiaDatabase $database, InfluxDatabase $influxDB, callable $logError): void
+    {
+    }
+
+    public function action(UtopiaDatabase $dbForConsole, InfluxDatabase $influxDB, Registry $register, callable $getProjectDB, callable $logError)
+    {
+        Console::title('Usage Aggregation V1');
+        Console::success(APP_NAME . ' usage aggregation process v1 has started');
+
+        $errorLogger = fn(Throwable $error, string $action = 'syncUsageStats') => $logError($error, "usage", $action);
+
+        $interval = (int) App::getEnv('_APP_USAGE_AGGREGATION_INTERVAL', '30'); // 30 seconds (by default)
+        $region = App::getEnv('region', 'default');
+        $usage = new TimeSeries($region, $dbForConsole, $influxDB, $getProjectDB, $register, $errorLogger);
+
+        Console::loop(function () use ($interval, $usage) {
+            $now = date('d-m-Y H:i:s', time());
+            Console::info("[{$now}] Aggregating Timeseries Usage data every {$interval} seconds");
+            $loopStart = microtime(true);
+
+            $usage->collect();
+
+            $loopTook = microtime(true) - $loopStart;
+            $now = date('d-m-Y H:i:s', time());
+            Console::info("[{$now}] Aggregation took {$loopTook} seconds");
+        }, $interval);
+    }
+}

src/Appwrite/Resque/Worker.php

@@ -223,7 +223,7 @@ abstract class Worker
 
         if (isset(self::$databases[$databaseName])) {
             $database = self::$databases[$databaseName];
-            $database->setNamespace('console');
+            $database->setNamespace('_console');
             return $database;
         }
 
@@ -237,7 +237,7 @@ abstract class Worker
 
         self::$databases[$databaseName] = $database;
 
-        $database->setNamespace('console');
+        $database->setNamespace('_console');
 
         return $database;
     }
@@ -328,38 +328,82 @@ abstract class Worker
     public function getDevice(string $root): Device
     {
         $connection = App::getEnv('_APP_CONNECTIONS_STORAGE', '');
-        $acl = 'private';
-        $device = Storage::DEVICE_LOCAL;
-        $accessKey = '';
-        $accessSecret = '';
-        $bucket = '';
-        $region = '';
 
-        try {
-            $dsn = new DSN($connection);
-            $device = $dsn->getScheme();
-            $accessKey = $dsn->getUser();
-            $accessSecret = $dsn->getPassword();
-            $bucket = $dsn->getPath();
-            $region = $dsn->getParam('region');
-        } catch (\Exception $e) {
-            Console::error($e->getMessage() . 'Invalid DSN. Defaulting to Local device.');
-        }
+        if (!empty($connection)) {
+            $acl = 'private';
+            $device = Storage::DEVICE_LOCAL;
+            $accessKey = '';
+            $accessSecret = '';
+            $bucket = '';
+            $region = '';
 
-        switch ($device) {
-            case Storage::DEVICE_S3:
-                return new S3($root, $accessKey, $accessSecret, $bucket, $region, $acl);
-            case STORAGE::DEVICE_DO_SPACES:
-                return new DOSpaces($root, $accessKey, $accessSecret, $bucket, $region, $acl);
-            case Storage::DEVICE_BACKBLAZE:
-                return new Backblaze($root, $accessKey, $accessSecret, $bucket, $region, $acl);
-            case Storage::DEVICE_LINODE:
-                return new Linode($root, $accessKey, $accessSecret, $bucket, $region, $acl);
-            case Storage::DEVICE_WASABI:
-                return new Wasabi($root, $accessKey, $accessSecret, $bucket, $region, $acl);
-            case Storage::DEVICE_LOCAL:
-            default:
-                return new Local($root);
+            try {
+                $dsn = new DSN($connection);
+                $device = $dsn->getScheme();
+                $accessKey = $dsn->getUser() ?? '';
+                $accessSecret = $dsn->getPassword() ?? '';
+                $bucket = $dsn->getPath() ?? '';
+                $region = $dsn->getParam('region');
+            } catch (\Exception $e) {
+                Console::warning($e->getMessage() . 'Invalid DSN. Defaulting to Local device.');
+            }
+
+            switch ($device) {
+                case Storage::DEVICE_S3:
+                    return new S3($root, $accessKey, $accessSecret, $bucket, $region, $acl);
+                case STORAGE::DEVICE_DO_SPACES:
+                    return new DOSpaces($root, $accessKey, $accessSecret, $bucket, $region, $acl);
+                case Storage::DEVICE_BACKBLAZE:
+                    return new Backblaze($root, $accessKey, $accessSecret, $bucket, $region, $acl);
+                case Storage::DEVICE_LINODE:
+                    return new Linode($root, $accessKey, $accessSecret, $bucket, $region, $acl);
+                case Storage::DEVICE_WASABI:
+                    return new Wasabi($root, $accessKey, $accessSecret, $bucket, $region, $acl);
+                case Storage::DEVICE_LOCAL:
+                default:
+                    return new Local($root);
+            }
+        } else {
+            switch (strtolower(App::getEnv('_APP_STORAGE_DEVICE', Storage::DEVICE_LOCAL) ?? '')) {
+                case Storage::DEVICE_LOCAL:
+                default:
+                    return new Local($root);
+                case Storage::DEVICE_S3:
+                    $s3AccessKey = App::getEnv('_APP_STORAGE_S3_ACCESS_KEY', '');
+                    $s3SecretKey = App::getEnv('_APP_STORAGE_S3_SECRET', '');
+                    $s3Region = App::getEnv('_APP_STORAGE_S3_REGION', '');
+                    $s3Bucket = App::getEnv('_APP_STORAGE_S3_BUCKET', '');
+                    $s3Acl = 'private';
+                    return new S3($root, $s3AccessKey, $s3SecretKey, $s3Bucket, $s3Region, $s3Acl);
+                case Storage::DEVICE_DO_SPACES:
+                    $doSpacesAccessKey = App::getEnv('_APP_STORAGE_DO_SPACES_ACCESS_KEY', '');
+                    $doSpacesSecretKey = App::getEnv('_APP_STORAGE_DO_SPACES_SECRET', '');
+                    $doSpacesRegion = App::getEnv('_APP_STORAGE_DO_SPACES_REGION', '');
+                    $doSpacesBucket = App::getEnv('_APP_STORAGE_DO_SPACES_BUCKET', '');
+                    $doSpacesAcl = 'private';
+                    return new DOSpaces($root, $doSpacesAccessKey, $doSpacesSecretKey, $doSpacesBucket, $doSpacesRegion, $doSpacesAcl);
+                case Storage::DEVICE_BACKBLAZE:
+                    $backblazeAccessKey = App::getEnv('_APP_STORAGE_BACKBLAZE_ACCESS_KEY', '');
+                    $backblazeSecretKey = App::getEnv('_APP_STORAGE_BACKBLAZE_SECRET', '');
+                    $backblazeRegion = App::getEnv('_APP_STORAGE_BACKBLAZE_REGION', '');
+                    $backblazeBucket = App::getEnv('_APP_STORAGE_BACKBLAZE_BUCKET', '');
+                    $backblazeAcl = 'private';
+                    return new Backblaze($root, $backblazeAccessKey, $backblazeSecretKey, $backblazeBucket, $backblazeRegion, $backblazeAcl);
+                case Storage::DEVICE_LINODE:
+                    $linodeAccessKey = App::getEnv('_APP_STORAGE_LINODE_ACCESS_KEY', '');
+                    $linodeSecretKey = App::getEnv('_APP_STORAGE_LINODE_SECRET', '');
+                    $linodeRegion = App::getEnv('_APP_STORAGE_LINODE_REGION', '');
+                    $linodeBucket = App::getEnv('_APP_STORAGE_LINODE_BUCKET', '');
+                    $linodeAcl = 'private';
+                    return new Linode($root, $linodeAccessKey, $linodeSecretKey, $linodeBucket, $linodeRegion, $linodeAcl);
+                case Storage::DEVICE_WASABI:
+                    $wasabiAccessKey = App::getEnv('_APP_STORAGE_WASABI_ACCESS_KEY', '');
+                    $wasabiSecretKey = App::getEnv('_APP_STORAGE_WASABI_SECRET', '');
+                    $wasabiRegion = App::getEnv('_APP_STORAGE_WASABI_REGION', '');
+                    $wasabiBucket = App::getEnv('_APP_STORAGE_WASABI_BUCKET', '');
+                    $wasabiAcl = 'private';
+                    return new Wasabi($root, $wasabiAccessKey, $wasabiSecretKey, $wasabiBucket, $wasabiRegion, $wasabiAcl);
+            }
         }
     }
 }