benchmarks

bin/benchmark

@@ -0,0 +1,3 @@
+#!/usr/bin/env sh
+
+/usr/src/code/vendor/bin/phpbench run --config /usr/src/code/phpbench.json --report appwrite $@

phpbench.json

@@ -0,0 +1,17 @@
+{
+  "$schema": "./vendor/phpbench/phpbench/phpbench.schema.json",
+  "runner.path": "tests/benchmarks",
+  "runner.file_pattern": "*Bench.php",
+  "runner.bootstrap": "app/init.php",
+  "runner.revs": 1000,
+  "runner.iterations": 3,
+  "runner.retry_threshold": 5,
+  "runner.warmup": 1,
+  "report.generators": {
+    "appwrite": {
+      "extends": "aggregate",
+      "cols": ["benchmark", "subject", "set" ,"revs", "its", "worst", "best", "mean", "mode", "rstdev"],
+      "break": ["benchmark"]
+    }
+  }
+}

tests/benchmarks/Scopes/Scope.php

@@ -0,0 +1,14 @@
+<?php
+
+namespace Tests\Benchmarks\Scopes;
+
+use PhpBench\Attributes\AfterMethods;
+use PhpBench\Attributes\BeforeMethods;
+use Tests\E2E\Scopes\Scope as E2EScope;
+
+#[BeforeMethods(['setUp'])]
+#[AfterMethods(['tearDown'])]
+abstract class Scope extends E2EScope
+{
+    protected $endpoint = 'http://localhost/v1';
+}

tests/benchmarks/Services/Databases/Base.php

@@ -0,0 +1,151 @@
+<?php
+
+namespace Tests\Benchmarks\Services\Databases;
+
+use PhpBench\Attributes\BeforeMethods;
+use PhpBench\Attributes\ParamProviders;
+use Tests\Benchmarks\Scopes\Scope;
+use Tests\E2E\Client;
+use Tests\E2E\Scopes\ProjectCustom;
+use Utopia\Database\ID;
+use Utopia\Database\Permission;
+use Utopia\Database\Role;
+
+abstract class Base extends Scope
+{
+    use ProjectCustom;
+
+    protected static string $databaseId;
+    protected static string $collectionId;
+    protected static string $documentId;
+
+    #[BeforeMethods(['createDatabase', 'createCollection'])]
+    public function benchDocumentCreate()
+    {
+        $this->client->call(Client::METHOD_POST, '/databases/' . static::$databaseId . '/collections/' . static::$collectionId . '/documents', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), [
+            'documentId' => ID::unique(),
+            'data' => [
+                'title' => 'The Matrix',
+            ],
+            'permissions' => [
+                Permission::read(Role::user($this->getUser()['$id'])),
+                Permission::write(Role::user($this->getUser()['$id'])),
+            ],
+        ]);
+    }
+
+    #[ParamProviders(['provideCounts'])]
+    #[BeforeMethods(['createDatabase', 'createCollection', 'createDocuments'])]
+    public function benchDocumentReadList(array $params)
+    {
+        $this->client->call(Client::METHOD_GET, '/databases/' . static::$databaseId . '/collections/' . static::$collectionId . '/documents', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), [
+            'queries' => ['limit(' . $params['documents'] . ')'],
+        ]);
+    }
+
+    #[BeforeMethods(['createDatabase', 'createCollection', 'createDocuments'])]
+    public function benchDocumentRead()
+    {
+        $this->client->call(Client::METHOD_GET, '/databases/' . static::$databaseId . '/collections/' . static::$collectionId . '/documents/' . static::$documentId, array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()));
+    }
+
+    #[BeforeMethods(['createDatabase', 'createCollection', 'createDocuments'])]
+    public function benchDocumentUpdate()
+    {
+        $this->client->call(Client::METHOD_PATCH, '/databases/' . static::$databaseId . '/collections/' . static::$collectionId . '/documents/' . static::$documentId, array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), [
+            'data' => [
+                'title' => 'The Matrix Reloaded',
+            ],
+        ]);
+    }
+
+    public function provideCounts(): array
+    {
+        return [
+            '1 Document' => ['documents' => 1],
+            '10 Documents' => ['documents' => 10],
+            '100 Documents' => ['documents' => 100],
+        ];
+    }
+
+    public function createDatabase(array $params = [])
+    {
+        $database = $this->client->call(Client::METHOD_POST, '/databases', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+            'x-appwrite-key' => $this->getProject()['apiKey']
+        ], [
+            'databaseId' => ID::unique(),
+            'name' => 'Test Database'
+        ]);
+        static::$databaseId = $database['body']['$id'];
+    }
+
+    public function createCollection(array $params = [])
+    {
+        $collection = $this->client->call(Client::METHOD_POST, '/databases/' . static::$databaseId . '/collections', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+            'x-appwrite-key' => $this->getProject()['apiKey']
+        ], [
+            'collectionId' => ID::unique(),
+            'name' => 'Movies',
+            'documentSecurity' => true,
+            'permissions' => [
+                Permission::read(Role::user($this->getUser()['$id'])),
+                Permission::write(Role::user($this->getUser()['$id'])),
+            ],
+        ]);
+        static::$collectionId = $collection['body']['$id'];
+
+        // Create attribute
+        $this->client->call(Client::METHOD_POST, '/databases/' . static::$databaseId . '/collections/' . static::$collectionId . '/attributes/string', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+            'x-appwrite-key' => $this->getProject()['apiKey']
+        ], [
+            'key' => 'title',
+            'size' => 256,
+            'required' => true,
+        ]);
+
+        // Wait for attribute to be ready
+        sleep(2);
+    }
+
+    public function createDocuments(array $params = [])
+    {
+        $count = $params['documents'] ?? 1;
+
+        for ($i = 0; $i < $count; $i++) {
+            $response = $this->client->call(Client::METHOD_POST, '/databases/' . static::$databaseId . '/collections/' . static::$collectionId . '/documents', [
+                'content-type' => 'application/json',
+                'x-appwrite-project' => $this->getProject()['$id'],
+                'x-appwrite-key' => $this->getProject()['apiKey']
+            ], [
+                'documentId' => ID::unique(),
+                'data' => [
+                    'title' => 'Captain America' . $i,
+                ],
+                'permissions' => [
+                    Permission::read(Role::user($this->getUser()['$id'])),
+                    Permission::write(Role::user($this->getUser()['$id'])),
+                ]
+            ]);
+
+            static::$documentId = $response['body']['$id'];
+        }
+    }
+}

tests/benchmarks/Services/Databases/DatabasesCustomClientBench.php

@@ -0,0 +1,10 @@
+<?php
+
+namespace Tests\Benchmarks\Services\Databases;
+
+use Tests\E2E\Scopes\SideClient;
+
+class DatabasesCustomClientBench extends Base
+{
+    use SideClient;
+}

tests/benchmarks/Services/Databases/DatabasesCustomServerBench.php

@@ -0,0 +1,10 @@
+<?php
+
+namespace Tests\Benchmarks\Services\Databases;
+
+use Tests\E2E\Scopes\SideServer;
+
+class DatabasesCustomServerBench extends Base
+{
+    use SideServer;
+}

tests/benchmarks/Services/Functions/Base.php

@@ -0,0 +1,113 @@
+<?php
+
+namespace Tests\Benchmarks\Services\Functions;
+
+use CURLFile;
+use PhpBench\Attributes\BeforeMethods;
+use Tests\Benchmarks\Scopes\Scope;
+use Tests\E2E\Client;
+use Tests\E2E\Scopes\ProjectCustom;
+use Utopia\CLI\Console;
+use Utopia\Database\ID;
+use Utopia\Database\Role;
+
+abstract class Base extends Scope
+{
+    use ProjectCustom;
+
+    protected static string $functionId;
+    protected static string $deploymentId;
+    protected static string $executionId;
+
+    #[BeforeMethods(['createFunction', 'prepareDeployment', 'createDeployment', 'patchDeployment'])]
+    public function benchExecutionCreate()
+    {
+        $this->client->call(Client::METHOD_POST, '/functions/' . static::$functionId . '/executions', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()));
+    }
+
+    public function createFunction()
+    {
+        $response = $this->client->call(Client::METHOD_POST, '/functions', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+            'x-appwrite-key' => $this->getProject()['apiKey']
+        ], [
+            'functionId' => ID::unique(),
+            'name' => 'Test',
+            'runtime' => 'php-8.0',
+            'timeout' => 10,
+            'execute' => [Role::users()->toString()]
+        ]);
+        static::$functionId = $response['body']['$id'];
+    }
+
+    public function prepareDeployment()
+    {
+        $stdout = '';
+        $stderr = '';
+
+        Console::execute(
+            'cd ' . realpath(__DIR__ . "/../../resources/functions/php") . " && \
+            tar --exclude code.tar.gz -czf code.tar.gz .",
+            '',
+            $stdout,
+            $stderr
+        );
+    }
+
+    public function createDeployment()
+    {
+        $code = realpath(__DIR__ . '/../../resources/functions/php/code.tar.gz');
+
+        $response = $this->client->call(Client::METHOD_POST, '/functions/' . static::$functionId . '/deployments', [
+            'content-type' => 'multipart/form-data',
+            'x-appwrite-project' => $this->getProject()['$id'],
+            'x-appwrite-key' => $this->getProject()['apiKey']
+        ], [
+            'entrypoint' => 'index.php',
+            'code' => new CURLFile(
+                $code,
+                'application/x-gzip',
+                \basename($code)
+            ),
+        ]);
+
+        static::$deploymentId = $response['body']['$id'];
+
+        while (true) {
+            $response = $this->client->call(Client::METHOD_GET, '/functions/' . static::$functionId . '/deployments/' . static::$deploymentId, [
+                'content-type' => 'application/json',
+                'x-appwrite-project' => $this->getProject()['$id'],
+                'x-appwrite-key' => $this->getProject()['apiKey']
+            ]);
+
+            $status = $response['body']['status'] ?? '';
+
+            switch ($status) {
+                case '':
+                case 'processing':
+                case 'building':
+                    usleep(200);
+                    break;
+                case 'ready':
+                    break 2;
+                case 'failed':
+                    throw new \Exception('Failed to build function');
+            }
+        }
+
+        sleep(1);
+    }
+
+    public function patchDeployment()
+    {
+        $this->client->call(Client::METHOD_PATCH, '/functions/' . static::$functionId . '/deployments/' . static::$deploymentId, [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+            'x-appwrite-key' => $this->getProject()['apiKey'],
+        ], []);
+    }
+}

tests/benchmarks/Services/Functions/FunctionsCustomClientBench.php

@@ -0,0 +1,10 @@
+<?php
+
+namespace Tests\Benchmarks\Services\Functions;
+
+use Tests\E2E\Scopes\SideClient;
+
+class FunctionsCustomClientBench extends Base
+{
+    use SideClient;
+}

tests/benchmarks/Services/Functions/FunctionsCustomServerBench.php

@@ -0,0 +1,21 @@
+<?php
+
+namespace Tests\Benchmarks\Services\Functions;
+
+use PhpBench\Attributes\BeforeMethods;
+use PhpBench\Attributes\Iterations;
+use PhpBench\Attributes\Revs;
+use Tests\E2E\Scopes\SideServer;
+
+class FunctionsCustomServerBench extends Base
+{
+    use SideServer;
+
+    #[Revs(1)]
+    #[Iterations(1)]
+    #[BeforeMethods(['createFunction', 'prepareDeployment'])]
+    public function benchDeploymentCreate()
+    {
+        $this->createDeployment();
+    }
+}

tests/benchmarks/Services/Storage/Base.php

@@ -0,0 +1,116 @@
+<?php
+
+namespace Tests\Benchmarks\Services\Storage;
+
+use CURLFile;
+use PhpBench\Attributes\BeforeMethods;
+use PhpBench\Attributes\ParamProviders;
+use Tests\Benchmarks\Scopes\Scope;
+use Tests\E2E\Client;
+use Tests\E2E\Scopes\ProjectCustom;
+use Utopia\Database\ID;
+use Utopia\Database\Permission;
+use Utopia\Database\Role;
+
+abstract class Base extends Scope
+{
+    use ProjectCustom;
+
+    protected static string $bucketId;
+    protected static string $fileId;
+
+    #[BeforeMethods(['createBucket'])]
+    public function benchFileCreate()
+    {
+        $this->client->call(Client::METHOD_POST, '/storage/buckets/' . static::$bucketId . '/files', array_merge([
+            'content-type' => 'multipart/form-data',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), [
+            'fileId' => ID::unique(),
+            'permissions' => [
+                Permission::read(Role::user($this->getUser()['$id'])),
+                Permission::write(Role::user($this->getUser()['$id'])),
+            ],
+            'file' => new CURLFile(realpath(__DIR__ . '/../../resources/logo.png'), 'image/png', 'logo.png'),
+        ]);
+    }
+
+    #[ParamProviders(['provideCounts'])]
+    #[BeforeMethods(['createBucket', 'createFiles'])]
+    public function benchFileReadList(array $params)
+    {
+        $this->client->call(Client::METHOD_GET, '/storage/buckets/' . static::$bucketId . '/files', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), [
+            'queries' => ['limit(' . $params['files'] . ')'],
+        ]);
+    }
+
+    #[BeforeMethods(['createBucket', 'createFiles'])]
+    public function benchFileRead()
+    {
+        $this->client->call(Client::METHOD_GET, '/storage/buckets/' . static::$bucketId . '/files/' . static::$fileId, array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()));
+    }
+
+    #[BeforeMethods(['createBucket', 'createFiles'])]
+    public function benchFileUpdate()
+    {
+        $this->client->call(Client::METHOD_PUT, '/storage/buckets/' . static::$bucketId . '/files/' . static::$fileId, array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), [
+            'name' => 'Updated name',
+            'permissions' => [],
+        ]);
+    }
+
+    public function provideCounts(): array
+    {
+        return [
+            '10 Files' => ['files' => 10],
+            '100 Files' => ['files' => 100],
+        ];
+    }
+
+    public function createBucket(array $params = [])
+    {
+        // Create bucket
+        $bucket = $this->client->call(Client::METHOD_POST, '/storage/buckets', [
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+            'x-appwrite-key' => $this->getProject()['apiKey']
+        ], [
+            'bucketId' => ID::unique(),
+            'name' => 'Test Bucket',
+            'permissions' => [
+                Permission::read(Role::user($this->getUser()['$id'])),
+                Permission::write(Role::user($this->getUser()['$id'])),
+            ],
+            'fileSecurity' => true
+        ]);
+        static::$bucketId = $bucket['body']['$id'];
+    }
+
+    public function createFiles(array $params = [])
+    {
+        $count = $params['files'] ?? 1;
+
+        // Create files
+        for ($i = 0; $i < $count; $i++) {
+            $response = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . static::$bucketId . '/files', [
+                'content-type' => 'multipart/form-data',
+                'x-appwrite-project' => $this->getProject()['$id'],
+                'x-appwrite-key' => $this->getProject()['apiKey']
+            ], [
+                'fileId' => ID::unique(),
+                'file' => new CURLFile(realpath(__DIR__ . '/../../resources/logo.png'), 'image/png', 'logo.png'),
+            ]);
+
+            static::$fileId = $response['body']['$id'];
+        }
+    }
+}

tests/benchmarks/Services/Storage/StorageCustomClientBench.php

@@ -0,0 +1,10 @@
+<?php
+
+namespace Tests\Benchmarks\Services\Storage;
+
+use Tests\E2E\Scopes\SideClient;
+
+class StorageCustomClientBench extends Base
+{
+    use SideClient;
+}

tests/benchmarks/Services/Storage/StorageCustomServerBench.php

@@ -0,0 +1,10 @@
+<?php
+
+namespace Tests\Benchmarks\Services\Storage;
+
+use Tests\E2E\Scopes\SideServer;
+
+class StorageCustomServerBench extends Base
+{
+    use SideServer;
+}

tests/benchmarks/Services/Users/UserCustomServerBench.php

@@ -0,0 +1,94 @@
+<?php
+
+namespace Tests\Benchmarks\Services\Users;
+
+use PhpBench\Attributes\BeforeMethods;
+use PhpBench\Attributes\ParamProviders;
+use Tests\Benchmarks\Scopes\Scope;
+use Tests\E2E\Client;
+use Tests\E2E\Scopes\ProjectCustom;
+use Tests\E2E\Scopes\SideServer;
+use Utopia\Database\ID;
+
+class UserCustomServerBench extends Scope
+{
+    use ProjectCustom;
+    use SideServer;
+
+    protected static string $userId;
+
+    public function benchUserCreate()
+    {
+        $id = ID::unique();
+
+        $this->client->call(Client::METHOD_POST, '/users', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), [
+            'userId' => $id,
+            'email' => 'test' . $id . '@example.com',
+            'password' => 'password',
+        ]);
+    }
+
+    #[ParamProviders(['provideCounts'])]
+    #[BeforeMethods(['createUsers'])]
+    public function benchUserReadList(array $params)
+    {
+        $this->client->call(Client::METHOD_GET, '/users', array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), [
+            'queries' => ['limit(' . $params['users'] . ')'],
+        ]);
+    }
+
+    #[BeforeMethods(['createUsers'])]
+    public function benchUserRead()
+    {
+        $this->client->call(Client::METHOD_GET, '/users/' . static::$userId, array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()));
+    }
+
+    #[BeforeMethods(['createUsers'])]
+    public function benchUserUpdate()
+    {
+        $this->client->call(Client::METHOD_PUT, '/users/' . static::$userId, array_merge([
+            'content-type' => 'application/json',
+            'x-appwrite-project' => $this->getProject()['$id'],
+        ], $this->getHeaders()), [
+            'name' => 'New Name',
+        ]);
+    }
+
+    public function createUsers(array $params = [])
+    {
+        $count = $params['documents'] ?? 1;
+
+        for ($i = 0; $i < $count; $i++) {
+            $id = ID::unique();
+
+            $response = $this->client->call(Client::METHOD_POST, '/users', array_merge([
+                'content-type' => 'application/json',
+                'x-appwrite-project' => $this->getProject()['$id'],
+            ], $this->getHeaders()), [
+                'userId' => $id,
+                'email' => 'test' . $id . '@example.com',
+                'password' => 'password',
+            ]);
+
+            static::$userId = $response['body']['$id'];
+        }
+    }
+
+    public function provideCounts(): array
+    {
+        return [
+            '1 User' => ['users' => 1],
+            '10 Users' => ['users' => 10],
+            '100 Users' => ['users' => 100],
+        ];
+    }
+}

tests/benchmarks/http.js

@@ -1,34 +0,0 @@
-import http from 'k6/http';
-import { check } from 'k6';
-import { Counter } from 'k6/metrics';
-
-// A simple counter for http requests
-export const requests = new Counter('http_reqs');
-
-// you can specify stages of your test (ramp up/down patterns) through the options object
-// target is the number of VUs you are aiming for
-
-export const options = {
-    stages: [
-    { target: 50, duration: '1m' },
-    // { target: 15, duration: '1m' },
-    // { target: 0, duration: '1m' },
-    ],
-    thresholds: {
-        requests: ['count < 100'],
-    },
-};
-
-export default function () {
-    const config = {
-        headers: {
-            'X-Appwrite-Key': '24356eb021863f81eb7dd77c7750304d0464e141cad6e9a8befa1f7d2b066fde190df3dab1e8d2639dbb82ee848da30501424923f4cd80d887ee40ad77ded62763ee489448523f6e39667f290f9a54b2ab8fad131a0bc985e6c0f760015f7f3411e40626c75646bb19d2bb2f7bf2f63130918220a206758cbc48845fd725a695',
-            'X-Appwrite-Project': '60479fe35d95d'
-        }}
-
-    const resDb = http.get('http://localhost:9501/', config);
-
-    check(resDb, {
-        'status is 200': (r) => r.status === 200,
-    });
-}

tests/benchmarks/ws.js

@@ -1,59 +0,0 @@
-// k6 run tests/benchmarks/ws.js
-
-import { URL } from 'https://jslib.k6.io/url/1.0.0/index.js';
-import ws from 'k6/ws';
-import { check } from 'k6';
-
-export let options = {
-    stages: [
-        {
-            duration: '10s',
-            target: 500
-    },
-        {
-            duration: '1m',
-            target: 500
-    },
-    ],
-}
-
-export default function () {
-    // const url = new URL('wss://appwrite-realtime.monitor-api.com/v1/realtime');
-    // url.searchParams.append('project', '604249e6b1a9f');
-    const url = new URL('ws://localhost/v1/realtime');
-    url.searchParams.append('project', 'console');
-    url.searchParams.append('channels[]', 'files');
-
-    const res = ws.connect(url.toString(), function (socket) {
-        let connection = false;
-        let checked = false;
-        let payload = null;
-        socket.on('open', () => {
-            connection = true;
-        });
-
-        socket.on('message', (data) => {
-            payload = data;
-            checked = true;
-        });
-
-        socket.setTimeout(function () {
-            check(payload, {
-                'connection opened': (r) => connection,
-                'message received': (r) => checked,
-                'channels are right': (r) => r === JSON.stringify({
-                    "type": "connected",
-                    "data": {
-                        "channels": [
-                            "files"
-                        ],
-                        "user": null
-                    }
-                })
-            })
-            socket.close();
-        }, 5000);
-    });
-
-    check(res, { 'status is 101': (r) => r && r.status === 101 });
-}