Merge pull request #8477 from appwrite/feat-add-multipart-support

Add multipart support
This commit is contained in:
Christy Jacob 2024-08-13 00:31:45 +04:00 committed by GitHub
commit 2de6a93b77
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 507 additions and 18 deletions

View file

@ -10,6 +10,7 @@ use Appwrite\Event\Usage;
use Appwrite\Event\Validator\FunctionEvent;
use Appwrite\Extend\Exception;
use Appwrite\Extend\Exception as AppwriteException;
use Appwrite\Functions\Validator\Headers;
use Appwrite\Messaging\Adapter\Realtime;
use Appwrite\Platform\Tasks\ScheduleExecutions;
use Appwrite\Task\Validator\Cron;
@ -44,6 +45,7 @@ use Utopia\Storage\Validator\FileSize;
use Utopia\Storage\Validator\Upload;
use Utopia\Swoole\Request;
use Utopia\System\System;
use Utopia\Validator\AnyOf;
use Utopia\Validator\ArrayList;
use Utopia\Validator\Assoc;
use Utopia\Validator\Boolean;
@ -1600,13 +1602,14 @@ App::post('/v1/functions/:functionId/executions')
->label('sdk.response.type', Response::CONTENT_TYPE_JSON)
->label('sdk.response.model', Response::MODEL_EXECUTION)
->param('functionId', '', new UID(), 'Function ID.')
->param('body', '', new Text(0, 0), 'HTTP body of execution. Default value is empty string.', true)
->param('body', '', new Text(10485760, 0), 'HTTP body of execution. Default value is empty string.', true)
->param('async', false, new Boolean(), 'Execute code in the background. Default value is false.', true)
->param('path', '/', new Text(2048), 'HTTP path of execution. Path can include query params. Default value is /', true)
->param('method', 'POST', new Whitelist(['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'OPTIONS'], true), 'HTTP method of execution. Default value is GET.', true)
->param('headers', [], new Assoc(), 'HTTP headers of execution. Defaults to empty.', true)
->param('headers', [], new AnyOf([new Text(65535), new Assoc()], AnyOf::TYPE_MIXED), 'HTTP headers of execution. Defaults to empty.', true)
->param('scheduledAt', null, new DatetimeValidator(requireDateInFuture: true), 'Scheduled execution time in [ISO 8601](https://www.iso.org/iso-8601-date-and-time-format.html) format. DateTime value must be in future.', true)
->inject('response')
->inject('request')
->inject('project')
->inject('dbForProject')
->inject('dbForConsole')
@ -1615,12 +1618,35 @@ App::post('/v1/functions/:functionId/executions')
->inject('queueForUsage')
->inject('queueForFunctions')
->inject('geodb')
->action(function (string $functionId, string $body, bool $async, string $path, string $method, array $headers, ?string $scheduledAt, Response $response, Document $project, Database $dbForProject, Database $dbForConsole, Document $user, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb) {
->action(function (string $functionId, string $body, bool $async, string $path, string $method, mixed $headers, ?string $scheduledAt, Response $response, Request $request, Document $project, Database $dbForProject, Database $dbForConsole, Document $user, Event $queueForEvents, Usage $queueForUsage, Func $queueForFunctions, Reader $geodb) {
if(!$async && !is_null($scheduledAt)) {
throw new Exception(Exception::GENERAL_BAD_REQUEST, 'Scheduled executions must run asynchronously. Set scheduledAt to a future date, or set async to true.');
}
/**
* @var array<string, mixed> $headers
*/
$assocParams = ['headers'];
foreach ($assocParams as $assocParam) {
if (!empty('headers') && !is_array($$assocParam)) {
$$assocParam = \json_decode($$assocParam, true);
}
}
$booleanParams = ['async'];
foreach ($booleanParams as $booleamParam) {
if (!empty($$booleamParam) && !is_bool($$booleamParam)) {
$$booleamParam = $$booleamParam === "true" ? true : false;
}
}
// 'headers' validator
$validator = new Headers();
if (!$validator->isValid($headers)) {
throw new Exception($validator->getDescription(), 400);
}
$function = Authorization::skip(fn () => $dbForProject->getDocument('functions', $functionId));
$isAPIKey = Auth::isAppUser(Authorization::getRoles());
@ -1927,6 +1953,17 @@ App::post('/v1/functions/:functionId/executions')
$execution->setAttribute('responseBody', $executionResponse['body'] ?? '');
$execution->setAttribute('responseHeaders', $headers);
$acceptTypes = \explode(', ', $request->getHeader('accept'));
foreach ($acceptTypes as $acceptType) {
if(\str_starts_with($acceptType, 'application/json') || \str_starts_with($acceptType, 'application/*')) {
$response->setContentType(Response::CONTENT_TYPE_JSON);
break;
} elseif (\str_starts_with($acceptType, 'multipart/form-data') || \str_starts_with($acceptType, 'multipart/*')) {
$response->setContentType(Response::CONTENT_TYPE_MULTIPART);
break;
}
}
$response
->setStatusCode(Response::STATUS_CODE_CREATED)
->dynamic($execution, Response::MODEL_EXECUTION);

2
composer.lock generated
View file

@ -5617,5 +5617,5 @@
"platform-overrides": {
"php": "8.3"
},
"plugin-api-version": "2.6.0"
"plugin-api-version": "2.3.0"
}

View file

@ -923,7 +923,7 @@ services:
hostname: proxy
<<: *x-logging
stop_signal: SIGINT
image: openruntimes/proxy:0.3.1
image: openruntimes/proxy:0.5.4
networks:
- appwrite
- runtimes

View file

@ -0,0 +1,103 @@
<?php
namespace Appwrite\Functions\Validator;
use Utopia\Validator;
/**
* Headers.
*
* Validates user provided headers
*/
class Headers extends Validator
{
protected bool $allowEmpty;
public function __construct(bool $allowEmpty = true)
{
$this->allowEmpty = $allowEmpty;
}
/**
* Get Description.
*
* Returns validator description
*
* @return string
*/
public function getDescription(): string
{
return 'Invalid header format. Header keys can only contain alphanumeric characters, underscores, and hyphens. Header keys cannot start with "x-appwrite-" prefix.';
}
/**
* Is valid.
*
* @param mixed $value
*
* @return bool
*/
public function isValid($value): bool
{
if ($this->allowEmpty && empty($value)) {
return true;
}
if (!\is_array($value)) {
return false;
}
if (\is_array($value)) {
foreach ($value as $key => $val) {
$length = \strlen($key);
// Reject non-string keys
if (!\is_string($key) || $length === 0) {
return false;
}
// Check first and last character
if (!ctype_alnum($key[0]) || !ctype_alnum($key[$length - 1])) {
return false;
}
// Check middle characters
for ($i = 1; $i < $length - 1; $i++) {
if (!ctype_alnum($key[$i]) && $key[$i] !== '-') {
return false;
}
}
// Check for x-appwrite- prefix
if (str_starts_with($key, 'x-appwrite-')) {
return false;
}
}
return true;
}
return false;
}
/**
* Is array
*
* Function will return true if object is array.
*
* @return bool
*/
public function isArray(): bool
{
return false;
}
/**
* Get Type
*
* Returns validator type.
*
* @return string
*/
public function getType(): string
{
return self::TYPE_OBJECT;
}
}

View file

@ -2,6 +2,7 @@
namespace Appwrite\Utopia;
use Appwrite\Utopia\Fetch\BodyMultipart;
use Appwrite\Utopia\Response\Filter;
use Appwrite\Utopia\Response\Model;
use Appwrite\Utopia\Response\Model\Account;
@ -107,6 +108,7 @@ use Appwrite\Utopia\Response\Model\Variable;
use Appwrite\Utopia\Response\Model\VcsContent;
use Appwrite\Utopia\Response\Model\Webhook;
use Exception;
use JsonException;
use Swoole\Http\Response as SwooleHTTPResponse;
// Keep last
use Utopia\Database\Document;
@ -486,6 +488,7 @@ class Response extends SwooleResponse
*/
public const CONTENT_TYPE_YAML = 'application/x-yaml';
public const CONTENT_TYPE_NULL = 'null';
public const CONTENT_TYPE_MULTIPART = 'multipart/form-data';
/**
* List of defined output objects
@ -556,7 +559,11 @@ class Response extends SwooleResponse
switch ($this->getContentType()) {
case self::CONTENT_TYPE_JSON:
$this->json(!empty($output) ? $output : new \stdClass());
try {
$this->json(!empty($output) ? $output : new \stdClass());
} catch (JsonException $e) {
throw new Exception('Failed to parse response: ' . $e->getMessage(), 400);
}
break;
case self::CONTENT_TYPE_YAML:
@ -566,6 +573,10 @@ class Response extends SwooleResponse
case self::CONTENT_TYPE_NULL:
break;
case self::CONTENT_TYPE_MULTIPART:
$this->multipart(!empty($output) ? $output : new \stdClass());
break;
default:
if ($model === self::MODEL_NONE) {
$this->noContent();
@ -697,6 +708,49 @@ class Response extends SwooleResponse
->send(\yaml_emit($data, YAML_UTF8_ENCODING));
}
/**
* Multipart
*
* This helper is for sending multipart/form-data HTTP response.
* It sets relevant content type header ('multipart/form-data') and convert a PHP array ($data) to valid Multipart using BodyMultipart
*
* @param array $data
*
* @return void
*/
public function multipart(array $data): void
{
$multipart = new BodyMultipart();
foreach ($data as $key => $value) {
$multipart->setPart($key, $value);
}
$this
->send($multipart->exportBody());
}
/**
* JSON
*
* This helper is for sending JSON HTTP response.
* It sets relevant content type header ('application/json') and convert a PHP array ($data) to valid JSON using native json_encode
*
* @see http://en.wikipedia.org/wiki/JSON
*
* @param mixed $data
* @return void
*/
public function json($data): void
{
if (!is_array($data) && !$data instanceof \stdClass) {
throw new \Exception('Response body is not a valid JSON object.');
}
$this
->setContentType(Response::CONTENT_TYPE_JSON, self::CHARSET_UTF8)
->send(\json_encode($data, JSON_UNESCAPED_UNICODE | JSON_THROW_ON_ERROR));
}
/**
* @return array
*/

View file

@ -2,6 +2,7 @@
namespace Tests\E2E;
use Appwrite\Utopia\Fetch\BodyMultipart;
use Exception;
class Client
@ -224,18 +225,35 @@ class Client
$responseType = $responseHeaders['content-type'] ?? '';
$responseStatus = curl_getinfo($ch, CURLINFO_HTTP_CODE);
if ($decode && substr($responseType, 0, strpos($responseType, ';')) == 'application/json') {
$json = json_decode($responseBody, true);
if ($decode) {
$strpos = strpos($responseType, ';');
$strpos = \is_bool($strpos) ? \strlen($responseType) : $strpos;
switch (substr($responseType, 0, $strpos)) {
case 'multipart/form-data':
$boundary = \explode('boundary=', $responseHeaders['content-type'] ?? '')[1] ?? '';
$multipartResponse = new BodyMultipart($boundary);
$multipartResponse->load(\is_bool($responseBody) ? '' : $responseBody);
if ($json === null) {
throw new Exception('Failed to parse response: ' . $responseBody);
$responseBody = $multipartResponse->getParts();
break;
case 'application/json':
if (\is_bool($responseBody)) {
throw new Exception('Response is not a valid JSON.');
}
$json = json_decode($responseBody, true);
if ($json === null) {
throw new Exception('Failed to parse response: ' . $responseBody);
}
$responseBody = $json;
$json = null;
break;
}
$responseBody = $json;
$json = null;
}
if ((curl_errno($ch))) {
if ((curl_errno($ch)/* || 200 != $responseStatus*/)) {
throw new Exception(curl_error($ch) . ' with status code ' . $responseStatus, $responseStatus);
}
@ -244,7 +262,7 @@ class Client
$responseHeaders['status-code'] = $responseStatus;
if ($responseStatus === 500) {
echo 'Server error(' . $method . ': ' . $path . '. Params: ' . json_encode($params) . '): ' . json_encode($responseBody) . "\n";
echo 'Server error(' . $method . ': ' . $path . '. Params: ' . json_encode($params) . '): ' . json_encode($responseBody) . '\n';
}
return [

View file

@ -131,7 +131,6 @@ class HTTPTest extends Scope
'content-type' => 'application/json',
], json_decode(file_get_contents($directory . $file), true));
$response['body'] = json_decode($response['body'], true);
$this->assertEquals(200, $response['headers']['status-code']);
// looks like recent change in the validator
$this->assertTrue(empty($response['body']['schemaValidationMessages']));

View file

@ -1435,6 +1435,176 @@ class FunctionsCustomServerTest extends Scope
$this->assertEquals(204, $response['headers']['status-code']);
}
public function testCreateCustomExecutionBinaryResponse()
{
$timeout = 15;
$code = realpath(__DIR__ . '/../../../resources/functions') . "/php-binary-response/code.tar.gz";
$this->packageCode('php-binary-response');
$function = $this->client->call(Client::METHOD_POST, '/functions', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'functionId' => ID::unique(),
'name' => 'Test PHP Binary executions',
'runtime' => 'php-8.0',
'entrypoint' => 'index.php',
'timeout' => $timeout,
'execute' => ['any']
]);
$functionId = $function['body']['$id'] ?? '';
$this->assertEquals(201, $function['headers']['status-code']);
$deployment = $this->client->call(Client::METHOD_POST, '/functions/' . $functionId . '/deployments', array_merge([
'content-type' => 'multipart/form-data',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'entrypoint' => 'index.php',
'code' => new CURLFile($code, 'application/x-gzip', basename($code)),
'activate' => true
]);
$deploymentId = $deployment['body']['$id'] ?? '';
$this->assertEquals(202, $deployment['headers']['status-code']);
$this->awaitDeploymentIsBuilt($function['body']['$id'], $deploymentId, checkForSuccess: false);
$deployment = $this->client->call(Client::METHOD_PATCH, '/functions/' . $functionId . '/deployments/' . $deploymentId, array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(200, $deployment['headers']['status-code']);
// Wait a little for activation to finish
sleep(5);
$execution = $this->client->call(Client::METHOD_POST, '/functions/' . $functionId . '/executions', array_merge([
'x-appwrite-project' => $this->getProject()['$id'],
'accept' => 'multipart/form-data',
], $this->getHeaders()), [
'body' => null,
]);
$this->assertEquals(201, $execution['headers']['status-code']);
$this->assertStringContainsString('multipart/form-data', $execution['headers']['content-type']);
$bytes = unpack('C*byte', $execution['body']['responseBody']);
$this->assertCount(3, $bytes);
$this->assertEquals(0, $bytes['byte1']);
$this->assertEquals(10, $bytes['byte2']);
$this->assertEquals(255, $bytes['byte3']);
/**
* Test for FAILURE
*/
$execution = $this->client->call(Client::METHOD_POST, '/functions/' . $functionId . '/executions', array_merge([
'x-appwrite-project' => $this->getProject()['$id'],
'accept' => 'application/json',
], $this->getHeaders()), [
'body' => null,
]);
$this->assertEquals(400, $execution['headers']['status-code']);
$this->assertStringContainsString('Failed to parse response', $execution['body']['message']);
// Cleanup : Delete function
$response = $this->client->call(Client::METHOD_DELETE, '/functions/' . $functionId, [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey'],
], []);
$this->assertEquals(204, $response['headers']['status-code']);
}
public function testCreateCustomExecutionBinaryRequest()
{
$timeout = 15;
$code = realpath(__DIR__ . '/../../../resources/functions') . "/php-binary-request/code.tar.gz";
$this->packageCode('php-binary-request');
$function = $this->client->call(Client::METHOD_POST, '/functions', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'functionId' => ID::unique(),
'name' => 'Test PHP Binary executions',
'runtime' => 'php-8.0',
'entrypoint' => 'index.php',
'timeout' => $timeout,
'execute' => ['any']
]);
$functionId = $function['body']['$id'] ?? '';
$this->assertEquals(201, $function['headers']['status-code']);
$deployment = $this->client->call(Client::METHOD_POST, '/functions/' . $functionId . '/deployments', array_merge([
'content-type' => 'multipart/form-data',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'entrypoint' => 'index.php',
'code' => new CURLFile($code, 'application/x-gzip', basename($code)),
'activate' => true
]);
$deploymentId = $deployment['body']['$id'] ?? '';
$this->assertEquals(202, $deployment['headers']['status-code']);
$this->awaitDeploymentIsBuilt($function['body']['$id'], $deploymentId, checkForSuccess: false);
$deployment = $this->client->call(Client::METHOD_PATCH, '/functions/' . $functionId . '/deployments/' . $deploymentId, array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(200, $deployment['headers']['status-code']);
// Wait a little for activation to finish
sleep(5);
$bytes = pack('C*', ...[0, 20, 255]);
$execution = $this->client->call(Client::METHOD_POST, '/functions/' . $functionId . '/executions', array_merge([
'content-type' => 'multipart/form-data',
'x-appwrite-project' => $this->getProject()['$id'],
'accept' => 'application/json',
], $this->getHeaders()), [
'body' => $bytes,
], false);
$executionBody = json_decode($execution['body'], true);
$this->assertEquals(201, $execution['headers']['status-code']);
$this->assertEquals(\md5($bytes), $executionBody['responseBody']);
$this->assertStringStartsWith('application/json', $execution['headers']['content-type']);
/**
* Test for FAILURE
*/
$execution = $this->client->call(Client::METHOD_POST, '/functions/' . $functionId . '/executions', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'accept' => 'application/json',
], $this->getHeaders()), [
'body' => $bytes,
], false);
$executionBody = json_decode($execution['body'], true);
$this->assertNotEquals(\md5($bytes), $executionBody['responseBody']);
// Cleanup : Delete function
$response = $this->client->call(Client::METHOD_DELETE, '/functions/' . $functionId, [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey'],
], []);
$this->assertEquals(204, $response['headers']['status-code']);
}
public function testv2Function()
{
$timeout = 15;
@ -1877,7 +2047,7 @@ class FunctionsCustomServerTest extends Scope
$this->assertEquals(204, $response['headers']['status-code']);
}
public function testFunctionsDomainBianryResponse()
public function testFunctionsDomainBinaryResponse()
{
$timeout = 15;
$code = realpath(__DIR__ . '/../../../resources/functions') . "/php-binary-response/code.tar.gz";
@ -1963,7 +2133,7 @@ class FunctionsCustomServerTest extends Scope
$this->assertEquals(204, $response['headers']['status-code']);
}
public function testFunctionsDomainBianryRequest()
public function testFunctionsDomainBinaryRequest()
{
$timeout = 15;
$code = realpath(__DIR__ . '/../../../resources/functions') . "/php-binary-request/code.tar.gz";

View file

@ -0,0 +1,108 @@
<?php
namespace Tests\Unit\Functions\Validator;
use Appwrite\Functions\Validator\Headers;
use PHPUnit\Framework\TestCase;
class HeadersTest extends TestCase
{
protected ?Headers $object = null;
public function setUp(): void
{
$this->object = new Headers();
}
public function testValues(): void
{
$headers = [
'headerKey' => 'headerValue',
];
$this->assertEquals($this->object->isValid($headers), true);
$headers = [
'headerKey' => 'headerValue',
'x-appwrite-key' => 'headerValue',
];
$this->assertEquals($this->object->isValid($headers), false);
$headers = [
'headerKey' => 'headerValue',
'headerKey2' => 'headerValue2',
];
$this->assertEquals($this->object->isValid($headers), true);
$headers = [
'headerKey' => 'headerValue',
'x-appwrite-project' => 'headerValue',
'headerKey2' => 'headerValue2',
];
$this->assertEquals($this->object->isValid($headers), false);
$headers = [
'header/////Key' => 'headerValue',
];
$this->assertEquals($this->object->isValid($headers), false);
$headers = [
'Content-Type' => 'application/json',
'X-Custom-Header' => 'value'
];
$this->assertEquals($this->object->isValid($headers), true);
$headers = [
'X-Custom-Header_With-Hyphens_and_Underscores' => 'value'
];
$this->assertFalse($this->object->isValid($headers));
$headers = [
'X-Header-123' => 'value'
];
$this->assertTrue($this->object->isValid($headers));
$headers = [
'X-Header<>' => 'value'
];
$this->assertFalse($this->object->isValid($headers));
$headers = [
'X Header' => 'value'
];
$this->assertFalse($this->object->isValid($headers));
$headers = [
'' => 'value'
];
$this->assertFalse($this->object->isValid($headers));
$headers = [
null => 'value',
];
$this->assertFalse($this->object->isValid($headers));
$headers = [
'X-Header' => null,
];
$this->assertTrue($this->object->isValid($headers));
$headers = [
true => 'value',
];
$this->assertFalse($this->object->isValid($headers));
$headers = [
'a' => 'b',
];
$this->assertTrue($this->object->isValid($headers));
$headers = 123;
$this->assertFalse($this->object->isValid($headers));
$headers = true;
$this->assertFalse($this->object->isValid($headers));
$headers = 'string';
$this->assertFalse($this->object->isValid($headers));
}
}