From 6b228bf57bd47bdfc46471d9c94264837951bf98 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Mon, 6 Jun 2022 10:57:23 +0100 Subject: [PATCH 1/9] Add Runtime state tracking using a Swoole table --- src/Executor/Executor.php | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/Executor/Executor.php b/src/Executor/Executor.php index fa17ce2a8a..a85a184ade 100644 --- a/src/Executor/Executor.php +++ b/src/Executor/Executor.php @@ -4,7 +4,7 @@ namespace Executor; use Exception; use Utopia\App; -use Utopia\CLI\Console; +use Swoole\Table; class Executor { @@ -19,6 +19,7 @@ class Executor const METHOD_TRACE = 'TRACE'; private $endpoint; + private $runtimeQueue; private $selfSigned = false; @@ -32,6 +33,9 @@ class Executor throw new Exception('Unsupported endpoint'); } $this->endpoint = $endpoint; + $this->runtimeQueue = new Table(1024); + $this->runtimeQueue->column('id', Table::TYPE_STRING, 128); + $this->runtimeQueue->column('state', Table::TYPE_STRING, 128); } /** @@ -83,6 +87,11 @@ class Executor 'commands' => $commands ]; + $this->runtimeQueue->set($params['runtimeId'], [ + 'id' => $params['runtimeId'], + 'state' => 'pending' + ]); + $timeout = (int) App::getEnv('_APP_FUNCTIONS_BUILD_TIMEOUT', 900); $response = $this->call(self::METHOD_POST, $route, $headers, $params, true, $timeout); @@ -174,6 +183,16 @@ class Executor case $status < 400: return $response['body']; case $status === 404: + if ($this->runtimeQueue->get($params['runtimeId'])) { + if ($this->runtimeQueue->get($params['runtimeId'])['state'] === 'pending') { + sleep(1); + continue 2; + } + } + $this->runtimeQueue->set($params['runtimeId'], [ + 'id' => $params['runtimeId'], + 'state' => 'pending' + ]); $response = $this->createRuntime( deploymentId: $deploymentId, projectId: $projectId, From d2c4bb9872156c2b8bf82c0bc93ecffae7e7974a Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Tue, 7 Jun 2022 14:55:44 +0100 Subject: [PATCH 2/9] Continue work on implementing local runtime state --- app/executor.php | 19 +++++++++++++++++-- src/Executor/Executor.php | 32 +++++++++----------------------- 2 files changed, 26 insertions(+), 25 deletions(-) diff --git a/app/executor.php b/app/executor.php index 83f83e6b54..1eb73df85c 100644 --- a/app/executor.php +++ b/app/executor.php @@ -177,6 +177,12 @@ App::post('/v1/runtimes') ->inject('response') ->action(function (string $runtimeId, string $source, string $destination, array $vars, array $commands, string $runtime, string $baseImage, string $entrypoint, bool $remove, string $workdir, $orchestrationPool, $activeRuntimes, Response $response) { if ($activeRuntimes->exists($runtimeId)) { + + if ($activeRuntimes->get($runtimeId)['key'] == 'Pending') { + sleep(1); + throw new \Exception('A runtime with the same ID is already being created.', 500); + } + throw new Exception('Runtime already exists.', 409); } @@ -188,6 +194,17 @@ App::post('/v1/runtimes') $endTime = 0; $orchestration = $orchestrationPool->get(); + $secret = \bin2hex(\random_bytes(16)); + + $activeRuntimes->set($runtimeId, [ + 'id' => $containerId, + 'name' => $runtimeId, + 'created' => $startTime, + 'updated' => $endTime, + 'status' => 'Pending', + 'key' => $secret, + ]); + try { Console::info('Building container : ' . $runtimeId); @@ -219,7 +236,6 @@ App::post('/v1/runtimes') /** * Create container */ - $secret = \bin2hex(\random_bytes(16)); $vars = \array_merge($vars, [ 'INTERNAL_RUNTIME_KEY' => $secret, 'INTERNAL_RUNTIME_ENTRYPOINT' => $entrypoint, @@ -440,7 +456,6 @@ App::post('/v1/execution') ->inject('response') ->action( function (string $runtimeId, array $vars, string $data, $timeout, $activeRuntimes, Response $response) { - if (!$activeRuntimes->exists($runtimeId)) { throw new Exception('Runtime not found. Please create the runtime.', 404); } diff --git a/src/Executor/Executor.php b/src/Executor/Executor.php index ba6d67df04..4ca1704705 100644 --- a/src/Executor/Executor.php +++ b/src/Executor/Executor.php @@ -4,7 +4,6 @@ namespace Executor; use Exception; use Utopia\App; -use Swoole\Table; class Executor { @@ -19,7 +18,6 @@ class Executor public const METHOD_TRACE = 'TRACE'; private $endpoint; - private $runtimeQueue; private $selfSigned = false; @@ -33,9 +31,6 @@ class Executor throw new Exception('Unsupported endpoint'); } $this->endpoint = $endpoint; - $this->runtimeQueue = new Table(1024); - $this->runtimeQueue->column('id', Table::TYPE_STRING, 128); - $this->runtimeQueue->column('state', Table::TYPE_STRING, 128); } /** @@ -87,11 +82,6 @@ class Executor 'commands' => $commands ]; - $this->runtimeQueue->set($params['runtimeId'], [ - 'id' => $params['runtimeId'], - 'state' => 'pending' - ]); - $timeout = (int) App::getEnv('_APP_FUNCTIONS_BUILD_TIMEOUT', 900); $response = $this->call(self::METHOD_POST, $route, $headers, $params, true, $timeout); @@ -174,25 +164,15 @@ class Executor /* Add 2 seconds as a buffer to the actual timeout value since there can be a slight variance*/ $requestTimeout = $timeout + 2; - $response = $this->call(self::METHOD_POST, $route, $headers, $params, true, $requestTimeout); - $status = $response['headers']['status-code']; - for ($attempts = 0; $attempts < 10; $attempts++) { + $response = $this->call(self::METHOD_POST, $route, $headers, $params, true, $requestTimeout); + $status = $response['headers']['status-code']; + try { switch (true) { case $status < 400: return $response['body']; case $status === 404: - if ($this->runtimeQueue->get($params['runtimeId'])) { - if ($this->runtimeQueue->get($params['runtimeId'])['state'] === 'pending') { - sleep(1); - continue 2; - } - } - $this->runtimeQueue->set($params['runtimeId'], [ - 'id' => $params['runtimeId'], - 'state' => 'pending' - ]); $response = $this->createRuntime( deploymentId: $deploymentId, projectId: $projectId, @@ -205,6 +185,12 @@ class Executor ); $response = $this->call(self::METHOD_POST, $route, $headers, $params, true, $requestTimeout); $status = $response['headers']['status-code']; + + // 500 usually means that the runtime is being created but is not ready, retry. + if ($status == 500) { + continue 2; + } + break; case $status === 406: $response = $this->call(self::METHOD_POST, $route, $headers, $params, true, $requestTimeout); From 7fce57f35d50163db76a176368e7ea2385c57eeb Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Wed, 8 Jun 2022 16:15:17 +0100 Subject: [PATCH 3/9] Clean up unused code and get tests working --- app/executor.php | 36 +++++++++++++++++++++++++----------- src/Executor/Executor.php | 10 ++++------ 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/app/executor.php b/app/executor.php index 1eb73df85c..8df9f7090f 100644 --- a/app/executor.php +++ b/app/executor.php @@ -178,9 +178,8 @@ App::post('/v1/runtimes') ->action(function (string $runtimeId, string $source, string $destination, array $vars, array $commands, string $runtime, string $baseImage, string $entrypoint, bool $remove, string $workdir, $orchestrationPool, $activeRuntimes, Response $response) { if ($activeRuntimes->exists($runtimeId)) { - if ($activeRuntimes->get($runtimeId)['key'] == 'Pending') { - sleep(1); - throw new \Exception('A runtime with the same ID is already being created.', 500); + if ($activeRuntimes->get($runtimeId)['key'] == 'pending') { + throw new \Exception('A runtime with the same ID is already being created. Attempt a execution soon.', 500); } throw new Exception('Runtime already exists.', 409); @@ -196,14 +195,16 @@ App::post('/v1/runtimes') $secret = \bin2hex(\random_bytes(16)); - $activeRuntimes->set($runtimeId, [ - 'id' => $containerId, - 'name' => $runtimeId, - 'created' => $startTime, - 'updated' => $endTime, - 'status' => 'Pending', - 'key' => $secret, - ]); + if (!$remove) { + $activeRuntimes->set($runtimeId, [ + 'id' => $containerId, + 'name' => $runtimeId, + 'created' => $startTime, + 'updated' => $endTime, + 'status' => 'pending', + 'key' => $secret, + ]); + } try { Console::info('Building container : ' . $runtimeId); @@ -460,6 +461,19 @@ App::post('/v1/execution') throw new Exception('Runtime not found. Please create the runtime.', 404); } + for ($i = 0; $i < 5; $i++) { + if ($activeRuntimes->get($runtimeId)['status'] === 'pending') { + Console::info('Waiting for runtime to be ready...'); + sleep(1); + } else { + break; + } + + if ($i === 4) { + throw new Exception('Runtime failed to launch in allocated time.', 500); + } + } + $runtime = $activeRuntimes->get($runtimeId); $secret = $runtime['key']; if (empty($secret)) { diff --git a/src/Executor/Executor.php b/src/Executor/Executor.php index 4ca1704705..ed00cd0b08 100644 --- a/src/Executor/Executor.php +++ b/src/Executor/Executor.php @@ -161,11 +161,13 @@ class Executor 'timeout' => $timeout, ]; + /* Add 2 seconds as a buffer to the actual timeout value since there can be a slight variance*/ $requestTimeout = $timeout + 2; + $response = $this->call(self::METHOD_POST, $route, $headers, $params, true, $requestTimeout); + for ($attempts = 0; $attempts < 10; $attempts++) { - $response = $this->call(self::METHOD_POST, $route, $headers, $params, true, $requestTimeout); $status = $response['headers']['status-code']; try { @@ -183,14 +185,10 @@ class Executor entrypoint: $entrypoint, commands: [] ); + $response = $this->call(self::METHOD_POST, $route, $headers, $params, true, $requestTimeout); $status = $response['headers']['status-code']; - // 500 usually means that the runtime is being created but is not ready, retry. - if ($status == 500) { - continue 2; - } - break; case $status === 406: $response = $this->call(self::METHOD_POST, $route, $headers, $params, true, $requestTimeout); From 41ae3a87db98467a63ee6eb7842ecf0b9a7b420d Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Thu, 9 Jun 2022 10:38:10 +0100 Subject: [PATCH 4/9] Delete Executor.php --- src/Executor/Executor.php | 350 -------------------------------------- 1 file changed, 350 deletions(-) delete mode 100644 src/Executor/Executor.php diff --git a/src/Executor/Executor.php b/src/Executor/Executor.php deleted file mode 100644 index ed00cd0b08..0000000000 --- a/src/Executor/Executor.php +++ /dev/null @@ -1,350 +0,0 @@ - '', - ]; - - public function __construct(string $endpoint) - { - if (!filter_var($endpoint, FILTER_VALIDATE_URL)) { - throw new Exception('Unsupported endpoint'); - } - $this->endpoint = $endpoint; - } - - /** - * Create runtime - * - * Launches a runtime container for a deployment ready for execution - * - * @param string $deploymentId - * @param string $projectId - * @param string $source - * @param string $runtime - * @param string $baseImage - * @param bool $remove - * @param string $entrypoint - * @param string $workdir - * @param string $destinaction - * @param string $network - * @param array $vars - * @param array $commands - */ - public function createRuntime( - string $deploymentId, - string $projectId, - string $source, - string $runtime, - string $baseImage, - bool $remove = false, - string $entrypoint = '', - string $workdir = '', - string $destination = '', - array $vars = [], - array $commands = [] - ) { - $route = "/runtimes"; - $headers = [ - 'content-type' => 'application/json', - 'x-appwrite-executor-key' => App::getEnv('_APP_EXECUTOR_SECRET', '') - ]; - $params = [ - 'runtimeId' => "$projectId-$deploymentId", - 'source' => $source, - 'destination' => $destination, - 'runtime' => $runtime, - 'baseImage' => $baseImage, - 'entrypoint' => $entrypoint, - 'workdir' => $workdir, - 'vars' => $vars, - 'remove' => $remove, - 'commands' => $commands - ]; - - $timeout = (int) App::getEnv('_APP_FUNCTIONS_BUILD_TIMEOUT', 900); - - $response = $this->call(self::METHOD_POST, $route, $headers, $params, true, $timeout); - - $status = $response['headers']['status-code']; - if ($status >= 400) { - throw new \Exception($response['body']['message'], $status); - } - - return $response['body']; - } - - /** - * Delete Runtime - * - * Deletes a runtime and cleans up any containers remaining. - * - * @param string $projectId - * @param string $deploymentId - */ - public function deleteRuntime(string $projectId, string $deploymentId) - { - $runtimeId = "$projectId-$deploymentId"; - $route = "/runtimes/$runtimeId"; - $headers = [ - 'content-type' => 'application/json', - 'x-appwrite-executor-key' => App::getEnv('_APP_EXECUTOR_SECRET', '') - ]; - - $params = []; - - $response = $this->call(self::METHOD_DELETE, $route, $headers, $params, true, 30); - - $status = $response['headers']['status-code']; - if ($status >= 400) { - throw new \Exception($response['body']['message'], $status); - } - - return $response['body']; - } - - /** - * Create an execution - * - * @param string $projectId - * @param string $deploymentId - * @param string $path - * @param array $vars - * @param string $entrypoint - * @param string $data - * @param string runtime - * @param string $baseImage - * @param int $timeout - * - * @return array - */ - public function createExecution( - string $projectId, - string $deploymentId, - string $path, - array $vars, - string $entrypoint, - string $data, - string $runtime, - string $baseImage, - $timeout - ) { - $route = "/execution"; - $headers = [ - 'content-type' => 'application/json', - 'x-appwrite-executor-key' => App::getEnv('_APP_EXECUTOR_SECRET', '') - ]; - $params = [ - 'runtimeId' => "$projectId-$deploymentId", - 'vars' => $vars, - 'data' => $data, - 'timeout' => $timeout, - ]; - - - /* Add 2 seconds as a buffer to the actual timeout value since there can be a slight variance*/ - $requestTimeout = $timeout + 2; - - $response = $this->call(self::METHOD_POST, $route, $headers, $params, true, $requestTimeout); - - for ($attempts = 0; $attempts < 10; $attempts++) { - $status = $response['headers']['status-code']; - - try { - switch (true) { - case $status < 400: - return $response['body']; - case $status === 404: - $response = $this->createRuntime( - deploymentId: $deploymentId, - projectId: $projectId, - source: $path, - runtime: $runtime, - baseImage: $baseImage, - vars: $vars, - entrypoint: $entrypoint, - commands: [] - ); - - $response = $this->call(self::METHOD_POST, $route, $headers, $params, true, $requestTimeout); - $status = $response['headers']['status-code']; - - break; - case $status === 406: - $response = $this->call(self::METHOD_POST, $route, $headers, $params, true, $requestTimeout); - $status = $response['headers']['status-code']; - break; - default: - throw new \Exception($response['body']['message'], $status); - } - } catch (\Exception $e) { - throw new \Exception($e->getMessage(), $e->getCode()); - } - sleep(2); - } - - throw new Exception($response['body']['message'], 503); - } - - /** - * Call - * - * Make an API call - * - * @param string $method - * @param string $path - * @param array $params - * @param array $headers - * @param bool $decode - * @return array|string - * @throws Exception - */ - public function call(string $method, string $path = '', array $headers = [], array $params = [], bool $decode = true, int $timeout = 15) - { - $headers = array_merge($this->headers, $headers); - $ch = curl_init($this->endpoint . $path . (($method == self::METHOD_GET && !empty($params)) ? '?' . http_build_query($params) : '')); - $responseHeaders = []; - $responseStatus = -1; - $responseType = ''; - $responseBody = ''; - - switch ($headers['content-type']) { - case 'application/json': - $query = json_encode($params); - break; - - case 'multipart/form-data': - $query = $this->flatten($params); - break; - - default: - $query = http_build_query($params); - break; - } - - foreach ($headers as $i => $header) { - $headers[] = $i . ':' . $header; - unset($headers[$i]); - } - - curl_setopt($ch, CURLOPT_CUSTOMREQUEST, $method); - curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); - curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); - curl_setopt($ch, CURLOPT_HTTPHEADER, $headers); - curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 0); - curl_setopt($ch, CURLOPT_TIMEOUT, $timeout); - curl_setopt($ch, CURLOPT_HEADERFUNCTION, function ($curl, $header) use (&$responseHeaders) { - $len = strlen($header); - $header = explode(':', $header, 2); - - if (count($header) < 2) { // ignore invalid headers - return $len; - } - - $responseHeaders[strtolower(trim($header[0]))] = trim($header[1]); - - return $len; - }); - - if ($method != self::METHOD_GET) { - curl_setopt($ch, CURLOPT_POSTFIELDS, $query); - } - - // Allow self signed certificates - if ($this->selfSigned) { - curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false); - curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); - } - - $responseBody = curl_exec($ch); - $responseType = $responseHeaders['content-type'] ?? ''; - $responseStatus = curl_getinfo($ch, CURLINFO_HTTP_CODE); - - if ($decode) { - switch (substr($responseType, 0, strpos($responseType, ';'))) { - case 'application/json': - $json = json_decode($responseBody, true); - - if ($json === null) { - throw new Exception('Failed to parse response: ' . $responseBody); - } - - $responseBody = $json; - $json = null; - break; - } - } - - if ((curl_errno($ch)/* || 200 != $responseStatus*/)) { - throw new Exception(curl_error($ch) . ' with status code ' . $responseStatus, $responseStatus); - } - - curl_close($ch); - - $responseHeaders['status-code'] = $responseStatus; - - return [ - 'headers' => $responseHeaders, - 'body' => $responseBody - ]; - } - - /** - * Parse Cookie String - * - * @param string $cookie - * @return array - */ - public function parseCookie(string $cookie): array - { - $cookies = []; - - parse_str(strtr($cookie, array('&' => '%26', '+' => '%2B', ';' => '&')), $cookies); - - return $cookies; - } - - /** - * Flatten params array to PHP multiple format - * - * @param array $data - * @param string $prefix - * @return array - */ - protected function flatten(array $data, string $prefix = ''): array - { - $output = []; - - foreach ($data as $key => $value) { - $finalKey = $prefix ? "{$prefix}[{$key}]" : $key; - - if (is_array($value)) { - $output += $this->flatten($value, $finalKey); // @todo: handle name collision here if needed - } else { - $output[$finalKey] = $value; - } - } - - return $output; - } -} From 16dd539401ff8f9d6073f0e2539ab8af48d15d0b Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Thu, 9 Jun 2022 10:39:04 +0100 Subject: [PATCH 5/9] Create Executor.php --- src/Executor/Executor.php | 347 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 347 insertions(+) create mode 100644 src/Executor/Executor.php diff --git a/src/Executor/Executor.php b/src/Executor/Executor.php new file mode 100644 index 0000000000..82b09ffe6b --- /dev/null +++ b/src/Executor/Executor.php @@ -0,0 +1,347 @@ + '', + ]; + + public function __construct(string $endpoint) + { + if (!filter_var($endpoint, FILTER_VALIDATE_URL)) { + throw new Exception('Unsupported endpoint'); + } + $this->endpoint = $endpoint; + } + + /** + * Create runtime + * + * Launches a runtime container for a deployment ready for execution + * + * @param string $deploymentId + * @param string $projectId + * @param string $source + * @param string $runtime + * @param string $baseImage + * @param bool $remove + * @param string $entrypoint + * @param string $workdir + * @param string $destinaction + * @param string $network + * @param array $vars + * @param array $commands + */ + public function createRuntime( + string $deploymentId, + string $projectId, + string $source, + string $runtime, + string $baseImage, + bool $remove = false, + string $entrypoint = '', + string $workdir = '', + string $destination = '', + array $vars = [], + array $commands = [] + ) { + $route = "/runtimes"; + $headers = [ + 'content-type' => 'application/json', + 'x-appwrite-executor-key' => App::getEnv('_APP_EXECUTOR_SECRET', '') + ]; + $params = [ + 'runtimeId' => "$projectId-$deploymentId", + 'source' => $source, + 'destination' => $destination, + 'runtime' => $runtime, + 'baseImage' => $baseImage, + 'entrypoint' => $entrypoint, + 'workdir' => $workdir, + 'vars' => $vars, + 'remove' => $remove, + 'commands' => $commands + ]; + + $timeout = (int) App::getEnv('_APP_FUNCTIONS_BUILD_TIMEOUT', 900); + + $response = $this->call(self::METHOD_POST, $route, $headers, $params, true, $timeout); + + $status = $response['headers']['status-code']; + if ($status >= 400) { + throw new \Exception($response['body']['message'], $status); + } + + return $response['body']; + } + + /** + * Delete Runtime + * + * Deletes a runtime and cleans up any containers remaining. + * + * @param string $projectId + * @param string $deploymentId + */ + public function deleteRuntime(string $projectId, string $deploymentId) + { + $runtimeId = "$projectId-$deploymentId"; + $route = "/runtimes/$runtimeId"; + $headers = [ + 'content-type' => 'application/json', + 'x-appwrite-executor-key' => App::getEnv('_APP_EXECUTOR_SECRET', '') + ]; + + $params = []; + + $response = $this->call(self::METHOD_DELETE, $route, $headers, $params, true, 30); + + $status = $response['headers']['status-code']; + if ($status >= 400) { + throw new \Exception($response['body']['message'], $status); + } + + return $response['body']; + } + + /** + * Create an execution + * + * @param string $projectId + * @param string $deploymentId + * @param string $path + * @param array $vars + * @param string $entrypoint + * @param string $data + * @param string runtime + * @param string $baseImage + * @param int $timeout + * + * @return array + */ + public function createExecution( + string $projectId, + string $deploymentId, + string $path, + array $vars, + string $entrypoint, + string $data, + string $runtime, + string $baseImage, + $timeout + ) { + $route = "/execution"; + $headers = [ + 'content-type' => 'application/json', + 'x-appwrite-executor-key' => App::getEnv('_APP_EXECUTOR_SECRET', '') + ]; + $params = [ + 'runtimeId' => "$projectId-$deploymentId", + 'vars' => $vars, + 'data' => $data, + 'timeout' => $timeout, + ]; + + /* Add 2 seconds as a buffer to the actual timeout value since there can be a slight variance*/ + $requestTimeout = $timeout + 2; + + $response = $this->call(self::METHOD_POST, $route, $headers, $params, true, $requestTimeout); + $status = $response['headers']['status-code']; + + for ($attempts = 0; $attempts < 10; $attempts++) { + try { + switch (true) { + case $status < 400: + return $response['body']; + case $status === 404: + $response = $this->createRuntime( + deploymentId: $deploymentId, + projectId: $projectId, + source: $path, + runtime: $runtime, + baseImage: $baseImage, + vars: $vars, + entrypoint: $entrypoint, + commands: [] + ); + $response = $this->call(self::METHOD_POST, $route, $headers, $params, true, $requestTimeout); + $status = $response['headers']['status-code']; + break; + case $status === 406: + $response = $this->call(self::METHOD_POST, $route, $headers, $params, true, $requestTimeout); + $status = $response['headers']['status-code']; + break; + default: + throw new \Exception($response['body']['message'], $status); + } + } catch (\Exception $e) { + throw new \Exception($e->getMessage(), $e->getCode()); + } + sleep(2); + } + + throw new Exception($response['body']['message'], 503); + } + + /** + * Call + * + * Make an API call + * + * @param string $method + * @param string $path + * @param array $params + * @param array $headers + * @param bool $decode + * @return array|string + * @throws Exception + */ + public function call(string $method, string $path = '', array $headers = [], array $params = [], bool $decode = true, int $timeout = 15) + { + $headers = array_merge($this->headers, $headers); + $ch = curl_init($this->endpoint . $path . (($method == self::METHOD_GET && !empty($params)) ? '?' . http_build_query($params) : '')); + $responseHeaders = []; + $responseStatus = -1; + $responseType = ''; + $responseBody = ''; + + switch ($headers['content-type']) { + case 'application/json': + $query = json_encode($params); + break; + + case 'multipart/form-data': + $query = $this->flatten($params); + break; + + default: + $query = http_build_query($params); + break; + } + + foreach ($headers as $i => $header) { + $headers[] = $i . ':' . $header; + unset($headers[$i]); + } + + curl_setopt($ch, CURLOPT_CUSTOMREQUEST, $method); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); + curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); + curl_setopt($ch, CURLOPT_HTTPHEADER, $headers); + curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 0); + curl_setopt($ch, CURLOPT_TIMEOUT, $timeout); + curl_setopt($ch, CURLOPT_HEADERFUNCTION, function ($curl, $header) use (&$responseHeaders) { + $len = strlen($header); + $header = explode(':', $header, 2); + + if (count($header) < 2) { // ignore invalid headers + return $len; + } + + $responseHeaders[strtolower(trim($header[0]))] = trim($header[1]); + + return $len; + }); + + if ($method != self::METHOD_GET) { + curl_setopt($ch, CURLOPT_POSTFIELDS, $query); + } + + // Allow self signed certificates + if ($this->selfSigned) { + curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false); + curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); + } + + $responseBody = curl_exec($ch); + $responseType = $responseHeaders['content-type'] ?? ''; + $responseStatus = curl_getinfo($ch, CURLINFO_HTTP_CODE); + + if ($decode) { + switch (substr($responseType, 0, strpos($responseType, ';'))) { + case 'application/json': + $json = json_decode($responseBody, true); + + if ($json === null) { + throw new Exception('Failed to parse response: ' . $responseBody); + } + + $responseBody = $json; + $json = null; + break; + } + } + + if ((curl_errno($ch)/* || 200 != $responseStatus*/)) { + throw new Exception(curl_error($ch) . ' with status code ' . $responseStatus, $responseStatus); + } + + curl_close($ch); + + $responseHeaders['status-code'] = $responseStatus; + + return [ + 'headers' => $responseHeaders, + 'body' => $responseBody + ]; + } + + /** + * Parse Cookie String + * + * @param string $cookie + * @return array + */ + public function parseCookie(string $cookie): array + { + $cookies = []; + + parse_str(strtr($cookie, array('&' => '%26', '+' => '%2B', ';' => '&')), $cookies); + + return $cookies; + } + + /** + * Flatten params array to PHP multiple format + * + * @param array $data + * @param string $prefix + * @return array + */ + protected function flatten(array $data, string $prefix = ''): array + { + $output = []; + + foreach ($data as $key => $value) { + $finalKey = $prefix ? "{$prefix}[{$key}]" : $key; + + if (is_array($value)) { + $output += $this->flatten($value, $finalKey); // @todo: handle name collision here if needed + } else { + $output[$finalKey] = $value; + } + } + + return $output; + } +} From 515c8c2e3212dd6e93ecccfed4a297a38cd6f035 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Thu, 9 Jun 2022 10:45:32 +0100 Subject: [PATCH 6/9] Run Linter --- app/executor.php | 1 - 1 file changed, 1 deletion(-) diff --git a/app/executor.php b/app/executor.php index 8df9f7090f..6e12b38f6b 100644 --- a/app/executor.php +++ b/app/executor.php @@ -177,7 +177,6 @@ App::post('/v1/runtimes') ->inject('response') ->action(function (string $runtimeId, string $source, string $destination, array $vars, array $commands, string $runtime, string $baseImage, string $entrypoint, bool $remove, string $workdir, $orchestrationPool, $activeRuntimes, Response $response) { if ($activeRuntimes->exists($runtimeId)) { - if ($activeRuntimes->get($runtimeId)['key'] == 'pending') { throw new \Exception('A runtime with the same ID is already being created. Attempt a execution soon.', 500); } From 98e615fa5f80508b518db2cb7fc767df77ee499f Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Thu, 9 Jun 2022 12:46:08 +0100 Subject: [PATCH 7/9] Improve Error handling --- app/executor.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/executor.php b/app/executor.php index 6e12b38f6b..acff22a318 100644 --- a/app/executor.php +++ b/app/executor.php @@ -343,6 +343,7 @@ App::post('/v1/runtimes') Console::success('Build Stage completed in ' . ($endTime - $startTime) . ' seconds'); } catch (Throwable $th) { Console::error('Build failed: ' . $th->getMessage() . $stdout); + throw new Exception($th->getMessage() . $stdout, 500); } finally { // Container cleanup @@ -350,11 +351,13 @@ App::post('/v1/runtimes') if (!empty($containerId)) { // If container properly created $orchestration->remove($containerId, true); + $activeRuntimes->del($runtimeId); } else { // If whole creation failed, but container might have been initialized try { // Try to remove with contaier name instead of ID $orchestration->remove($runtimeId, true); + $activeRuntimes->del($runtimeId); } catch (Throwable $th) { // If fails, means initialization also failed. // Contianer is not there, no need to remove From 44dc9346c2dc277d795f5e93ad7b8abda625a1c9 Mon Sep 17 00:00:00 2001 From: Everly Precia Suresh Date: Mon, 20 Jun 2022 09:57:18 +0000 Subject: [PATCH 8/9] fix oauth docs url for yammer --- app/config/providers.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/config/providers.php b/app/config/providers.php index c4f4f90fa7..83ad923a2a 100644 --- a/app/config/providers.php +++ b/app/config/providers.php @@ -273,7 +273,7 @@ return [ // Ordered by ABC. ], 'yammer' => [ 'name' => 'Yammer', - 'developers' => 'https://developer.yammer.com/docs/oauth-2', + 'developers' => 'https://docs.microsoft.com/en-us/rest/api/yammer/oauth-2/', 'icon' => 'icon-yammer', 'enabled' => true, 'sandbox' => false, From 11ad99cdcd406ea0862f83f095336316ec2d560d Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Mon, 20 Jun 2022 13:35:43 +0100 Subject: [PATCH 9/9] Update app/executor.php Co-authored-by: Christy Jacob --- app/executor.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/executor.php b/app/executor.php index acff22a318..01efe4bd59 100644 --- a/app/executor.php +++ b/app/executor.php @@ -177,7 +177,7 @@ App::post('/v1/runtimes') ->inject('response') ->action(function (string $runtimeId, string $source, string $destination, array $vars, array $commands, string $runtime, string $baseImage, string $entrypoint, bool $remove, string $workdir, $orchestrationPool, $activeRuntimes, Response $response) { if ($activeRuntimes->exists($runtimeId)) { - if ($activeRuntimes->get($runtimeId)['key'] == 'pending') { + if ($activeRuntimes->get($runtimeId)['status'] == 'pending') { throw new \Exception('A runtime with the same ID is already being created. Attempt a execution soon.', 500); }