From 47fddbe8e5a39efef70c8b674ee8e5cf65080140 Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Sun, 10 Mar 2024 21:33:57 +0100 Subject: [PATCH 1/6] fix(vcs): fix deployments stuck at processing Ensure project is passed in the event because the init hook isn't able to detect the project. Make sure the build is triggered for each function. Reset the build event so the shutdown hook doesn't trigger again. --- app/controllers/api/vcs.php | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/app/controllers/api/vcs.php b/app/controllers/api/vcs.php index ea30d6b11d..761fb4b350 100644 --- a/app/controllers/api/vcs.php +++ b/app/controllers/api/vcs.php @@ -238,11 +238,16 @@ $createGitDeployments = function (GitHub $github, string $providerInstallationId $queueForBuilds ->setType(BUILD_TYPE_DEPLOYMENT) ->setResource($function) - ->setDeployment($deployment); + ->setDeployment($deployment) + ->setProject($project); // set the project because it won't be set for git deployments + + $queueForBuilds->trigger(); // must trigger here so that we create a build for each function //TODO: Add event? } } + + $queueForBuilds->setType(''); // prevent shutdown hook from triggering again }; App::get('/v1/vcs/github/authorize') From 9d6595f85d001bd6bc5771e56024a7e720ec946e Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Sun, 10 Mar 2024 21:43:22 +0100 Subject: [PATCH 2/6] fix(vcs): prevent an error with one function deployment stopping others --- app/controllers/api/vcs.php | 355 ++++++++++++++++++------------------ 1 file changed, 182 insertions(+), 173 deletions(-) diff --git a/app/controllers/api/vcs.php b/app/controllers/api/vcs.php index 761fb4b350..5ad389fca4 100644 --- a/app/controllers/api/vcs.php +++ b/app/controllers/api/vcs.php @@ -41,213 +41,222 @@ use Utopia\VCS\Exception\RepositoryNotFound; use function Swoole\Coroutine\batch; $createGitDeployments = function (GitHub $github, string $providerInstallationId, array $repositories, string $providerBranch, string $providerBranchUrl, string $providerRepositoryName, string $providerRepositoryUrl, string $providerRepositoryOwner, string $providerCommitHash, string $providerCommitAuthor, string $providerCommitAuthorUrl, string $providerCommitMessage, string $providerCommitUrl, string $providerPullRequestId, bool $external, Database $dbForConsole, Build $queueForBuilds, callable $getProjectDB, Request $request) { + $errors = []; foreach ($repositories as $resource) { - $resourceType = $resource->getAttribute('resourceType'); + try { + $resourceType = $resource->getAttribute('resourceType'); - if ($resourceType === "function") { - $projectId = $resource->getAttribute('projectId'); - $project = Authorization::skip(fn () => $dbForConsole->getDocument('projects', $projectId)); - $dbForProject = $getProjectDB($project); + if ($resourceType === "function") { + $projectId = $resource->getAttribute('projectId'); + $project = Authorization::skip(fn () => $dbForConsole->getDocument('projects', $projectId)); + $dbForProject = $getProjectDB($project); - $functionId = $resource->getAttribute('resourceId'); - $function = Authorization::skip(fn () => $dbForProject->getDocument('functions', $functionId)); - $functionInternalId = $function->getInternalId(); + $functionId = $resource->getAttribute('resourceId'); + $function = Authorization::skip(fn () => $dbForProject->getDocument('functions', $functionId)); + $functionInternalId = $function->getInternalId(); - $deploymentId = ID::unique(); - $repositoryId = $resource->getId(); - $repositoryInternalId = $resource->getInternalId(); - $providerRepositoryId = $resource->getAttribute('providerRepositoryId'); - $installationId = $resource->getAttribute('installationId'); - $installationInternalId = $resource->getAttribute('installationInternalId'); - $productionBranch = $function->getAttribute('providerBranch'); - $activate = false; + $deploymentId = ID::unique(); + $repositoryId = $resource->getId(); + $repositoryInternalId = $resource->getInternalId(); + $providerRepositoryId = $resource->getAttribute('providerRepositoryId'); + $installationId = $resource->getAttribute('installationId'); + $installationInternalId = $resource->getAttribute('installationInternalId'); + $productionBranch = $function->getAttribute('providerBranch'); + $activate = false; - if ($providerBranch == $productionBranch && $external === false) { - $activate = true; - } + if ($providerBranch == $productionBranch && $external === false) { + $activate = true; + } + + $owner = $github->getOwnerName($providerInstallationId) ?? ''; + try { + $repositoryName = $github->getRepositoryName($providerRepositoryId) ?? ''; + if (empty($repositoryName)) { + throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); + } + } catch (RepositoryNotFound $e) { + throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); + } - $owner = $github->getOwnerName($providerInstallationId) ?? ''; - try { - $repositoryName = $github->getRepositoryName($providerRepositoryId) ?? ''; if (empty($repositoryName)) { throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); } - } catch (RepositoryNotFound $e) { - throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); - } - if (empty($repositoryName)) { - throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); - } + $isAuthorized = !$external; - $isAuthorized = !$external; - - if (!$isAuthorized && !empty($providerPullRequestId)) { - if (\in_array($providerPullRequestId, $resource->getAttribute('providerPullRequestIds', []))) { - $isAuthorized = true; - } - } - - $commentStatus = $isAuthorized ? 'waiting' : 'failed'; - - $authorizeUrl = $request->getProtocol() . '://' . $request->getHostname() . "/git/authorize-contributor?projectId={$projectId}&installationId={$installationId}&repositoryId={$repositoryId}&providerPullRequestId={$providerPullRequestId}"; - - $action = $isAuthorized ? ['type' => 'logs'] : ['type' => 'authorize', 'url' => $authorizeUrl]; - - $latestCommentId = ''; - - if (!empty($providerPullRequestId)) { - $latestComment = Authorization::skip(fn () => $dbForConsole->findOne('vcsComments', [ - Query::equal('providerRepositoryId', [$providerRepositoryId]), - Query::equal('providerPullRequestId', [$providerPullRequestId]), - Query::orderDesc('$createdAt'), - ])); - - if ($latestComment !== false && !$latestComment->isEmpty()) { - $latestCommentId = $latestComment->getAttribute('providerCommentId', ''); - $comment = new Comment(); - $comment->parseComment($github->getComment($owner, $repositoryName, $latestCommentId)); - $comment->addBuild($project, $function, $commentStatus, $deploymentId, $action); - - $latestCommentId = \strval($github->updateComment($owner, $repositoryName, $latestCommentId, $comment->generateComment())); - } else { - $comment = new Comment(); - $comment->addBuild($project, $function, $commentStatus, $deploymentId, $action); - $latestCommentId = \strval($github->createComment($owner, $repositoryName, $providerPullRequestId, $comment->generateComment())); - - if (!empty($latestCommentId)) { - $teamId = $project->getAttribute('teamId', ''); - - $latestComment = Authorization::skip(fn () => $dbForConsole->createDocument('vcsComments', new Document([ - '$id' => ID::unique(), - '$permissions' => [ - Permission::read(Role::team(ID::custom($teamId))), - Permission::update(Role::team(ID::custom($teamId), 'owner')), - Permission::update(Role::team(ID::custom($teamId), 'developer')), - Permission::delete(Role::team(ID::custom($teamId), 'owner')), - Permission::delete(Role::team(ID::custom($teamId), 'developer')), - ], - 'installationInternalId' => $installationInternalId, - 'installationId' => $installationId, - 'projectInternalId' => $project->getInternalId(), - 'projectId' => $project->getId(), - 'providerRepositoryId' => $providerRepositoryId, - 'providerBranch' => $providerBranch, - 'providerPullRequestId' => $providerPullRequestId, - 'providerCommentId' => $latestCommentId - ]))); + if (!$isAuthorized && !empty($providerPullRequestId)) { + if (\in_array($providerPullRequestId, $resource->getAttribute('providerPullRequestIds', []))) { + $isAuthorized = true; } } - } elseif (!empty($providerBranch)) { - $latestComments = Authorization::skip(fn () => $dbForConsole->find('vcsComments', [ - Query::equal('providerRepositoryId', [$providerRepositoryId]), - Query::equal('providerBranch', [$providerBranch]), - Query::orderDesc('$createdAt'), - ])); - foreach ($latestComments as $comment) { - $latestCommentId = $comment->getAttribute('providerCommentId', ''); - $comment = new Comment(); - $comment->parseComment($github->getComment($owner, $repositoryName, $latestCommentId)); - $comment->addBuild($project, $function, $commentStatus, $deploymentId, $action); + $commentStatus = $isAuthorized ? 'waiting' : 'failed'; - $latestCommentId = \strval($github->updateComment($owner, $repositoryName, $latestCommentId, $comment->generateComment())); + $authorizeUrl = $request->getProtocol() . '://' . $request->getHostname() . "/git/authorize-contributor?projectId={$projectId}&installationId={$installationId}&repositoryId={$repositoryId}&providerPullRequestId={$providerPullRequestId}"; + + $action = $isAuthorized ? ['type' => 'logs'] : ['type' => 'authorize', 'url' => $authorizeUrl]; + + $latestCommentId = ''; + + if (!empty($providerPullRequestId)) { + $latestComment = Authorization::skip(fn () => $dbForConsole->findOne('vcsComments', [ + Query::equal('providerRepositoryId', [$providerRepositoryId]), + Query::equal('providerPullRequestId', [$providerPullRequestId]), + Query::orderDesc('$createdAt'), + ])); + + if ($latestComment !== false && !$latestComment->isEmpty()) { + $latestCommentId = $latestComment->getAttribute('providerCommentId', ''); + $comment = new Comment(); + $comment->parseComment($github->getComment($owner, $repositoryName, $latestCommentId)); + $comment->addBuild($project, $function, $commentStatus, $deploymentId, $action); + + $latestCommentId = \strval($github->updateComment($owner, $repositoryName, $latestCommentId, $comment->generateComment())); + } else { + $comment = new Comment(); + $comment->addBuild($project, $function, $commentStatus, $deploymentId, $action); + $latestCommentId = \strval($github->createComment($owner, $repositoryName, $providerPullRequestId, $comment->generateComment())); + + if (!empty($latestCommentId)) { + $teamId = $project->getAttribute('teamId', ''); + + $latestComment = Authorization::skip(fn () => $dbForConsole->createDocument('vcsComments', new Document([ + '$id' => ID::unique(), + '$permissions' => [ + Permission::read(Role::team(ID::custom($teamId))), + Permission::update(Role::team(ID::custom($teamId), 'owner')), + Permission::update(Role::team(ID::custom($teamId), 'developer')), + Permission::delete(Role::team(ID::custom($teamId), 'owner')), + Permission::delete(Role::team(ID::custom($teamId), 'developer')), + ], + 'installationInternalId' => $installationInternalId, + 'installationId' => $installationId, + 'projectInternalId' => $project->getInternalId(), + 'projectId' => $project->getId(), + 'providerRepositoryId' => $providerRepositoryId, + 'providerBranch' => $providerBranch, + 'providerPullRequestId' => $providerPullRequestId, + 'providerCommentId' => $latestCommentId + ]))); + } + } + } elseif (!empty($providerBranch)) { + $latestComments = Authorization::skip(fn () => $dbForConsole->find('vcsComments', [ + Query::equal('providerRepositoryId', [$providerRepositoryId]), + Query::equal('providerBranch', [$providerBranch]), + Query::orderDesc('$createdAt'), + ])); + + foreach ($latestComments as $comment) { + $latestCommentId = $comment->getAttribute('providerCommentId', ''); + $comment = new Comment(); + $comment->parseComment($github->getComment($owner, $repositoryName, $latestCommentId)); + $comment->addBuild($project, $function, $commentStatus, $deploymentId, $action); + + $latestCommentId = \strval($github->updateComment($owner, $repositoryName, $latestCommentId, $comment->generateComment())); + } } - } - if (!$isAuthorized) { - $functionName = $function->getAttribute('name'); - $projectName = $project->getAttribute('name'); - $name = "{$functionName} ({$projectName})"; - $message = 'Authorization required for external contributor.'; + if (!$isAuthorized) { + $functionName = $function->getAttribute('name'); + $projectName = $project->getAttribute('name'); + $name = "{$functionName} ({$projectName})"; + $message = 'Authorization required for external contributor.'; - $providerRepositoryId = $resource->getAttribute('providerRepositoryId'); - try { - $repositoryName = $github->getRepositoryName($providerRepositoryId) ?? ''; - if (empty($repositoryName)) { + $providerRepositoryId = $resource->getAttribute('providerRepositoryId'); + try { + $repositoryName = $github->getRepositoryName($providerRepositoryId) ?? ''; + if (empty($repositoryName)) { + throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); + } + } catch (RepositoryNotFound $e) { throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); } - } catch (RepositoryNotFound $e) { - throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); + $owner = $github->getOwnerName($providerInstallationId); + $github->updateCommitStatus($repositoryName, $providerCommitHash, $owner, 'failure', $message, $authorizeUrl, $name); + continue; } - $owner = $github->getOwnerName($providerInstallationId); - $github->updateCommitStatus($repositoryName, $providerCommitHash, $owner, 'failure', $message, $authorizeUrl, $name); - continue; - } - if ($external) { - $pullRequestResponse = $github->getPullRequest($owner, $repositoryName, $providerPullRequestId); - $providerRepositoryName = $pullRequestResponse['head']['repo']['owner']['login']; - $providerRepositoryOwner = $pullRequestResponse['head']['repo']['name']; - } + if ($external) { + $pullRequestResponse = $github->getPullRequest($owner, $repositoryName, $providerPullRequestId); + $providerRepositoryName = $pullRequestResponse['head']['repo']['owner']['login']; + $providerRepositoryOwner = $pullRequestResponse['head']['repo']['name']; + } - $deployment = $dbForProject->createDocument('deployments', new Document([ - '$id' => $deploymentId, - '$permissions' => [ - Permission::read(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), - ], - 'resourceId' => $functionId, - 'resourceInternalId' => $functionInternalId, - 'resourceType' => 'functions', - 'entrypoint' => $function->getAttribute('entrypoint'), - 'commands' => $function->getAttribute('commands'), - 'type' => 'vcs', - 'installationId' => $installationId, - 'installationInternalId' => $installationInternalId, - 'providerRepositoryId' => $providerRepositoryId, - 'repositoryId' => $repositoryId, - 'repositoryInternalId' => $repositoryInternalId, - 'providerBranchUrl' => $providerBranchUrl, - 'providerRepositoryName' => $providerRepositoryName, - 'providerRepositoryOwner' => $providerRepositoryOwner, - 'providerRepositoryUrl' => $providerRepositoryUrl, - 'providerCommitHash' => $providerCommitHash, - 'providerCommitAuthorUrl' => $providerCommitAuthorUrl, - 'providerCommitAuthor' => $providerCommitAuthor, - 'providerCommitMessage' => $providerCommitMessage, - 'providerCommitUrl' => $providerCommitUrl, - 'providerCommentId' => \strval($latestCommentId), - 'providerBranch' => $providerBranch, - 'search' => implode(' ', [$deploymentId, $function->getAttribute('entrypoint')]), - 'activate' => $activate, - ])); + $deployment = $dbForProject->createDocument('deployments', new Document([ + '$id' => $deploymentId, + '$permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + 'resourceId' => $functionId, + 'resourceInternalId' => $functionInternalId, + 'resourceType' => 'functions', + 'entrypoint' => $function->getAttribute('entrypoint'), + 'commands' => $function->getAttribute('commands'), + 'type' => 'vcs', + 'installationId' => $installationId, + 'installationInternalId' => $installationInternalId, + 'providerRepositoryId' => $providerRepositoryId, + 'repositoryId' => $repositoryId, + 'repositoryInternalId' => $repositoryInternalId, + 'providerBranchUrl' => $providerBranchUrl, + 'providerRepositoryName' => $providerRepositoryName, + 'providerRepositoryOwner' => $providerRepositoryOwner, + 'providerRepositoryUrl' => $providerRepositoryUrl, + 'providerCommitHash' => $providerCommitHash, + 'providerCommitAuthorUrl' => $providerCommitAuthorUrl, + 'providerCommitAuthor' => $providerCommitAuthor, + 'providerCommitMessage' => $providerCommitMessage, + 'providerCommitUrl' => $providerCommitUrl, + 'providerCommentId' => \strval($latestCommentId), + 'providerBranch' => $providerBranch, + 'search' => implode(' ', [$deploymentId, $function->getAttribute('entrypoint')]), + 'activate' => $activate, + ])); - if (!empty($providerCommitHash) && $function->getAttribute('providerSilentMode', false) === false) { - $functionName = $function->getAttribute('name'); - $projectName = $project->getAttribute('name'); - $name = "{$functionName} ({$projectName})"; - $message = 'Starting...'; + if (!empty($providerCommitHash) && $function->getAttribute('providerSilentMode', false) === false) { + $functionName = $function->getAttribute('name'); + $projectName = $project->getAttribute('name'); + $name = "{$functionName} ({$projectName})"; + $message = 'Starting...'; - $providerRepositoryId = $resource->getAttribute('providerRepositoryId'); - try { - $repositoryName = $github->getRepositoryName($providerRepositoryId) ?? ''; - if (empty($repositoryName)) { + $providerRepositoryId = $resource->getAttribute('providerRepositoryId'); + try { + $repositoryName = $github->getRepositoryName($providerRepositoryId) ?? ''; + if (empty($repositoryName)) { + throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); + } + } catch (RepositoryNotFound $e) { throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); } - } catch (RepositoryNotFound $e) { - throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); + $owner = $github->getOwnerName($providerInstallationId); + + $providerTargetUrl = $request->getProtocol() . '://' . $request->getHostname() . "/console/project-$projectId/functions/function-$functionId"; + $github->updateCommitStatus($repositoryName, $providerCommitHash, $owner, 'pending', $message, $providerTargetUrl, $name); } - $owner = $github->getOwnerName($providerInstallationId); - $providerTargetUrl = $request->getProtocol() . '://' . $request->getHostname() . "/console/project-$projectId/functions/function-$functionId"; - $github->updateCommitStatus($repositoryName, $providerCommitHash, $owner, 'pending', $message, $providerTargetUrl, $name); + $queueForBuilds + ->setType(BUILD_TYPE_DEPLOYMENT) + ->setResource($function) + ->setDeployment($deployment) + ->setProject($project); // set the project because it won't be set for git deployments + + $queueForBuilds->trigger(); // must trigger here so that we create a build for each function + + //TODO: Add event? } - - $queueForBuilds - ->setType(BUILD_TYPE_DEPLOYMENT) - ->setResource($function) - ->setDeployment($deployment) - ->setProject($project); // set the project because it won't be set for git deployments - - $queueForBuilds->trigger(); // must trigger here so that we create a build for each function - - //TODO: Add event? + } catch (Throwable $e) { + $errors[] = $e->getMessage(); } } $queueForBuilds->setType(''); // prevent shutdown hook from triggering again + + if (!empty($errors)) { + throw new Exception(Exception::GENERAL_UNKNOWN, \implode("\n", $errors)); + } }; App::get('/v1/vcs/github/authorize') From 8e348dbd9288e1cbcc31068d84057c8d89c37387 Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Sun, 10 Mar 2024 21:53:57 +0100 Subject: [PATCH 3/6] refactor(vcs): reduce nested code --- app/controllers/api/vcs.php | 368 ++++++++++++++++++------------------ 1 file changed, 185 insertions(+), 183 deletions(-) diff --git a/app/controllers/api/vcs.php b/app/controllers/api/vcs.php index 5ad389fca4..bc62e87aa9 100644 --- a/app/controllers/api/vcs.php +++ b/app/controllers/api/vcs.php @@ -46,29 +46,127 @@ $createGitDeployments = function (GitHub $github, string $providerInstallationId try { $resourceType = $resource->getAttribute('resourceType'); - if ($resourceType === "function") { - $projectId = $resource->getAttribute('projectId'); - $project = Authorization::skip(fn () => $dbForConsole->getDocument('projects', $projectId)); - $dbForProject = $getProjectDB($project); + if ($resourceType !== "function") { + continue; + } - $functionId = $resource->getAttribute('resourceId'); - $function = Authorization::skip(fn () => $dbForProject->getDocument('functions', $functionId)); - $functionInternalId = $function->getInternalId(); + $projectId = $resource->getAttribute('projectId'); + $project = Authorization::skip(fn () => $dbForConsole->getDocument('projects', $projectId)); + $dbForProject = $getProjectDB($project); - $deploymentId = ID::unique(); - $repositoryId = $resource->getId(); - $repositoryInternalId = $resource->getInternalId(); - $providerRepositoryId = $resource->getAttribute('providerRepositoryId'); - $installationId = $resource->getAttribute('installationId'); - $installationInternalId = $resource->getAttribute('installationInternalId'); - $productionBranch = $function->getAttribute('providerBranch'); - $activate = false; + $functionId = $resource->getAttribute('resourceId'); + $function = Authorization::skip(fn () => $dbForProject->getDocument('functions', $functionId)); + $functionInternalId = $function->getInternalId(); - if ($providerBranch == $productionBranch && $external === false) { - $activate = true; + $deploymentId = ID::unique(); + $repositoryId = $resource->getId(); + $repositoryInternalId = $resource->getInternalId(); + $providerRepositoryId = $resource->getAttribute('providerRepositoryId'); + $installationId = $resource->getAttribute('installationId'); + $installationInternalId = $resource->getAttribute('installationInternalId'); + $productionBranch = $function->getAttribute('providerBranch'); + $activate = false; + + if ($providerBranch == $productionBranch && $external === false) { + $activate = true; + } + + $owner = $github->getOwnerName($providerInstallationId) ?? ''; + try { + $repositoryName = $github->getRepositoryName($providerRepositoryId) ?? ''; + if (empty($repositoryName)) { + throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); } + } catch (RepositoryNotFound $e) { + throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); + } - $owner = $github->getOwnerName($providerInstallationId) ?? ''; + if (empty($repositoryName)) { + throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); + } + + $isAuthorized = !$external; + + if (!$isAuthorized && !empty($providerPullRequestId)) { + if (\in_array($providerPullRequestId, $resource->getAttribute('providerPullRequestIds', []))) { + $isAuthorized = true; + } + } + + $commentStatus = $isAuthorized ? 'waiting' : 'failed'; + + $authorizeUrl = $request->getProtocol() . '://' . $request->getHostname() . "/git/authorize-contributor?projectId={$projectId}&installationId={$installationId}&repositoryId={$repositoryId}&providerPullRequestId={$providerPullRequestId}"; + + $action = $isAuthorized ? ['type' => 'logs'] : ['type' => 'authorize', 'url' => $authorizeUrl]; + + $latestCommentId = ''; + + if (!empty($providerPullRequestId)) { + $latestComment = Authorization::skip(fn () => $dbForConsole->findOne('vcsComments', [ + Query::equal('providerRepositoryId', [$providerRepositoryId]), + Query::equal('providerPullRequestId', [$providerPullRequestId]), + Query::orderDesc('$createdAt'), + ])); + + if ($latestComment !== false && !$latestComment->isEmpty()) { + $latestCommentId = $latestComment->getAttribute('providerCommentId', ''); + $comment = new Comment(); + $comment->parseComment($github->getComment($owner, $repositoryName, $latestCommentId)); + $comment->addBuild($project, $function, $commentStatus, $deploymentId, $action); + + $latestCommentId = \strval($github->updateComment($owner, $repositoryName, $latestCommentId, $comment->generateComment())); + } else { + $comment = new Comment(); + $comment->addBuild($project, $function, $commentStatus, $deploymentId, $action); + $latestCommentId = \strval($github->createComment($owner, $repositoryName, $providerPullRequestId, $comment->generateComment())); + + if (!empty($latestCommentId)) { + $teamId = $project->getAttribute('teamId', ''); + + $latestComment = Authorization::skip(fn () => $dbForConsole->createDocument('vcsComments', new Document([ + '$id' => ID::unique(), + '$permissions' => [ + Permission::read(Role::team(ID::custom($teamId))), + Permission::update(Role::team(ID::custom($teamId), 'owner')), + Permission::update(Role::team(ID::custom($teamId), 'developer')), + Permission::delete(Role::team(ID::custom($teamId), 'owner')), + Permission::delete(Role::team(ID::custom($teamId), 'developer')), + ], + 'installationInternalId' => $installationInternalId, + 'installationId' => $installationId, + 'projectInternalId' => $project->getInternalId(), + 'projectId' => $project->getId(), + 'providerRepositoryId' => $providerRepositoryId, + 'providerBranch' => $providerBranch, + 'providerPullRequestId' => $providerPullRequestId, + 'providerCommentId' => $latestCommentId + ]))); + } + } + } elseif (!empty($providerBranch)) { + $latestComments = Authorization::skip(fn () => $dbForConsole->find('vcsComments', [ + Query::equal('providerRepositoryId', [$providerRepositoryId]), + Query::equal('providerBranch', [$providerBranch]), + Query::orderDesc('$createdAt'), + ])); + + foreach ($latestComments as $comment) { + $latestCommentId = $comment->getAttribute('providerCommentId', ''); + $comment = new Comment(); + $comment->parseComment($github->getComment($owner, $repositoryName, $latestCommentId)); + $comment->addBuild($project, $function, $commentStatus, $deploymentId, $action); + + $latestCommentId = \strval($github->updateComment($owner, $repositoryName, $latestCommentId, $comment->generateComment())); + } + } + + if (!$isAuthorized) { + $functionName = $function->getAttribute('name'); + $projectName = $project->getAttribute('name'); + $name = "{$functionName} ({$projectName})"; + $message = 'Authorization required for external contributor.'; + + $providerRepositoryId = $resource->getAttribute('providerRepositoryId'); try { $repositoryName = $github->getRepositoryName($providerRepositoryId) ?? ''; if (empty($repositoryName)) { @@ -77,176 +175,80 @@ $createGitDeployments = function (GitHub $github, string $providerInstallationId } catch (RepositoryNotFound $e) { throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); } + $owner = $github->getOwnerName($providerInstallationId); + $github->updateCommitStatus($repositoryName, $providerCommitHash, $owner, 'failure', $message, $authorizeUrl, $name); + continue; + } - if (empty($repositoryName)) { + if ($external) { + $pullRequestResponse = $github->getPullRequest($owner, $repositoryName, $providerPullRequestId); + $providerRepositoryName = $pullRequestResponse['head']['repo']['owner']['login']; + $providerRepositoryOwner = $pullRequestResponse['head']['repo']['name']; + } + + $deployment = $dbForProject->createDocument('deployments', new Document([ + '$id' => $deploymentId, + '$permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + 'resourceId' => $functionId, + 'resourceInternalId' => $functionInternalId, + 'resourceType' => 'functions', + 'entrypoint' => $function->getAttribute('entrypoint'), + 'commands' => $function->getAttribute('commands'), + 'type' => 'vcs', + 'installationId' => $installationId, + 'installationInternalId' => $installationInternalId, + 'providerRepositoryId' => $providerRepositoryId, + 'repositoryId' => $repositoryId, + 'repositoryInternalId' => $repositoryInternalId, + 'providerBranchUrl' => $providerBranchUrl, + 'providerRepositoryName' => $providerRepositoryName, + 'providerRepositoryOwner' => $providerRepositoryOwner, + 'providerRepositoryUrl' => $providerRepositoryUrl, + 'providerCommitHash' => $providerCommitHash, + 'providerCommitAuthorUrl' => $providerCommitAuthorUrl, + 'providerCommitAuthor' => $providerCommitAuthor, + 'providerCommitMessage' => $providerCommitMessage, + 'providerCommitUrl' => $providerCommitUrl, + 'providerCommentId' => \strval($latestCommentId), + 'providerBranch' => $providerBranch, + 'search' => implode(' ', [$deploymentId, $function->getAttribute('entrypoint')]), + 'activate' => $activate, + ])); + + if (!empty($providerCommitHash) && $function->getAttribute('providerSilentMode', false) === false) { + $functionName = $function->getAttribute('name'); + $projectName = $project->getAttribute('name'); + $name = "{$functionName} ({$projectName})"; + $message = 'Starting...'; + + $providerRepositoryId = $resource->getAttribute('providerRepositoryId'); + try { + $repositoryName = $github->getRepositoryName($providerRepositoryId) ?? ''; + if (empty($repositoryName)) { + throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); + } + } catch (RepositoryNotFound $e) { throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); } + $owner = $github->getOwnerName($providerInstallationId); - $isAuthorized = !$external; - - if (!$isAuthorized && !empty($providerPullRequestId)) { - if (\in_array($providerPullRequestId, $resource->getAttribute('providerPullRequestIds', []))) { - $isAuthorized = true; - } - } - - $commentStatus = $isAuthorized ? 'waiting' : 'failed'; - - $authorizeUrl = $request->getProtocol() . '://' . $request->getHostname() . "/git/authorize-contributor?projectId={$projectId}&installationId={$installationId}&repositoryId={$repositoryId}&providerPullRequestId={$providerPullRequestId}"; - - $action = $isAuthorized ? ['type' => 'logs'] : ['type' => 'authorize', 'url' => $authorizeUrl]; - - $latestCommentId = ''; - - if (!empty($providerPullRequestId)) { - $latestComment = Authorization::skip(fn () => $dbForConsole->findOne('vcsComments', [ - Query::equal('providerRepositoryId', [$providerRepositoryId]), - Query::equal('providerPullRequestId', [$providerPullRequestId]), - Query::orderDesc('$createdAt'), - ])); - - if ($latestComment !== false && !$latestComment->isEmpty()) { - $latestCommentId = $latestComment->getAttribute('providerCommentId', ''); - $comment = new Comment(); - $comment->parseComment($github->getComment($owner, $repositoryName, $latestCommentId)); - $comment->addBuild($project, $function, $commentStatus, $deploymentId, $action); - - $latestCommentId = \strval($github->updateComment($owner, $repositoryName, $latestCommentId, $comment->generateComment())); - } else { - $comment = new Comment(); - $comment->addBuild($project, $function, $commentStatus, $deploymentId, $action); - $latestCommentId = \strval($github->createComment($owner, $repositoryName, $providerPullRequestId, $comment->generateComment())); - - if (!empty($latestCommentId)) { - $teamId = $project->getAttribute('teamId', ''); - - $latestComment = Authorization::skip(fn () => $dbForConsole->createDocument('vcsComments', new Document([ - '$id' => ID::unique(), - '$permissions' => [ - Permission::read(Role::team(ID::custom($teamId))), - Permission::update(Role::team(ID::custom($teamId), 'owner')), - Permission::update(Role::team(ID::custom($teamId), 'developer')), - Permission::delete(Role::team(ID::custom($teamId), 'owner')), - Permission::delete(Role::team(ID::custom($teamId), 'developer')), - ], - 'installationInternalId' => $installationInternalId, - 'installationId' => $installationId, - 'projectInternalId' => $project->getInternalId(), - 'projectId' => $project->getId(), - 'providerRepositoryId' => $providerRepositoryId, - 'providerBranch' => $providerBranch, - 'providerPullRequestId' => $providerPullRequestId, - 'providerCommentId' => $latestCommentId - ]))); - } - } - } elseif (!empty($providerBranch)) { - $latestComments = Authorization::skip(fn () => $dbForConsole->find('vcsComments', [ - Query::equal('providerRepositoryId', [$providerRepositoryId]), - Query::equal('providerBranch', [$providerBranch]), - Query::orderDesc('$createdAt'), - ])); - - foreach ($latestComments as $comment) { - $latestCommentId = $comment->getAttribute('providerCommentId', ''); - $comment = new Comment(); - $comment->parseComment($github->getComment($owner, $repositoryName, $latestCommentId)); - $comment->addBuild($project, $function, $commentStatus, $deploymentId, $action); - - $latestCommentId = \strval($github->updateComment($owner, $repositoryName, $latestCommentId, $comment->generateComment())); - } - } - - if (!$isAuthorized) { - $functionName = $function->getAttribute('name'); - $projectName = $project->getAttribute('name'); - $name = "{$functionName} ({$projectName})"; - $message = 'Authorization required for external contributor.'; - - $providerRepositoryId = $resource->getAttribute('providerRepositoryId'); - try { - $repositoryName = $github->getRepositoryName($providerRepositoryId) ?? ''; - if (empty($repositoryName)) { - throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); - } - } catch (RepositoryNotFound $e) { - throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); - } - $owner = $github->getOwnerName($providerInstallationId); - $github->updateCommitStatus($repositoryName, $providerCommitHash, $owner, 'failure', $message, $authorizeUrl, $name); - continue; - } - - if ($external) { - $pullRequestResponse = $github->getPullRequest($owner, $repositoryName, $providerPullRequestId); - $providerRepositoryName = $pullRequestResponse['head']['repo']['owner']['login']; - $providerRepositoryOwner = $pullRequestResponse['head']['repo']['name']; - } - - $deployment = $dbForProject->createDocument('deployments', new Document([ - '$id' => $deploymentId, - '$permissions' => [ - Permission::read(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), - ], - 'resourceId' => $functionId, - 'resourceInternalId' => $functionInternalId, - 'resourceType' => 'functions', - 'entrypoint' => $function->getAttribute('entrypoint'), - 'commands' => $function->getAttribute('commands'), - 'type' => 'vcs', - 'installationId' => $installationId, - 'installationInternalId' => $installationInternalId, - 'providerRepositoryId' => $providerRepositoryId, - 'repositoryId' => $repositoryId, - 'repositoryInternalId' => $repositoryInternalId, - 'providerBranchUrl' => $providerBranchUrl, - 'providerRepositoryName' => $providerRepositoryName, - 'providerRepositoryOwner' => $providerRepositoryOwner, - 'providerRepositoryUrl' => $providerRepositoryUrl, - 'providerCommitHash' => $providerCommitHash, - 'providerCommitAuthorUrl' => $providerCommitAuthorUrl, - 'providerCommitAuthor' => $providerCommitAuthor, - 'providerCommitMessage' => $providerCommitMessage, - 'providerCommitUrl' => $providerCommitUrl, - 'providerCommentId' => \strval($latestCommentId), - 'providerBranch' => $providerBranch, - 'search' => implode(' ', [$deploymentId, $function->getAttribute('entrypoint')]), - 'activate' => $activate, - ])); - - if (!empty($providerCommitHash) && $function->getAttribute('providerSilentMode', false) === false) { - $functionName = $function->getAttribute('name'); - $projectName = $project->getAttribute('name'); - $name = "{$functionName} ({$projectName})"; - $message = 'Starting...'; - - $providerRepositoryId = $resource->getAttribute('providerRepositoryId'); - try { - $repositoryName = $github->getRepositoryName($providerRepositoryId) ?? ''; - if (empty($repositoryName)) { - throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); - } - } catch (RepositoryNotFound $e) { - throw new Exception(Exception::PROVIDER_REPOSITORY_NOT_FOUND); - } - $owner = $github->getOwnerName($providerInstallationId); - - $providerTargetUrl = $request->getProtocol() . '://' . $request->getHostname() . "/console/project-$projectId/functions/function-$functionId"; - $github->updateCommitStatus($repositoryName, $providerCommitHash, $owner, 'pending', $message, $providerTargetUrl, $name); - } - - $queueForBuilds - ->setType(BUILD_TYPE_DEPLOYMENT) - ->setResource($function) - ->setDeployment($deployment) - ->setProject($project); // set the project because it won't be set for git deployments - - $queueForBuilds->trigger(); // must trigger here so that we create a build for each function - - //TODO: Add event? + $providerTargetUrl = $request->getProtocol() . '://' . $request->getHostname() . "/console/project-$projectId/functions/function-$functionId"; + $github->updateCommitStatus($repositoryName, $providerCommitHash, $owner, 'pending', $message, $providerTargetUrl, $name); } + + $queueForBuilds + ->setType(BUILD_TYPE_DEPLOYMENT) + ->setResource($function) + ->setDeployment($deployment) + ->setProject($project); // set the project because it won't be set for git deployments + + $queueForBuilds->trigger(); // must trigger here so that we create a build for each function + + //TODO: Add event? } catch (Throwable $e) { $errors[] = $e->getMessage(); } From b7be370a5372b11f8c290a44ffb54949202ff9d3 Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Sun, 10 Mar 2024 23:13:55 +0100 Subject: [PATCH 4/6] fix(builds): use standard Exception --- src/Appwrite/Platform/Workers/Builds.php | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Appwrite/Platform/Workers/Builds.php b/src/Appwrite/Platform/Workers/Builds.php index b9f02a8f67..31bf961c30 100644 --- a/src/Appwrite/Platform/Workers/Builds.php +++ b/src/Appwrite/Platform/Workers/Builds.php @@ -73,7 +73,7 @@ class Builds extends Action $payload = $message->getPayload() ?? []; if (empty($payload)) { - throw new Exception('Missing payload'); + throw new \Exception('Missing payload'); } $type = $payload['type'] ?? ''; @@ -124,7 +124,7 @@ class Builds extends Action $function = $dbForProject->getDocument('functions', $functionId); if ($function->isEmpty()) { - throw new Exception('Function not found', 404); + throw new \Exception('Function not found', 404); } $deploymentId = $deployment->getId(); @@ -132,11 +132,11 @@ class Builds extends Action $deployment = $dbForProject->getDocument('deployments', $deploymentId); if ($deployment->isEmpty()) { - throw new Exception('Deployment not found', 404); + throw new \Exception('Deployment not found', 404); } if (empty($deployment->getAttribute('entrypoint', ''))) { - throw new Exception('Entrypoint for your Appwrite Function is missing. Please specify it when making deployment or update the entrypoint under your function\'s "Settings" > "Configuration" > "Entrypoint".', 500); + throw new \Exception('Entrypoint for your Appwrite Function is missing. Please specify it when making deployment or update the entrypoint under your function\'s "Settings" > "Configuration" > "Entrypoint".', 500); } $version = $function->getAttribute('version', 'v2'); @@ -144,7 +144,7 @@ class Builds extends Action $key = $function->getAttribute('runtime'); $runtime = $runtimes[$key] ?? null; if (\is_null($runtime)) { - throw new Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported'); + throw new \Exception('Runtime "' . $function->getAttribute('runtime', '') . '" is not supported'); } // Realtime preparation @@ -306,7 +306,7 @@ class Builds extends Action $directorySize = $localDevice->getDirectorySize($tmpDirectory); $functionsSizeLimit = (int) App::getEnv('_APP_FUNCTIONS_SIZE_LIMIT', '30000000'); if ($directorySize > $functionsSizeLimit) { - throw new Exception('Repository directory size should be less than ' . number_format($functionsSizeLimit / 1048576, 2) . ' MBs.'); + throw new \Exception('Repository directory size should be less than ' . number_format($functionsSizeLimit / 1048576, 2) . ' MBs.'); } Console::execute('tar --exclude code.tar.gz -czf ' . $tmpPathFile . ' -C /tmp/builds/' . \escapeshellcmd($buildId) . '/code' . (empty($rootDirectory) ? '' : '/' . $rootDirectory) . ' .', '', $stdout, $stderr); @@ -431,7 +431,7 @@ class Builds extends Action $build = $dbForProject->getDocument('builds', $build->getId()); if ($build->isEmpty()) { - throw new Exception('Build not found', 404); + throw new \Exception('Build not found', 404); } $build = $build->setAttribute('logs', $build->getAttribute('logs', '') . $logs); From b9b891a90b49a39ed39294dba225b39afbd6f2c0 Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Sun, 10 Mar 2024 23:16:29 +0100 Subject: [PATCH 5/6] fix(builds): fix float to int warning Implicit conversion of float to int is deprecated and emits: Deprecated: Implicit conversion from float N to int loses precision Use floor to truncate. --- src/Appwrite/Platform/Workers/Builds.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Appwrite/Platform/Workers/Builds.php b/src/Appwrite/Platform/Workers/Builds.php index 31bf961c30..42cd910a1f 100644 --- a/src/Appwrite/Platform/Workers/Builds.php +++ b/src/Appwrite/Platform/Workers/Builds.php @@ -472,7 +472,7 @@ class Builds extends Action $durationEnd = \microtime(true); /** Update the build document */ - $build->setAttribute('startTime', DateTime::format((new \DateTime())->setTimestamp($response['startTime']))); + $build->setAttribute('startTime', DateTime::format((new \DateTime())->setTimestamp(floor($response['startTime'])))); $build->setAttribute('endTime', $endTime); $build->setAttribute('duration', \intval(\ceil($durationEnd - $durationStart))); $build->setAttribute('status', 'ready'); From b19efb619b9ea5bd911838bc21bd7840127e9fb3 Mon Sep 17 00:00:00 2001 From: Steven Nguyen Date: Mon, 11 Mar 2024 11:17:40 +0100 Subject: [PATCH 6/6] feat(events): update build event reset to reset everything Instead of using setType('') to prevent events from triggering, it makes more sense to use reset(). However, reset() didn't properly reset type, resource, deployment, or template. This change makes sure to reset all those private variables. --- app/controllers/api/vcs.php | 2 +- src/Appwrite/Event/Build.php | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/app/controllers/api/vcs.php b/app/controllers/api/vcs.php index bc62e87aa9..c041155ef7 100644 --- a/app/controllers/api/vcs.php +++ b/app/controllers/api/vcs.php @@ -254,7 +254,7 @@ $createGitDeployments = function (GitHub $github, string $providerInstallationId } } - $queueForBuilds->setType(''); // prevent shutdown hook from triggering again + $queueForBuilds->reset(); // prevent shutdown hook from triggering again if (!empty($errors)) { throw new Exception(Exception::GENERAL_UNKNOWN, \implode("\n", $errors)); diff --git a/src/Appwrite/Event/Build.php b/src/Appwrite/Event/Build.php index 496db87d64..b8cb62a6f8 100644 --- a/src/Appwrite/Event/Build.php +++ b/src/Appwrite/Event/Build.php @@ -122,4 +122,20 @@ class Build extends Event 'template' => $this->template ]); } + + /** + * Resets event. + * + * @return self + */ + public function reset(): self + { + $this->type = ''; + $this->resource = null; + $this->deployment = null; + $this->template = null; + parent::reset(); + + return $this; + } }