feat(applicationset): reuse repo-creds for an existing GitHub App (#10092)

Closes #10079

Signed-off-by: Noah Perks Sloan <noah_sloan@securityjourney.com>

Signed-off-by: Noah Perks Sloan <noah_sloan@securityjourney.com>
This commit is contained in:
noah 2022-08-19 15:57:49 -05:00 committed by GitHub
parent d545198493
commit 506bd3b282
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
21 changed files with 337 additions and 23 deletions

View file

@ -9,10 +9,11 @@ import (
corev1 "k8s.io/api/core/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
"github.com/gosimple/slug"
"github.com/argoproj/argo-cd/v2/applicationset/services/pull_request"
pullrequest "github.com/argoproj/argo-cd/v2/applicationset/services/pull_request"
argoprojiov1alpha1 "github.com/argoproj/argo-cd/v2/pkg/apis/applicationset/v1alpha1"
"github.com/gosimple/slug"
)
var _ Generator = (*PullRequestGenerator)(nil)
@ -24,11 +25,13 @@ const (
type PullRequestGenerator struct {
client client.Client
selectServiceProviderFunc func(context.Context, *argoprojiov1alpha1.PullRequestGenerator, *argoprojiov1alpha1.ApplicationSet) (pullrequest.PullRequestService, error)
auth SCMAuthProviders
}
func NewPullRequestGenerator(client client.Client) Generator {
func NewPullRequestGenerator(client client.Client, auth SCMAuthProviders) Generator {
g := &PullRequestGenerator{
client: client,
auth: auth,
}
g.selectServiceProviderFunc = g.selectServiceProvider
return g
@ -101,12 +104,7 @@ func (g *PullRequestGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha
// selectServiceProvider selects the provider to get pull requests from the configuration
func (g *PullRequestGenerator) selectServiceProvider(ctx context.Context, generatorConfig *argoprojiov1alpha1.PullRequestGenerator, applicationSetInfo *argoprojiov1alpha1.ApplicationSet) (pullrequest.PullRequestService, error) {
if generatorConfig.Github != nil {
providerConfig := generatorConfig.Github
token, err := g.getSecretRef(ctx, providerConfig.TokenRef, applicationSetInfo.Namespace)
if err != nil {
return nil, fmt.Errorf("error fetching Secret token: %v", err)
}
return pullrequest.NewGithubService(ctx, token, providerConfig.API, providerConfig.Owner, providerConfig.Repo, providerConfig.Labels)
return g.github(ctx, generatorConfig.Github, applicationSetInfo)
}
if generatorConfig.GitLab != nil {
providerConfig := generatorConfig.GitLab
@ -139,6 +137,24 @@ func (g *PullRequestGenerator) selectServiceProvider(ctx context.Context, genera
return nil, fmt.Errorf("no Pull Request provider implementation configured")
}
func (g *PullRequestGenerator) github(ctx context.Context, cfg *argoprojiov1alpha1.PullRequestGeneratorGithub, applicationSetInfo *argoprojiov1alpha1.ApplicationSet) (pullrequest.PullRequestService, error) {
// use an app if it was configured
if cfg.AppSecretName != "" {
auth, err := g.auth.GitHubApps.GetAuthSecret(ctx, cfg.AppSecretName)
if err != nil {
return nil, fmt.Errorf("error getting GitHub App secret: %v", err)
}
return pullrequest.NewGithubAppService(*auth, cfg.API, cfg.Owner, cfg.Repo, cfg.Labels)
}
// always default to token, even if not set (public access)
token, err := g.getSecretRef(ctx, cfg.TokenRef, applicationSetInfo.Namespace)
if err != nil {
return nil, fmt.Errorf("error fetching Secret token: %v", err)
}
return pullrequest.NewGithubService(ctx, token, cfg.API, cfg.Owner, cfg.Repo, cfg.Labels)
}
// getSecretRef gets the value of the key for the specified Secret resource.
func (g *PullRequestGenerator) getSecretRef(ctx context.Context, ref *argoprojiov1alpha1.SecretRef, namespace string) (string, error) {
if ref == nil {

View file

@ -9,6 +9,7 @@ import (
corev1 "k8s.io/api/core/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
"github.com/argoproj/argo-cd/v2/applicationset/services/github_app_auth"
"github.com/argoproj/argo-cd/v2/applicationset/services/scm_provider"
"github.com/argoproj/argo-cd/v2/applicationset/utils"
argoprojiov1alpha1 "github.com/argoproj/argo-cd/v2/pkg/apis/applicationset/v1alpha1"
@ -24,10 +25,18 @@ type SCMProviderGenerator struct {
client client.Client
// Testing hooks.
overrideProvider scm_provider.SCMProviderService
SCMAuthProviders
}
func NewSCMProviderGenerator(client client.Client) Generator {
return &SCMProviderGenerator{client: client}
type SCMAuthProviders struct {
GitHubApps github_app_auth.Credentials
}
func NewSCMProviderGenerator(client client.Client, providers SCMAuthProviders) Generator {
return &SCMProviderGenerator{
client: client,
SCMAuthProviders: providers,
}
}
// Testing generator
@ -66,13 +75,10 @@ func (g *SCMProviderGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha
if g.overrideProvider != nil {
provider = g.overrideProvider
} else if providerConfig.Github != nil {
token, err := g.getSecretRef(ctx, providerConfig.Github.TokenRef, applicationSetInfo.Namespace)
var err error
provider, err = g.githubProvider(ctx, providerConfig.Github, applicationSetInfo)
if err != nil {
return nil, fmt.Errorf("error fetching Github token: %v", err)
}
provider, err = scm_provider.NewGithubProvider(ctx, providerConfig.Github.Organization, token, providerConfig.Github.API, providerConfig.Github.AllBranches)
if err != nil {
return nil, fmt.Errorf("error initializing Github service: %v", err)
return nil, fmt.Errorf("scm provider: %w", err)
}
} else if providerConfig.Gitlab != nil {
token, err := g.getSecretRef(ctx, providerConfig.Gitlab.TokenRef, applicationSetInfo.Namespace)
@ -169,3 +175,25 @@ func (g *SCMProviderGenerator) getSecretRef(ctx context.Context, ref *argoprojio
}
return string(tokenBytes), nil
}
func (g *SCMProviderGenerator) githubProvider(ctx context.Context, github *argoprojiov1alpha1.SCMProviderGeneratorGithub, applicationSetInfo *argoprojiov1alpha1.ApplicationSet) (scm_provider.SCMProviderService, error) {
if github.AppSecretName != "" {
auth, err := g.GitHubApps.GetAuthSecret(ctx, github.AppSecretName)
if err != nil {
return nil, fmt.Errorf("error fetching Github app secret: %v", err)
}
return scm_provider.NewGithubAppProviderFor(
*auth,
github.Organization,
github.API,
github.AllBranches,
)
}
token, err := g.getSecretRef(ctx, github.TokenRef, applicationSetInfo.Namespace)
if err != nil {
return nil, fmt.Errorf("error fetching Github token: %v", err)
}
return scm_provider.NewGithubProvider(ctx, github.Organization, token, github.API, github.AllBranches)
}

View file

@ -0,0 +1,19 @@
package github_app_auth
import "context"
// Authentication has the authentication information required to access the GitHub API and repositories.
type Authentication struct {
// Id specifies the ID of the GitHub app used to access the repo
Id int64
// InstallationId specifies the installation ID of the GitHub App used to access the repo
InstallationId int64
// EnterpriseBaseURL specifies the base URL of GitHub Enterprise installation. If empty will default to https://api.github.com
EnterpriseBaseURL string
// PrivateKey in PEM format.
PrivateKey string
}
type Credentials interface {
GetAuthSecret(ctx context.Context, secretName string) (*Authentication, error)
}

View file

@ -0,0 +1,33 @@
package github_app
import (
"fmt"
"net/http"
"github.com/bradleyfalzon/ghinstallation/v2"
"github.com/google/go-github/v35/github"
"github.com/argoproj/argo-cd/v2/applicationset/services/github_app_auth"
)
// Client builds a github client for the given app authentication.
func Client(g github_app_auth.Authentication, url string) (*github.Client, error) {
rt, err := ghinstallation.New(http.DefaultTransport, g.Id, g.InstallationId, []byte(g.PrivateKey))
if err != nil {
return nil, fmt.Errorf("failed to create github app install: %w", err)
}
if url == "" {
url = g.EnterpriseBaseURL
}
var client *github.Client
httpClient := http.Client{Transport: rt}
if url == "" {
client = github.NewClient(&httpClient)
} else {
client, err = github.NewEnterpriseClient(url, url, &httpClient)
if err != nil {
return nil, fmt.Errorf("failed to create github enterprise client: %w", err)
}
}
return client, nil
}

View file

@ -58,7 +58,7 @@ func (g *GithubService) List(ctx context.Context) ([]*PullRequest, error) {
for {
pulls, resp, err := g.client.PullRequests.List(ctx, g.owner, g.repo, opts)
if err != nil {
return nil, fmt.Errorf("error listing pull requests for %s/%s: %v", g.owner, g.repo, err)
return nil, fmt.Errorf("error listing pull requests for %s/%s: %w", g.owner, g.repo, err)
}
for _, pull := range pulls {
if !containLabels(g.labels, pull.Labels) {

View file

@ -0,0 +1,19 @@
package pull_request
import (
"github.com/argoproj/argo-cd/v2/applicationset/services/github_app_auth"
"github.com/argoproj/argo-cd/v2/applicationset/services/internal/github_app"
)
func NewGithubAppService(g github_app_auth.Authentication, url, owner, repo string, labels []string) (PullRequestService, error) {
client, err := github_app.Client(g, url)
if err != nil {
return nil, err
}
return &GithubService{
client: client,
owner: owner,
repo: repo,
labels: labels,
}, nil
}

View file

@ -47,7 +47,7 @@ func (g *GithubProvider) GetBranches(ctx context.Context, repo *Repository) ([]*
repos := []*Repository{}
branches, err := g.listBranches(ctx, repo)
if err != nil {
return nil, fmt.Errorf("error listing branches for %s/%s: %v", repo.Organization, repo.Repository, err)
return nil, fmt.Errorf("error listing branches for %s/%s: %w", repo.Organization, repo.Repository, err)
}
for _, branch := range branches {
@ -72,7 +72,7 @@ func (g *GithubProvider) ListRepos(ctx context.Context, cloneProtocol string) ([
for {
githubRepos, resp, err := g.client.Repositories.ListByOrg(ctx, g.organization, opt)
if err != nil {
return nil, fmt.Errorf("error listing repositories for %s: %v", g.organization, err)
return nil, fmt.Errorf("error listing repositories for %s: %w", g.organization, err)
}
for _, githubRepo := range githubRepos {
var url string

View file

@ -0,0 +1,14 @@
package scm_provider
import (
"github.com/argoproj/argo-cd/v2/applicationset/services/github_app_auth"
"github.com/argoproj/argo-cd/v2/applicationset/services/internal/github_app"
)
func NewGithubAppProviderFor(g github_app_auth.Authentication, organization string, url string, allBranches bool) (*GithubProvider, error) {
client, err := github_app.Client(g, url)
if err != nil {
return nil, err
}
return &GithubProvider{client: client, organization: organization, allBranches: allBranches}, nil
}

View file

@ -19,6 +19,7 @@ import (
"github.com/argoproj/argo-cd/v2/common"
"github.com/argoproj/argo-cd/v2/reposerver/askpass"
"github.com/argoproj/argo-cd/v2/util/env"
"github.com/argoproj/argo-cd/v2/util/github_app"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
@ -136,13 +137,16 @@ func NewCommand() *cobra.Command {
argoCDDB := db.NewDB(namespace, argoSettingsMgr, k8sClient)
askPassServer := askpass.NewServer()
scmAuth := generators.SCMAuthProviders{
GitHubApps: github_app.NewAuthCredentials(argoCDDB.(db.RepoCredsDB)),
}
terminalGenerators := map[string]generators.Generator{
"List": generators.NewListGenerator(),
"Clusters": generators.NewClusterGenerator(mgr.GetClient(), ctx, k8sClient, namespace),
"Git": generators.NewGitGenerator(services.NewArgoCDService(argoCDDB, askPassServer, getSubmoduleEnabled())),
"SCMProvider": generators.NewSCMProviderGenerator(mgr.GetClient()),
"SCMProvider": generators.NewSCMProviderGenerator(mgr.GetClient(), scmAuth),
"ClusterDecisionResource": generators.NewDuckTypeGenerator(ctx, dynamicClient, k8sClient, namespace),
"PullRequest": generators.NewPullRequestGenerator(mgr.GetClient()),
"PullRequest": generators.NewPullRequestGenerator(mgr.GetClient(), scmAuth),
}
nestedGenerators := map[string]generators.Generator{

View file

@ -44,6 +44,8 @@ spec:
tokenRef:
secretName: github-token
key: token
# (optional) use a GitHub App to access the API instead of a PAT.
appSecretName: github-app-repo-creds
# Labels is used to filter the PRs that you want to target. (optional)
labels:
- preview
@ -57,6 +59,9 @@ spec:
* `api`: If using GitHub Enterprise, the URL to access it. (Optional)
* `tokenRef`: A `Secret` name and key containing the GitHub access token to use for requests. If not specified, will make anonymous requests which have a lower rate limit and can only see public repositories. (Optional)
* `labels`: Labels is used to filter the PRs that you want to target. (Optional)
* `appSecretName`: A `Secret` name containing a GitHub App secret in [repo-creds format][repo-creds].
[repo-creds]: ../declarative-setup.md#repository-credentials
## GitLab

View file

@ -48,6 +48,8 @@ spec:
tokenRef:
secretName: github-token
key: token
# (optional) use a GitHub App to access the API instead of a PAT.
appSecretName: gh-app-repo-creds
template:
# ...
```
@ -56,6 +58,9 @@ spec:
* `api`: If using GitHub Enterprise, the URL to access it.
* `allBranches`: By default (false) the template will only be evaluated for the default branch of each repo. If this is true, every branch of every repository will be passed to the filters. If using this flag, you likely want to use a `branchMatch` filter.
* `tokenRef`: A `Secret` name and key containing the GitHub access token to use for requests. If not specified, will make anonymous requests which have a lower rate limit and can only see public repositories.
* `appSecretName`: A `Secret` name containing a GitHub App secret in [repo-creds format][repo-creds].
[repo-creds]: ../declarative-setup.md#repository-credentials
For label filtering, the repository topics are used.

View file

@ -4527,6 +4527,8 @@ spec:
properties:
api:
type: string
appSecretName:
type: string
labels:
items:
type: string
@ -4964,6 +4966,8 @@ spec:
type: boolean
api:
type: string
appSecretName:
type: string
organization:
type: string
tokenRef:
@ -6753,6 +6757,8 @@ spec:
properties:
api:
type: string
appSecretName:
type: string
labels:
items:
type: string
@ -7190,6 +7196,8 @@ spec:
type: boolean
api:
type: string
appSecretName:
type: string
organization:
type: string
tokenRef:
@ -7844,6 +7852,8 @@ spec:
properties:
api:
type: string
appSecretName:
type: string
labels:
items:
type: string
@ -8281,6 +8291,8 @@ spec:
type: boolean
api:
type: string
appSecretName:
type: string
organization:
type: string
tokenRef:

View file

@ -2371,6 +2371,8 @@ spec:
properties:
api:
type: string
appSecretName:
type: string
labels:
items:
type: string
@ -2808,6 +2810,8 @@ spec:
type: boolean
api:
type: string
appSecretName:
type: string
organization:
type: string
tokenRef:
@ -4597,6 +4601,8 @@ spec:
properties:
api:
type: string
appSecretName:
type: string
labels:
items:
type: string
@ -5034,6 +5040,8 @@ spec:
type: boolean
api:
type: string
appSecretName:
type: string
organization:
type: string
tokenRef:
@ -5688,6 +5696,8 @@ spec:
properties:
api:
type: string
appSecretName:
type: string
labels:
items:
type: string
@ -6125,6 +6135,8 @@ spec:
type: boolean
api:
type: string
appSecretName:
type: string
organization:
type: string
tokenRef:

View file

@ -4527,6 +4527,8 @@ spec:
properties:
api:
type: string
appSecretName:
type: string
labels:
items:
type: string
@ -4964,6 +4966,8 @@ spec:
type: boolean
api:
type: string
appSecretName:
type: string
organization:
type: string
tokenRef:
@ -6753,6 +6757,8 @@ spec:
properties:
api:
type: string
appSecretName:
type: string
labels:
items:
type: string
@ -7190,6 +7196,8 @@ spec:
type: boolean
api:
type: string
appSecretName:
type: string
organization:
type: string
tokenRef:
@ -7844,6 +7852,8 @@ spec:
properties:
api:
type: string
appSecretName:
type: string
labels:
items:
type: string
@ -8281,6 +8291,8 @@ spec:
type: boolean
api:
type: string
appSecretName:
type: string
organization:
type: string
tokenRef:

View file

@ -4527,6 +4527,8 @@ spec:
properties:
api:
type: string
appSecretName:
type: string
labels:
items:
type: string
@ -4964,6 +4966,8 @@ spec:
type: boolean
api:
type: string
appSecretName:
type: string
organization:
type: string
tokenRef:
@ -6753,6 +6757,8 @@ spec:
properties:
api:
type: string
appSecretName:
type: string
labels:
items:
type: string
@ -7190,6 +7196,8 @@ spec:
type: boolean
api:
type: string
appSecretName:
type: string
organization:
type: string
tokenRef:
@ -7844,6 +7852,8 @@ spec:
properties:
api:
type: string
appSecretName:
type: string
labels:
items:
type: string
@ -8281,6 +8291,8 @@ spec:
type: boolean
api:
type: string
appSecretName:
type: string
organization:
type: string
tokenRef:

View file

@ -339,6 +339,8 @@ type SCMProviderGeneratorGithub struct {
API string `json:"api,omitempty"`
// Authentication token reference.
TokenRef *SecretRef `json:"tokenRef,omitempty"`
// AppSecretName is a reference to a GitHub App repo-creds secret.
AppSecretName string `json:"appSecretName,omitempty"`
// Scan all branches instead of just the default branch.
AllBranches bool `json:"allBranches,omitempty"`
}
@ -450,6 +452,8 @@ type PullRequestGeneratorGithub struct {
API string `json:"api,omitempty"`
// Authentication token reference.
TokenRef *SecretRef `json:"tokenRef,omitempty"`
// AppSecretName is a reference to a GitHub App repo-creds secret with permission to access pull requests.
AppSecretName string `json:"appSecretName,omitempty"`
// Labels is used to filter the PRs that you want to target
Labels []string `json:"labels,omitempty"`
}

15
util/db/repo_creds.go Normal file
View file

@ -0,0 +1,15 @@
package db
import (
"context"
appsv1 "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1"
)
type RepoCredsDB interface {
GetRepoCredsBySecretName(_ context.Context, secretName string) (*appsv1.RepoCreds, error)
}
func (db *db) GetRepoCredsBySecretName(ctx context.Context, secretName string) (*appsv1.RepoCreds, error) {
return (&secretsRepositoryBackend{db: db}).GetRepoCredsBySecretName(ctx, secretName)
}

View file

@ -1,10 +1,10 @@
package db
import (
"context"
"fmt"
"hash/fnv"
"context"
log "github.com/sirupsen/logrus"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"

View file

@ -1,10 +1,10 @@
package db
import (
"context"
"fmt"
"strings"
"context"
log "github.com/sirupsen/logrus"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
@ -73,6 +73,14 @@ func (s *secretsRepositoryBackend) hasRepoTypeLabel(secretName string) (bool, er
return false, nil
}
func (s *secretsRepositoryBackend) GetRepoCredsBySecretName(_ context.Context, name string) (*appsv1.RepoCreds, error) {
secret, err := s.db.getSecret(name, map[string]*corev1.Secret{})
if err != nil {
return nil, fmt.Errorf("failed to get secret %s: %v", name, err)
}
return s.secretToRepoCred(secret)
}
func (s *secretsRepositoryBackend) GetRepository(ctx context.Context, repoURL string) (*appsv1.Repository, error) {
secret, err := s.getRepositorySecret(repoURL)
if err != nil {

36
util/github_app/repos.go Normal file
View file

@ -0,0 +1,36 @@
package github_app
import (
"context"
"fmt"
"github.com/argoproj/argo-cd/v2/applicationset/services/github_app_auth"
"github.com/argoproj/argo-cd/v2/util/db"
)
// NewAuthCredentials returns a GtiHub App credentials lookup by repo-creds url.
func NewAuthCredentials(creds db.RepoCredsDB) github_app_auth.Credentials {
return &repoAsCredentials{RepoCredsDB: creds}
}
type repoAsCredentials struct {
db.RepoCredsDB
}
func (r *repoAsCredentials) GetAuthSecret(ctx context.Context, secretName string) (*github_app_auth.Authentication, error) {
repo, err := r.GetRepoCredsBySecretName(ctx, secretName)
if err != nil {
return nil, err
}
if repo == nil || repo.GithubAppPrivateKey == "" {
return nil, fmt.Errorf("no github app found for %s", secretName)
}
return &github_app_auth.Authentication{
Id: repo.GithubAppId,
InstallationId: repo.GithubAppInstallationId,
EnterpriseBaseURL: repo.GitHubAppEnterpriseBaseURL,
PrivateKey: repo.GithubAppPrivateKey,
}, nil
}
var _ github_app_auth.Credentials = (*repoAsCredentials)(nil)

View file

@ -0,0 +1,60 @@
package github_app
import (
"context"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/argoproj/argo-cd/v2/applicationset/services/github_app_auth"
"github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1"
)
type ArgocdRepositoryMock struct {
mock *mock.Mock
}
func (a ArgocdRepositoryMock) GetRepoCredsBySecretName(ctx context.Context, secretName string) (*v1alpha1.RepoCreds, error) {
args := a.mock.Called(ctx, secretName)
return args.Get(0).(*v1alpha1.RepoCreds), args.Error(1)
}
func Test_repoAsCredentials_GetAuth(t *testing.T) {
tests := []struct {
name string
repo v1alpha1.RepoCreds
want *github_app_auth.Authentication
wantErr bool
}{
{name: "missing", wantErr: true},
{name: "found", repo: v1alpha1.RepoCreds{
GithubAppId: 123,
GithubAppInstallationId: 456,
GithubAppPrivateKey: "private key",
}, want: &github_app_auth.Authentication{
Id: 123,
InstallationId: 456,
EnterpriseBaseURL: "",
PrivateKey: "private key",
}, wantErr: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
m := mock.Mock{}
m.On("GetRepoCredsBySecretName", mock.Anything, mock.Anything).Return(&tt.repo, nil)
creds := NewAuthCredentials(ArgocdRepositoryMock{mock: &m})
auth, err := creds.GetAuthSecret(context.Background(), "https://github.com/foo")
if tt.wantErr {
assert.Error(t, err)
return
}
assert.NoError(t, err)
assert.Equal(t, tt.want, auth)
})
}
}