diff --git a/common/common.go b/common/common.go index 2c8809be58..1575c9eef6 100644 --- a/common/common.go +++ b/common/common.go @@ -192,6 +192,8 @@ const ( LabelValueSecretTypeRepoCreds = "repo-creds" // LabelValueSecretTypeRepositoryWrite indicates a secret type of repository credentials for writing LabelValueSecretTypeRepositoryWrite = "repository-write" + // LabelValueSecretTypeRepoCredsWrite indicates a secret type of repository credentials for writing for templating + LabelValueSecretTypeRepoCredsWrite = "repo-write-creds" // LabelValueSecretTypeSCMCreds indicates a secret type of SCM credentials LabelValueSecretTypeSCMCreds = "scm-creds" diff --git a/docs/user-guide/source-hydrator.md b/docs/user-guide/source-hydrator.md index e2dab29d98..f6084e0f1d 100644 --- a/docs/user-guide/source-hydrator.md +++ b/docs/user-guide/source-hydrator.md @@ -263,6 +263,27 @@ specified more than once, the last one will be used. All trailers are optional. If a trailer is not specified, the corresponding field in the metadata will be omitted. +### Credential Templates + +Credential templates allow a single credential to be used for multiple repositories. The source hydrator supports credential templates. For example, if you setup credential templates for the URL prefix `https://github.com/argoproj`, these credentials will be used for all repositories with this URL as prefix (e.g. `https://github.com/argoproj/argocd-example-apps`) that do not have their own credentials configured. +For more information please refer [credential-template](private-repositories.md#credential-templates). +An example of repo-write-creds secret. + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: private-repo + namespace: argocd + labels: + argocd.argoproj.io/secret-type: repo-write-creds +stringData: + type: git + url: https://github.com/argoproj + password: my-password + username: my-username +``` + ## Limitations ### Signature Verification @@ -277,11 +298,6 @@ If all the Applications for a given destination repo/branch are under the same p available project-scoped push secrets. If two Applications for a given repo/branch are in different projects, then the hydrator will not be able to use a project-scoped push secret and will require a global push secret. -### Credential Templates - -Credential templates allow a single credential to be used for multiple repositories. The source hydrator does not -currently support credential templates. You will need a separate credential for each repository. - ### `manifest-generate-paths` Annotation Support The source hydrator does not currently support the [manifest-generate-paths annotation](../operator-manual/high_availability.md#manifest-paths-annotation) diff --git a/util/db/db_test.go b/util/db/db_test.go index 2c8c67bd92..50cd2abdaa 100644 --- a/util/db/db_test.go +++ b/util/db/db_test.go @@ -151,6 +151,38 @@ func TestCreateRepoCredentials(t *testing.T) { assert.Equal(t, "test-password", repo.Password) } +func TestCreateWriteRepoCredentials(t *testing.T) { + clientset := getClientset() + db := NewDB(testNamespace, settings.NewSettingsManager(t.Context(), clientset, testNamespace), clientset) + + creds, err := db.CreateWriteRepositoryCredentials(t.Context(), &v1alpha1.RepoCreds{ + URL: "https://github.com/argoproj/", + Username: "test-username", + Password: "test-password", + }) + require.NoError(t, err) + assert.Equal(t, "https://github.com/argoproj/", creds.URL) + + secret, err := clientset.CoreV1().Secrets(testNamespace).Get(t.Context(), RepoURLToSecretName(credSecretPrefix, creds.URL, ""), metav1.GetOptions{}) + require.NoError(t, err) + + assert.Equal(t, common.AnnotationValueManagedByArgoCD, secret.Annotations[common.AnnotationKeyManagedBy]) + assert.Equal(t, "test-username", string(secret.Data[username])) + assert.Equal(t, "test-password", string(secret.Data[password])) + assert.Empty(t, secret.Data[sshPrivateKey]) + + created, err := db.CreateWriteRepository(t.Context(), &v1alpha1.Repository{ + Repo: "https://github.com/argoproj/argo-cd", + }) + require.NoError(t, err) + assert.Equal(t, "https://github.com/argoproj/argo-cd", created.Repo) + + repo, err := db.GetWriteRepository(t.Context(), created.Repo, "") + require.NoError(t, err) + assert.Equal(t, "test-username", repo.Username) + assert.Equal(t, "test-password", repo.Password) +} + func TestGetRepositoryCredentials(t *testing.T) { clientset := getClientset() db := NewDB(testNamespace, settings.NewSettingsManager(t.Context(), clientset, testNamespace), clientset) @@ -197,6 +229,52 @@ func TestGetRepositoryCredentials(t *testing.T) { } } +func TestGetWriteRepositoryCredentials(t *testing.T) { + clientset := getClientset() + db := NewDB(testNamespace, settings.NewSettingsManager(t.Context(), clientset, testNamespace), clientset) + _, err := db.CreateWriteRepositoryCredentials(t.Context(), &v1alpha1.RepoCreds{ + URL: "https://secured", + Username: "test-username", + Password: "test-password", + }) + require.NoError(t, err) + + tests := []struct { + name string + repoURL string + want *v1alpha1.RepoCreds + }{ + { + name: "TestUnknownRepo", + repoURL: "https://unknown/repo", + want: nil, + }, + { + name: "TestKnownRepo", + repoURL: "https://known/repo", + want: nil, + }, + { + name: "TestSecuredRepo", + repoURL: "https://secured/repo", + want: &v1alpha1.RepoCreds{URL: "https://secured", Username: "test-username", Password: "test-password"}, + }, + { + name: "TestMissingRepo", + repoURL: "https://missing/repo", + want: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := db.GetWriteRepositoryCredentials(t.Context(), tt.repoURL) + + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} + func TestCreateExistingRepository(t *testing.T) { clientset := getClientset() db := NewDB(testNamespace, settings.NewSettingsManager(t.Context(), clientset, testNamespace), clientset) @@ -295,6 +373,84 @@ func TestGetRepository(t *testing.T) { } } +func TestGetWriteRepository(t *testing.T) { + clientset := getClientset(&corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNamespace, + Name: "known-repo-secret", + Annotations: map[string]string{ + common.AnnotationKeyManagedBy: common.AnnotationValueManagedByArgoCD, + }, + Labels: map[string]string{ + common.LabelKeySecretType: common.LabelValueSecretTypeRepositoryWrite, + }, + }, + Data: map[string][]byte{ + "url": []byte("https://known/repo"), + }, + }, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNamespace, + Name: "secured-repo-secret", + Annotations: map[string]string{ + common.AnnotationKeyManagedBy: common.AnnotationValueManagedByArgoCD, + }, + Labels: map[string]string{ + common.LabelKeySecretType: common.LabelValueSecretTypeRepositoryWrite, + }, + }, + Data: map[string][]byte{ + "url": []byte("https://secured/repo"), + }, + }, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNamespace, + Name: "secured-repo-creds-secret", + Annotations: map[string]string{ + common.AnnotationKeyManagedBy: common.AnnotationValueManagedByArgoCD, + }, + Labels: map[string]string{ + common.LabelKeySecretType: common.LabelValueSecretTypeRepoCredsWrite, + }, + }, + Data: map[string][]byte{ + "url": []byte("https://secured"), + "username": []byte("test-username"), + "password": []byte("test-password"), + }, + }) + db := NewDB(testNamespace, settings.NewSettingsManager(t.Context(), clientset, testNamespace), clientset) + + tests := []struct { + name string + repoURL string + want *v1alpha1.Repository + }{ + { + name: "TestUnknownRepo", + repoURL: "https://unknown/repo", + want: &v1alpha1.Repository{Repo: "https://unknown/repo"}, + }, + { + name: "TestKnownRepo", + repoURL: "https://known/repo", + want: &v1alpha1.Repository{Repo: "https://known/repo"}, + }, + { + name: "TestSecuredRepo", + repoURL: "https://secured/repo", + want: &v1alpha1.Repository{Repo: "https://secured/repo", Username: "test-username", Password: "test-password", InheritedCreds: true}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := db.GetWriteRepository(t.Context(), tt.repoURL, "") + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} + func TestCreateClusterSuccessful(t *testing.T) { server := "https://mycluster" clientset := getClientset() diff --git a/util/db/repository.go b/util/db/repository.go index 651cecfb88..052719fc60 100644 --- a/util/db/repository.go +++ b/util/db/repository.go @@ -99,10 +99,9 @@ func (db *db) GetWriteRepository(ctx context.Context, repoURL, project string) ( return repository, fmt.Errorf("unable to get write repository %q: %w", repoURL, err) } - // TODO: enrich with write credentials. - // if err := db.enrichCredsToRepo(ctx, repository); err != nil { - // return repository, fmt.Errorf("unable to enrich write repository %q info with credentials: %w", repoURL, err) - // } + if err := db.enrichWriteCredsToRepo(ctx, repository); err != nil { + return repository, fmt.Errorf("unable to enrich write repository %q info with credentials: %w", repoURL, err) + } return repository, err } @@ -292,23 +291,15 @@ func (db *db) GetRepositoryCredentials(ctx context.Context, repoURL string) (*v1 // GetWriteRepositoryCredentials retrieves a repository write credential set func (db *db) GetWriteRepositoryCredentials(ctx context.Context, repoURL string) (*v1alpha1.RepoCreds, error) { secretBackend := db.repoWriteBackend() - exists, err := secretBackend.RepoCredsExists(ctx, repoURL) - if err != nil { - return nil, fmt.Errorf("unable to check if repository write credentials for %q exists from secrets backend: %w", repoURL, err) - } - - if !exists { - return nil, nil - } - - // TODO: enrich with write credentials. - // if err := db.enrichCredsToRepo(ctx, repository); err != nil { - // return repository, fmt.Errorf("unable to enrich write repository %q info with credentials: %w", repoURL, err) - // } - creds, err := secretBackend.GetRepoCreds(ctx, repoURL) if err != nil { - return nil, fmt.Errorf("unable to get repository write credentials for %q from secrets backend: %w", repoURL, err) + if creds == nil { + return nil, fmt.Errorf("unable to check if repo write credentials for %q exists from secrets backend: %w", repoURL, err) + } + return nil, fmt.Errorf("unable to get repo write credentials for %q from secrets backend: %w", repoURL, err) + } + if creds == nil { // to cover for not found. In that case both creds and err are nil + return nil, nil } return creds, nil @@ -454,6 +445,23 @@ func (db *db) enrichCredsToRepo(ctx context.Context, repository *v1alpha1.Reposi return nil } +func (db *db) enrichWriteCredsToRepo(ctx context.Context, repository *v1alpha1.Repository) error { + if !repository.HasCredentials() { + creds, err := db.GetWriteRepositoryCredentials(ctx, repository.Repo) + if err != nil { + return fmt.Errorf("failed to get repository credentials for %q: %w", repository.Repo, err) + } + if creds != nil { + repository.CopyCredentialsFrom(creds) + repository.InheritedCreds = true + } + } else { + log.Debugf("%s has credentials", repository.Repo) + } + + return nil +} + // RepoURLToSecretName hashes repo URL to a secret name using a formula. This is used when // repositories are _imperatively_ created and need its credentials to be stored in a secret. // NOTE: this formula should not be considered stable and may change in future releases. diff --git a/util/db/repository_secrets.go b/util/db/repository_secrets.go index f39a7658b8..13c8a02e78 100644 --- a/util/db/repository_secrets.go +++ b/util/db/repository_secrets.go @@ -187,7 +187,7 @@ func (s *secretsRepositoryBackend) CreateRepoCreds(ctx context.Context, repoCred }, } - repoCredsToSecret(repoCreds, repoCredsSecret) + s.repoCredsToSecret(repoCreds, repoCredsSecret) _, err := s.db.createSecret(ctx, repoCredsSecret) if err != nil { @@ -237,7 +237,7 @@ func (s *secretsRepositoryBackend) UpdateRepoCreds(ctx context.Context, repoCred return nil, err } - repoCredsToSecret(repoCreds, repoCredsSecret) + s.repoCredsToSecret(repoCreds, repoCredsSecret) repoCredsSecret, err = s.db.kubeclientset.CoreV1().Secrets(s.db.ns).Update(ctx, repoCredsSecret, metav1.UpdateOptions{}) if err != nil { @@ -486,7 +486,7 @@ func (s *secretsRepositoryBackend) secretToRepoCred(secret *corev1.Secret) (*app return repository, nil } -func repoCredsToSecret(repoCreds *appsv1.RepoCreds, secret *corev1.Secret) { +func (s *secretsRepositoryBackend) repoCredsToSecret(repoCreds *appsv1.RepoCreds, secret *corev1.Secret) { if secret.Data == nil { secret.Data = make(map[string][]byte) } @@ -510,7 +510,7 @@ func repoCredsToSecret(repoCreds *appsv1.RepoCreds, secret *corev1.Secret) { updateSecretString(secret, "noProxy", repoCreds.NoProxy) updateSecretBool(secret, "forceHttpBasicAuth", repoCreds.ForceHttpBasicAuth) updateSecretBool(secret, "useAzureWorkloadIdentity", repoCreds.UseAzureWorkloadIdentity) - addSecretMetadata(secret, common.LabelValueSecretTypeRepoCreds) + addSecretMetadata(secret, s.getRepoCredSecretType()) } func (s *secretsRepositoryBackend) getRepositorySecret(repoURL, project string, allowFallback bool) (*corev1.Secret, error) { @@ -549,7 +549,7 @@ func (s *secretsRepositoryBackend) getRepositorySecret(repoURL, project string, } func (s *secretsRepositoryBackend) getRepoCredsSecret(repoURL string) (*corev1.Secret, error) { - secrets, err := s.db.listSecretsByType(common.LabelValueSecretTypeRepoCreds) + secrets, err := s.db.listSecretsByType(s.getRepoCredSecretType()) if err != nil { return nil, err } @@ -586,3 +586,10 @@ func (s *secretsRepositoryBackend) getSecretType() string { } return common.LabelValueSecretTypeRepository } + +func (s *secretsRepositoryBackend) getRepoCredSecretType() string { + if s.writeCreds { + return common.LabelValueSecretTypeRepoCredsWrite + } + return common.LabelValueSecretTypeRepoCreds +} diff --git a/util/db/repository_secrets_test.go b/util/db/repository_secrets_test.go index 7f40dc3e56..3fb0fc6e49 100644 --- a/util/db/repository_secrets_test.go +++ b/util/db/repository_secrets_test.go @@ -931,6 +931,12 @@ func TestSecretsRepositoryBackend_GetAllHelmRepoCreds(t *testing.T) { } func TestRepoCredsToSecret(t *testing.T) { + clientset := getClientset() + testee := &secretsRepositoryBackend{db: &db{ + ns: testNamespace, + kubeclientset: clientset, + settingsMgr: settings.NewSettingsManager(t.Context(), clientset, testNamespace), + }} s := &corev1.Secret{} creds := &appsv1.RepoCreds{ URL: "URL", @@ -946,7 +952,7 @@ func TestRepoCredsToSecret(t *testing.T) { GithubAppInstallationId: 456, GitHubAppEnterpriseBaseURL: "GitHubAppEnterpriseBaseURL", } - repoCredsToSecret(creds, s) + testee.repoCredsToSecret(creds, s) assert.Equal(t, []byte(creds.URL), s.Data["url"]) assert.Equal(t, []byte(creds.Username), s.Data["username"]) assert.Equal(t, []byte(creds.Password), s.Data["password"]) @@ -962,3 +968,45 @@ func TestRepoCredsToSecret(t *testing.T) { assert.Equal(t, map[string]string{common.AnnotationKeyManagedBy: common.AnnotationValueManagedByArgoCD}, s.Annotations) assert.Equal(t, map[string]string{common.LabelKeySecretType: common.LabelValueSecretTypeRepoCreds}, s.Labels) } + +func TestRepoWriteCredsToSecret(t *testing.T) { + clientset := getClientset() + testee := &secretsRepositoryBackend{ + db: &db{ + ns: testNamespace, + kubeclientset: clientset, + settingsMgr: settings.NewSettingsManager(t.Context(), clientset, testNamespace), + }, + writeCreds: true, + } + s := &corev1.Secret{} + creds := &appsv1.RepoCreds{ + URL: "URL", + Username: "Username", + Password: "Password", + SSHPrivateKey: "SSHPrivateKey", + EnableOCI: true, + TLSClientCertData: "TLSClientCertData", + TLSClientCertKey: "TLSClientCertKey", + Type: "Type", + GithubAppPrivateKey: "GithubAppPrivateKey", + GithubAppId: 123, + GithubAppInstallationId: 456, + GitHubAppEnterpriseBaseURL: "GitHubAppEnterpriseBaseURL", + } + testee.repoCredsToSecret(creds, s) + assert.Equal(t, []byte(creds.URL), s.Data["url"]) + assert.Equal(t, []byte(creds.Username), s.Data["username"]) + assert.Equal(t, []byte(creds.Password), s.Data["password"]) + assert.Equal(t, []byte(creds.SSHPrivateKey), s.Data["sshPrivateKey"]) + assert.Equal(t, []byte(strconv.FormatBool(creds.EnableOCI)), s.Data["enableOCI"]) + assert.Equal(t, []byte(creds.TLSClientCertData), s.Data["tlsClientCertData"]) + assert.Equal(t, []byte(creds.TLSClientCertKey), s.Data["tlsClientCertKey"]) + assert.Equal(t, []byte(creds.Type), s.Data["type"]) + assert.Equal(t, []byte(creds.GithubAppPrivateKey), s.Data["githubAppPrivateKey"]) + assert.Equal(t, []byte(strconv.FormatInt(creds.GithubAppId, 10)), s.Data["githubAppID"]) + assert.Equal(t, []byte(strconv.FormatInt(creds.GithubAppInstallationId, 10)), s.Data["githubAppInstallationID"]) + assert.Equal(t, []byte(creds.GitHubAppEnterpriseBaseURL), s.Data["githubAppEnterpriseBaseUrl"]) + assert.Equal(t, map[string]string{common.AnnotationKeyManagedBy: common.AnnotationValueManagedByArgoCD}, s.Annotations) + assert.Equal(t, map[string]string{common.LabelKeySecretType: common.LabelValueSecretTypeRepoCredsWrite}, s.Labels) +} diff --git a/util/db/repository_test.go b/util/db/repository_test.go index 90552ab847..5e08a5f25e 100644 --- a/util/db/repository_test.go +++ b/util/db/repository_test.go @@ -53,6 +53,46 @@ var repoArgoProj = &corev1.Secret{ }, } +var repoArgoCDWrite = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNamespace, + Name: "some-repo-secret", + Annotations: map[string]string{ + common.AnnotationKeyManagedBy: common.AnnotationValueManagedByArgoCD, + }, + Labels: map[string]string{ + common.LabelKeySecretType: common.LabelValueSecretTypeRepositoryWrite, + }, + }, + Data: map[string][]byte{ + "name": []byte("SomeRepo"), + "url": []byte("git@github.com:argoproj/argo-cd.git"), + "username": []byte("someUsername"), + "password": []byte("somePassword"), + "type": []byte("git"), + }, +} + +var repoArgoProjWrite = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNamespace, + Name: "some-other-repo-secret", + Annotations: map[string]string{ + common.AnnotationKeyManagedBy: common.AnnotationValueManagedByArgoCD, + }, + Labels: map[string]string{ + common.LabelKeySecretType: common.LabelValueSecretTypeRepositoryWrite, + }, + }, + Data: map[string][]byte{ + "name": []byte("OtherRepo"), + "url": []byte("git@github.com:argoproj/argoproj.git"), + "username": []byte("someUsername"), + "password": []byte("somePassword"), + "type": []byte("git"), + }, +} + func TestDb_CreateRepository(t *testing.T) { clientset := getClientset() settingsManager := settings.NewSettingsManager(t.Context(), clientset, testNamespace) @@ -108,6 +148,41 @@ func TestDb_GetRepository(t *testing.T) { assert.Equal(t, "git@github.com:argoproj/not-existing.git", repository.Repo) } +func TestDb_GetWriteRepository(t *testing.T) { + clientset := getClientset(repoArgoCDWrite, repoArgoProjWrite) + settingsManager := settings.NewSettingsManager(t.Context(), clientset, testNamespace) + testee := &db{ + ns: testNamespace, + kubeclientset: clientset, + settingsMgr: settingsManager, + } + + repository, err := testee.GetWriteRepository(t.Context(), "git@github.com:argoproj/argoproj.git", "") + require.NoError(t, err) + require.NotNil(t, repository) + assert.Equal(t, "OtherRepo", repository.Name) + + repository, err = testee.GetWriteRepository(t.Context(), "git@github.com:argoproj/argo-cd.git", "") + require.NoError(t, err) + require.NotNil(t, repository) + assert.Equal(t, "SomeRepo", repository.Name) +} + +func TestDb_GetWriteRepository_SecretNotFound_DefaultRepo(t *testing.T) { + clientset := getClientset(repoArgoCD) + settingsManager := settings.NewSettingsManager(t.Context(), clientset, testNamespace) + testee := &db{ + ns: testNamespace, + kubeclientset: clientset, + settingsMgr: settingsManager, + } + + repository, err := testee.GetWriteRepository(t.Context(), "git@github.com:argoproj/argo-cd.git", "") + require.NoError(t, err) + require.NotNil(t, repository) + assert.Empty(t, repository.Name) +} + func TestDb_ListRepositories(t *testing.T) { clientset := getClientset(repoArgoCD, repoArgoProj) settingsManager := settings.NewSettingsManager(t.Context(), clientset, testNamespace)