Auto-detect Helm repos + support Helm basic auth + fix bugs (#2309)

This commit is contained in:
Alex Collins 2019-09-19 17:35:27 +01:00 committed by GitHub
parent 70a97c0db8
commit 1e5c78e35f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 281 additions and 180 deletions

View file

@ -1005,20 +1005,29 @@ func (repo *Repository) IsLFSEnabled() bool {
return repo.EnableLFS
}
func (m *Repository) HasCredentials() bool {
return m.Username != "" || m.Password != "" || m.SSHPrivateKey != "" || m.InsecureIgnoreHostKey
}
func (m *Repository) CopyCredentialsFrom(source *Repository) {
if source != nil {
m.Username = source.Username
m.Password = source.Password
m.SSHPrivateKey = source.SSHPrivateKey
m.InsecureIgnoreHostKey = source.InsecureIgnoreHostKey
m.Insecure = source.Insecure
m.EnableLFS = source.EnableLFS
m.TLSClientCertData = source.TLSClientCertData
m.TLSClientCertKey = source.TLSClientCertKey
if m.Username == "" {
m.Username = source.Username
}
if m.Password == "" {
m.Password = source.Password
}
if m.SSHPrivateKey == "" {
m.SSHPrivateKey = source.SSHPrivateKey
}
m.InsecureIgnoreHostKey = m.InsecureIgnoreHostKey || source.InsecureIgnoreHostKey
m.Insecure = m.Insecure || source.Insecure
m.EnableLFS = m.EnableLFS || source.EnableLFS
if m.TLSClientCertData == "" {
m.TLSClientCertData = source.TLSClientCertData
}
if m.TLSClientCertKey == "" {
m.TLSClientCertKey = source.TLSClientCertKey
}
if m.TLSClientCAData == "" {
m.TLSClientCAData = source.TLSClientCAData
}
}
}

View file

@ -405,66 +405,39 @@ func TestAppProjectSpec_DestinationClusters(t *testing.T) {
}
}
func TestRepository_HasCredentials(t *testing.T) {
tests := []struct {
name string
repo Repository
want bool
}{
{
name: "TestHasRepo",
repo: Repository{Repo: "foo"},
want: false,
},
{
name: "TestHasUsername",
repo: Repository{Username: "foo"},
want: true,
},
{
name: "TestHasPassword",
repo: Repository{Password: "foo"},
want: true,
},
{
name: "TestHasSSHPrivateKey",
repo: Repository{SSHPrivateKey: "foo"},
want: true,
},
{
name: "TestHasInsecureHostKey",
repo: Repository{InsecureIgnoreHostKey: true},
want: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := tt.repo.HasCredentials(); got != tt.want {
t.Errorf("Repository.HasCredentials() = %v, want %v", got, tt.want)
}
})
}
}
func TestRepository_CopyCredentialsFrom(t *testing.T) {
tests := []struct {
name string
repo *Repository
source *Repository
want Repository
}{
{"TestNil", nil, Repository{}},
{"TestHasRepo", &Repository{Repo: "foo"}, Repository{}},
{"TestHasUsername", &Repository{Username: "foo"}, Repository{Username: "foo"}},
{"TestHasPassword", &Repository{Password: "foo"}, Repository{Password: "foo"}},
{"TestHasSSHPrivateKey", &Repository{SSHPrivateKey: "foo"}, Repository{SSHPrivateKey: "foo"}},
{"TestHasInsecureHostKey", &Repository{InsecureIgnoreHostKey: true}, Repository{InsecureIgnoreHostKey: true}},
{"Username", &Repository{Username: "foo"}, &Repository{}, Repository{Username: "foo"}},
{"Password", &Repository{Password: "foo"}, &Repository{}, Repository{Password: "foo"}},
{"SSHPrivateKey", &Repository{SSHPrivateKey: "foo"}, &Repository{}, Repository{SSHPrivateKey: "foo"}},
{"InsecureHostKey", &Repository{InsecureIgnoreHostKey: true}, &Repository{}, Repository{InsecureIgnoreHostKey: true}},
{"Insecure", &Repository{Insecure: true}, &Repository{}, Repository{Insecure: true}},
{"EnableLFS", &Repository{EnableLFS: true}, &Repository{}, Repository{EnableLFS: true}},
{"TLSClientCAData", &Repository{TLSClientCAData: "foo"}, &Repository{}, Repository{TLSClientCAData: "foo"}},
{"TLSClientCertData", &Repository{TLSClientCertData: "foo"}, &Repository{}, Repository{TLSClientCertData: "foo"}},
{"TLSClientCertKey", &Repository{TLSClientCertKey: "foo"}, &Repository{}, Repository{TLSClientCertKey: "foo"}},
{"SourceNil", &Repository{}, nil, Repository{}},
{"SourceUsername", &Repository{}, &Repository{Username: "foo"}, Repository{Username: "foo"}},
{"SourcePassword", &Repository{}, &Repository{Password: "foo"}, Repository{Password: "foo"}},
{"SourceSSHPrivateKey", &Repository{}, &Repository{SSHPrivateKey: "foo"}, Repository{SSHPrivateKey: "foo"}},
{"SourceInsecureHostKey", &Repository{}, &Repository{InsecureIgnoreHostKey: true}, Repository{InsecureIgnoreHostKey: true}},
{"SourceInsecure", &Repository{}, &Repository{Insecure: true}, Repository{Insecure: true}},
{"SourceEnableLFS", &Repository{}, &Repository{EnableLFS: true}, Repository{EnableLFS: true}},
{"SourceTLSClientCAData", &Repository{}, &Repository{TLSClientCAData: "foo"}, Repository{TLSClientCAData: "foo"}},
{"SourceTLSClientCertData", &Repository{}, &Repository{TLSClientCertData: "foo"}, Repository{TLSClientCertData: "foo"}},
{"SourceTLSClientCertKey", &Repository{}, &Repository{TLSClientCertKey: "foo"}, Repository{TLSClientCertKey: "foo"}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
repo := Repository{}
repo.CopyCredentialsFrom(tt.source)
assert.Equal(t, tt.want, repo)
r := tt.repo.DeepCopy()
r.CopyCredentialsFrom(tt.source)
assert.Equal(t, tt.want, *r)
})
}
}

View file

@ -96,6 +96,7 @@ func (s *Server) List(ctx context.Context, q *repositorypkg.RepoQuery) (*appsv1.
}
err = util.RunAllAsync(len(items), func(i int) error {
items[i].ConnectionState = s.getConnectionState(ctx, items[i].Repo)
_ = factory.DetectType(items[i], metrics.NopReporter)
return nil
})
if err != nil {
@ -176,6 +177,7 @@ func (s *Server) Create(ctx context.Context, q *repositorypkg.RepoCreateRequest)
return nil, err
}
detectedType := ""
// check we can connect to the repo, copying any existing creds
{
repo := q.Repo.DeepCopy()
@ -184,13 +186,19 @@ func (s *Server) Create(ctx context.Context, q *repositorypkg.RepoCreateRequest)
return nil, err
}
repo.CopyCredentialsFrom(creds)
_, err = factory.NewFactory().NewRepo(q.Repo, metrics.NopReporter)
err = factory.DetectType(repo, metrics.NopReporter)
if err != nil {
return nil, err
}
detectedType = repo.Type
_, err = factory.NewFactory().NewRepo(repo, metrics.NopReporter)
if err != nil {
return nil, err
}
}
r := q.Repo
r.Type = detectedType
r.ConnectionState = appsv1.ConnectionState{Status: appsv1.ConnectionStatusSuccessful}
repo, err := s.db.CreateRepository(ctx, r)
if status.Convert(err).Code() == codes.AlreadyExists {
@ -256,7 +264,16 @@ func (s *Server) ValidateAccess(ctx context.Context, q *repositorypkg.RepoAccess
TLSClientCertKey: q.TlsClientCertKey,
TLSClientCAData: q.TlsClientCAData,
}
_, err := factory.NewFactory().NewRepo(repo, metrics.NopReporter)
creds, err := s.db.GetRepository(ctx, q.Repo)
if err != nil {
return nil, err
}
repo.CopyCredentialsFrom(creds)
err = factory.DetectType(repo, metrics.NopReporter)
if err != nil {
return nil, err
}
_, err = factory.NewFactory().NewRepo(repo, metrics.NopReporter)
if err != nil {
return nil, err
}

View file

@ -69,7 +69,7 @@ export const ApplicationSummary = (props: {
edit: (formApi: FormApi) => <FormField formApi={formApi} field='spec.source.targetRevision' component={Text}/>,
},
{
title: 'PATH',
title: 'GIT PATH/HELM CHART',
view: app.spec.source.path,
edit: (formApi: FormApi) => <FormField formApi={formApi} field='spec.source.path' component={Text}/>,
},

View file

@ -58,7 +58,7 @@ export const ApplicationTiles = ({applications, syncApplication, refreshApplicat
<div className='columns small-9'>{app.spec.source.targetRevision || 'latest'}</div>
</div>
<div className='row'>
<div className='columns small-3'>Path:</div>
<div className='columns small-3'>Git Path/Helm Chart:</div>
<div className='columns small-9'>{app.spec.source.path}</div>
</div>
<div className='row'>

View file

@ -77,11 +77,9 @@ export class ReposList extends React.Component<RouteComponentProps<any>> {
<div className='argo-table-list__row' key={repo.repo}>
<div className='row'>
<div className='columns small-1'>
<i className={'icon argo-icon-' + (repo.type || 'git')}/>
</div>
<div className='columns small-1'>
{repo.type || 'git'}
<i className={'icon argo-icon-' + (repo.type)}/>
</div>
<div className='columns small-1'>{repo.type}</div>
<div className='columns small-2'>{repo.name}</div>
<div className='columns small-5'>
<Repo url={repo.repo}/>
@ -145,7 +143,7 @@ export class ReposList extends React.Component<RouteComponentProps<any>> {
<FormField formApi={formApi} label='Type' field='type' component={FormSelect} componentProps={{options: ['git', 'helm']}}/>
</div>
<div className='argo-form-row'>
<FormField formApi={formApi} label='Name (optional for Git)' field='name' component={Text}/>
<FormField formApi={formApi} label='Name (mandatory for Helm)' field='name' component={Text}/>
</div>
<div className='argo-form-row'>
<FormField formApi={formApi} label='Repository URL' field='url' component={Text}/>
@ -186,6 +184,7 @@ export class ReposList extends React.Component<RouteComponentProps<any>> {
<h4>Connect repo using SSH</h4>
<Form onSubmit={(params) => this.connectSSHRepo(params as NewSSHRepoParams)}
getApi={(api) => this.formApiSSH = api}
defaultValues={{type: 'git'}}
validateError={(params: NewSSHRepoParams) => ({
url: !params.url && 'Repo URL is required',
})}>
@ -195,7 +194,7 @@ export class ReposList extends React.Component<RouteComponentProps<any>> {
<FormField formApi={formApi} label='Type' field='type' component={FormSelect} componentProps={{options: ['git', 'helm']}}/>
</div>
<div className='argo-form-row'>
<FormField formApi={formApi} label='Name (optional for Git)' field='name' component={Text}/>
<FormField formApi={formApi} label='Name (mandatory for Helm)' field='name' component={Text}/>
</div>
<div className='argo-form-row'>
<FormField formApi={formApi} label='Repository URL' field='url' component={Text}/>

View file

@ -5,7 +5,6 @@ import (
"hash/fnv"
"strings"
log "github.com/sirupsen/logrus"
"golang.org/x/net/context"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
@ -95,18 +94,13 @@ func (db *db) GetRepository(ctx context.Context, repoURL string) (*appsv1.Reposi
}
}
if !repo.HasCredentials() {
index := getRepositoryCredentialIndex(repoCredentials, repoURL)
if index >= 0 {
credential, err := db.credentialsToRepository(repoCredentials[index])
if err != nil {
return nil, err
} else {
log.WithFields(log.Fields{"repoURL": repo.Repo, "credUrl": credential.Repo}).Info("copying credentials")
repo.CopyCredentialsFrom(credential)
}
index = getRepositoryCredentialIndex(repoCredentials, repoURL)
if index >= 0 {
credential, err := db.credentialsToRepository(repoCredentials[index])
if err != nil {
return nil, err
} else {
repo.CopyCredentialsFrom(credential)
}
}

87
util/helm/repo/index.go Normal file
View file

@ -0,0 +1,87 @@
package repo
import (
"encoding/base64"
"errors"
"fmt"
"net/http"
"time"
"github.com/patrickmn/go-cache"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
)
var indexCache = cache.New(5*time.Minute, 5*time.Minute)
type entry struct {
Version string
Created time.Time
}
type index struct {
Entries map[string][]entry
}
func Index(url, username, password string) (*index, error) {
cachedIndex, found := indexCache.Get(url)
if found {
log.WithFields(log.Fields{"url": url}).Debug("index cache hit")
i := cachedIndex.(index)
return &i, nil
}
start := time.Now()
req, err := http.NewRequest("GET", url+"/index.yaml", nil)
if err != nil {
return nil, err
}
if username != "" {
// only basic supported
token := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", username, password)))
req.Header.Add("Authorization", fmt.Sprintf("Basic %s", token))
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, err
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != 200 {
return nil, errors.New("failed to get index: " + resp.Status)
}
index := &index{}
err = yaml.NewDecoder(resp.Body).Decode(index)
log.WithFields(log.Fields{"seconds": time.Since(start).Seconds()}).Info("took to get index")
indexCache.Set(url, *index, cache.DefaultExpiration)
return index, err
}
func (i *index) contains(chart string) bool {
_, ok := i.Entries[chart]
return ok
}
func (i *index) entry(chart, version string) (*entry, error) {
for _, entry := range i.Entries[chart] {
if entry.Version == version {
return &entry, nil
}
}
return nil, fmt.Errorf("unknown chart \"%s/%s\"", chart, version)
}
func (i *index) latest(chart string) (string, error) {
for chartName := range i.Entries {
if chartName == chart {
return i.Entries[chartName][0].Version, nil
}
}
return "", fmt.Errorf("failed to find chart %s", chart)
}

View file

@ -0,0 +1,24 @@
package repo
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestIndex(t *testing.T) {
t.Run("Invalid", func(t *testing.T) {
_, err := Index("", "", "")
assert.Error(t, err)
})
t.Run("Stable", func(t *testing.T) {
index, err := Index("https://kubernetes-charts.storage.googleapis.com", "", "")
assert.NoError(t, err)
assert.NotNil(t, index)
})
t.Run("BasicAuth", func(t *testing.T) {
index, err := Index("https://kubernetes-charts.storage.googleapis.com", "my-username", "my-password")
assert.NoError(t, err)
assert.NotNil(t, index)
})
}

View file

@ -1,23 +1,13 @@
package repo
import (
"errors"
"fmt"
"net/http"
"path/filepath"
"strings"
"time"
"github.com/patrickmn/go-cache"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
"github.com/argoproj/argo-cd/util/helm"
"github.com/argoproj/argo-cd/util/repo"
)
var indexCache = cache.New(5*time.Minute, 5*time.Minute)
type helmRepo struct {
cmd *helm.Cmd
url, name, username, password string
@ -25,7 +15,7 @@ type helmRepo struct {
}
func (c helmRepo) Init() error {
_, err := c.getIndex()
_, err := c.index()
if err != nil {
return err
}
@ -46,78 +36,28 @@ func (c helmRepo) ResolveAppRevision(app, revision string) (string, error) {
return revision, nil
}
index, err := c.getIndex()
index, err := c.index()
if err != nil {
return "", err
}
for chartName := range index.Entries {
if chartName == app {
return index.Entries[chartName][0].Version, nil
}
}
return "", errors.New("failed to find chart " + app)
return index.latest(app)
}
func (c helmRepo) RevisionMetadata(app, resolvedRevision string) (*repo.RevisionMetadata, error) {
index, err := c.getIndex()
index, err := c.index()
if err != nil {
return nil, err
}
for _, entry := range index.Entries[app] {
if entry.Version == resolvedRevision {
return &repo.RevisionMetadata{Date: entry.Created}, nil
}
}
return nil, fmt.Errorf("unknown chart \"%s/%s\"", app, resolvedRevision)
}
type entry struct {
Version string
Created time.Time
}
type index struct {
Entries map[string][]entry
}
func (c helmRepo) getIndex() (*index, error) {
cachedIndex, found := indexCache.Get(c.url)
if found {
log.WithFields(log.Fields{"url": c.url}).Debug("index cache hit")
i := cachedIndex.(index)
return &i, nil
}
start := time.Now()
resp, err := http.Get(strings.TrimSuffix(c.url, "/") + "/index.yaml")
entry, err := index.entry(app, resolvedRevision)
if err != nil {
return nil, err
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != 200 {
return nil, errors.New("failed to get index: " + resp.Status)
}
index := &index{}
err = yaml.NewDecoder(resp.Body).Decode(index)
log.WithFields(log.Fields{"seconds": time.Since(start).Seconds()}).Info("took to get index")
indexCache.Set(c.url, *index, cache.DefaultExpiration)
return index, err
return &repo.RevisionMetadata{Date: entry.Created}, nil
}
func (c helmRepo) ListApps(_ string) (map[string]string, error) {
index, err := c.getIndex()
index, err := c.index()
if err != nil {
return nil, err
}
@ -151,24 +91,16 @@ func (c helmRepo) GetApp(app string, resolvedRevision string) (string, error) {
}
func (c helmRepo) checkKnownChart(chartName string) error {
knownChart, err := c.isKnownChart(chartName)
index, err := c.index()
if err != nil {
return err
}
if !knownChart {
if !index.contains(chartName) {
return fmt.Errorf("unknown chart \"%s\"", chartName)
}
return nil
}
func (c helmRepo) isKnownChart(chartName string) (bool, error) {
index, err := c.getIndex()
if err != nil {
return false, err
}
_, ok := index.Entries[chartName]
return ok, nil
func (c helmRepo) index() (*index, error) {
return Index(c.url, c.username, c.password)
}

View file

@ -50,11 +50,6 @@ func NewRepo(url, name, username, password string, caData, certData, keyData []b
certData: certData,
keyData: keyData,
}
err = r.Init()
if err != nil {
cmd.Close()
return nil, err
}
repoCache.Set(url, r, cache.DefaultExpiration)
return r, nil
}

View file

@ -6,17 +6,11 @@ import (
"github.com/stretchr/testify/assert"
)
func TestRepoFactory(t *testing.T) {
func TestNewRepo(t *testing.T) {
t.Run("Unnamed", func(t *testing.T) {
_, err := NewRepo("http://0.0.0.0", "", "", "", nil, nil, nil)
assert.EqualError(t, err, "must name repo")
})
t.Run("GarbageRepo", func(t *testing.T) {
_, err := NewRepo("http://0.0.0.0", "test", "", "", nil, nil, nil)
assert.Error(t, err)
})
t.Run("Valid", func(t *testing.T) {
_, err := NewRepo("https://kubernetes-charts.storage.googleapis.com", "test", "", "", nil, nil, nil)
assert.NoError(t, err)

View file

@ -9,6 +9,8 @@ import (
func TestRepo(t *testing.T) {
repo, err := NewRepo("https://kubernetes-charts.storage.googleapis.com", "test", "", "", nil, nil, nil)
assert.NoError(t, err)
err = repo.Init()
assert.NoError(t, err)
// TODO - this changes regularly
const latestWordpressVersion = "5.8.0"
@ -32,7 +34,7 @@ func TestRepo(t *testing.T) {
assert.Equal(t, latestWordpressVersion, resolvedRevision)
})
t.Run("Checkout", func(t *testing.T) {
t.Run("GetApp", func(t *testing.T) {
appPath, err := repo.GetApp("wordpress", latestWordpressVersion)
assert.NoError(t, err)
assert.NotEmpty(t, appPath)

View file

@ -0,0 +1,30 @@
package factory
import (
log "github.com/sirupsen/logrus"
"github.com/argoproj/argo-cd/pkg/apis/application/v1alpha1"
helmrepo "github.com/argoproj/argo-cd/util/helm/repo"
"github.com/argoproj/argo-cd/util/repo/metrics"
)
func DetectType(r *v1alpha1.Repository, reporter metrics.Reporter) error {
log.WithField("repo", r).Info("DetectType")
if r.Type != "" {
return nil
}
_, err := helmrepo.Index(r.Repo, r.Username, r.Password)
if err == nil {
r.Type = "helm"
return nil
}
gitRepo, err := gitRepo(r, reporter)
if err == nil {
err = gitRepo.Init()
if err == nil {
r.Type = "git"
return nil
}
}
return err
}

View file

@ -0,0 +1,37 @@
package factory
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/argoproj/argo-cd/pkg/apis/application/v1alpha1"
"github.com/argoproj/argo-cd/util/repo/metrics"
)
func TestDetect(t *testing.T) {
t.Run("Invalid", func(t *testing.T) {
r := &v1alpha1.Repository{Repo: "invalid"}
err := DetectType(r, metrics.NopReporter)
assert.Error(t, err)
assert.Empty(t, r.Type)
})
t.Run("Explicit", func(t *testing.T) {
r := &v1alpha1.Repository{Type: "my-type"}
err := DetectType(r, metrics.NopReporter)
assert.NoError(t, err)
assert.Equal(t, "my-type", r.Type)
})
t.Run("Helm", func(t *testing.T) {
r := &v1alpha1.Repository{Repo: "https://kubernetes-charts.storage.googleapis.com", Name: "stable"}
err := DetectType(r, metrics.NopReporter)
assert.NoError(t, err)
assert.Equal(t, "helm", r.Type)
})
t.Run("Git", func(t *testing.T) {
r := &v1alpha1.Repository{Repo: "https://github.com/argoproj/argocd-example-apps"}
err := DetectType(r, metrics.NopReporter)
assert.NoError(t, err)
assert.Equal(t, "git", r.Type)
})
}

View file

@ -24,8 +24,16 @@ type factory struct {
func (f *factory) NewRepo(r *v1alpha1.Repository, reporter metrics.Reporter) (repo.Repo, error) {
switch r.Type {
case "helm":
return helmrepo.NewRepo(r.Repo, r.Name, r.Username, r.Password, []byte(r.TLSClientCAData), []byte(r.TLSClientCertData), []byte(r.TLSClientCertKey))
return helmRepo(r)
default:
return gitrepo.NewRepo(r.Repo, creds.GetRepoCreds(r), r.IsInsecure(), r.EnableLFS, discovery.Discover, reporter)
return gitRepo(r, reporter)
}
}
func gitRepo(r *v1alpha1.Repository, reporter metrics.Reporter) (repo.Repo, error) {
return gitrepo.NewRepo(r.Repo, creds.GetRepoCreds(r), r.IsInsecure(), r.EnableLFS, discovery.Discover, reporter)
}
func helmRepo(r *v1alpha1.Repository) (repo.Repo, error) {
return helmrepo.NewRepo(r.Repo, r.Name, r.Username, r.Password, []byte(r.TLSClientCAData), []byte(r.TLSClientCertData), []byte(r.TLSClientCertKey))
}