diff --git a/Gopkg.lock b/Gopkg.lock index 9f812de8a5..4735537222 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -583,6 +583,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "0aa93ce59362fb4c878e843ba8ebed06d419f8aa29012f099a213aded7b4940c" + inputs-digest = "477de633045a1e9822c4e605e845790162eb15d9a0d414156b790b2b359ab48b" solver-name = "gps-cdcl" solver-version = 1 diff --git a/cmd/argocd-server/commands/root.go b/cmd/argocd-server/commands/root.go index de17536599..7169b0382d 100644 --- a/cmd/argocd-server/commands/root.go +++ b/cmd/argocd-server/commands/root.go @@ -4,7 +4,7 @@ import ( "github.com/argoproj/argo-cd/errors" appclientset "github.com/argoproj/argo-cd/pkg/client/clientset/versioned" "github.com/argoproj/argo-cd/server" - "github.com/argoproj/argo-cd/util/cmd" + "github.com/argoproj/argo-cd/util/cli" "github.com/argoproj/argo-cd/util/kube" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -40,6 +40,6 @@ func NewCommand() *cobra.Command { command.Flags().StringVar(&kubeConfig, "kubeconfig", "", "Kubernetes config (used when running outside of cluster)") command.Flags().StringVar(&configMap, "configmap", defaultArgoCDConfigMap, "Name of K8s configmap to retrieve argocd configuration") command.Flags().StringVar(&logLevel, "loglevel", "info", "Set the logging level. One of: debug|info|warn|error") - command.AddCommand(cmd.NewVersionCmd(cliName)) + command.AddCommand(cli.NewVersionCmd(cliName)) return command } diff --git a/cmd/argocd/commands/repo.go b/cmd/argocd/commands/repo.go index 99d6ae9f4d..bb09d3e690 100644 --- a/cmd/argocd/commands/repo.go +++ b/cmd/argocd/commands/repo.go @@ -1,17 +1,21 @@ package commands import ( + "bufio" "context" "fmt" "os" + "syscall" "text/tabwriter" "github.com/argoproj/argo-cd/errors" appsv1 "github.com/argoproj/argo-cd/pkg/apis/application/v1alpha1" "github.com/argoproj/argo-cd/server/repository" "github.com/argoproj/argo-cd/util" + "github.com/argoproj/argo-cd/util/git" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "golang.org/x/crypto/ssh/terminal" "google.golang.org/grpc" ) @@ -34,6 +38,9 @@ func NewRepoCommand() *cobra.Command { // NewRepoAddCommand returns a new instance of an `argocd repo add` command func NewRepoAddCommand() *cobra.Command { + var ( + repo appsv1.Respository + ) var command = &cobra.Command{ Use: "add", Short: fmt.Sprintf("%s repo add REPO", cliName), @@ -42,19 +49,44 @@ func NewRepoAddCommand() *cobra.Command { c.HelpFunc()(c, args) os.Exit(1) } + repo.Repo = args[0] + err := git.TestRepo(repo.Repo, repo.Username, repo.Password) + if err != nil { + if repo.Username != "" && repo.Password != "" { + // if everything was supplied, one of the inputs was definitely bad + log.Fatal(err) + } + // If we can't test the repo, it's probably private. Prompt for credentials and try again. + promptCredentials(&repo) + err = git.TestRepo(repo.Repo, repo.Username, repo.Password) + } + errors.CheckError(err) conn, repoIf := NewRepoClient() defer util.Close(conn) - repo := &appsv1.Respository{ - Repo: args[0], - } - repo, err := repoIf.Create(context.Background(), repo) + createdRepo, err := repoIf.Create(context.Background(), &repo) errors.CheckError(err) - fmt.Printf("repository '%s' added\n", repo.Repo) + fmt.Printf("repository '%s' added\n", createdRepo.Repo) }, } + command.Flags().StringVar(&repo.Username, "username", "", "username to the repository") + command.Flags().StringVar(&repo.Password, "password", "", "password to the repository") return command } +func promptCredentials(repo *appsv1.Respository) { + reader := bufio.NewReader(os.Stdin) + if repo.Username == "" { + fmt.Print("Username: ") + username, _ := reader.ReadString('\n') + repo.Username = username + } + if repo.Password == "" { + fmt.Print("Password: ") + bytePassword, _ := terminal.ReadPassword(int(syscall.Stdin)) + repo.Password = string(bytePassword) + } +} + // NewRepoRemoveCommand returns a new instance of an `argocd repo list` command func NewRepoRemoveCommand() *cobra.Command { var command = &cobra.Command{ @@ -87,10 +119,11 @@ func NewRepoListCommand() *cobra.Command { repos, err := repoIf.List(context.Background(), &repository.RepoQuery{}) errors.CheckError(err) w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) - fmt.Fprintf(w, "REPO\n") + fmt.Fprintf(w, "REPO\tUSER\n") for _, r := range repos.Items { - fmt.Fprintf(w, "%s\n", r.Repo) + fmt.Fprintf(w, "%s\t%s\n", r.Repo, r.Username) } + w.Flush() }, } return command diff --git a/cmd/argocd/commands/root.go b/cmd/argocd/commands/root.go index 6fc9ef0492..84beae52d0 100644 --- a/cmd/argocd/commands/root.go +++ b/cmd/argocd/commands/root.go @@ -1,7 +1,7 @@ package commands import ( - "github.com/argoproj/argo-cd/util/cmd" + "github.com/argoproj/argo-cd/util/cli" "github.com/spf13/cobra" "k8s.io/client-go/tools/clientcmd" // load the gcp plugin (required to authenticate against GKE clusters). @@ -34,7 +34,7 @@ func NewCommand() *cobra.Command { clientcmd.BindOverrideFlags(&globalArgs.kubeConfigOverrides, command.PersistentFlags(), clientcmd.RecommendedConfigOverrideFlags("")) command.PersistentFlags().StringVar(&globalArgs.logLevel, "loglevel", "info", "Set the logging level. One of: debug|info|warn|error") - command.AddCommand(cmd.NewVersionCmd(cliName)) + command.AddCommand(cli.NewVersionCmd(cliName)) command.AddCommand(NewClusterCommand()) command.AddCommand(NewApplicationCommand()) command.AddCommand(NewRepoCommand()) diff --git a/common/common.go b/common/common.go index 1c428d35e6..454197e4c8 100644 --- a/common/common.go +++ b/common/common.go @@ -3,9 +3,12 @@ package common const ( // MetadataPrefix is the prefix used for our labels and annotations MetadataPrefix = "argocd.argoproj.io" + + // SecretTypeRepository indicates the data type which argocd stores as a k8s secret + SecretTypeRepository = "repository" ) var ( - // LabelKeyRepo contains the repository URL - LabelKeyRepo = MetadataPrefix + "/repo" + // LabelKeySecretType contains the type of argocd secret (currently this is just 'repo') + LabelKeySecretType = MetadataPrefix + "/secret-type" ) diff --git a/errors/errors.go b/errors/errors.go index 83a31fa8c7..349ac66bd2 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -4,7 +4,7 @@ import ( log "github.com/sirupsen/logrus" ) -// CheckError is a convenience function to exit if there was error is non-nil +// CheckError is a convenience function to exit if an error is non-nil and exit if it was func CheckError(err error) { if err != nil { log.Fatal(err) diff --git a/server/repository/repository.go b/server/repository/repository.go index 9a79d6834a..a4a832ffcd 100644 --- a/server/repository/repository.go +++ b/server/repository/repository.go @@ -1,12 +1,19 @@ package repository import ( + "fmt" + "hash/fnv" + "strings" + "github.com/argoproj/argo-cd/common" appsv1 "github.com/argoproj/argo-cd/pkg/apis/application/v1alpha1" appclientset "github.com/argoproj/argo-cd/pkg/client/clientset/versioned" "github.com/argoproj/argo-cd/util/git" "golang.org/x/net/context" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" apiv1 "k8s.io/api/core/v1" + apierr "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/selection" @@ -33,7 +40,10 @@ func NewServer(namespace string, kubeclientset kubernetes.Interface, appclientse func (s *Server) List(ctx context.Context, q *RepoQuery) (*appsv1.RespositoryList, error) { listOpts := metav1.ListOptions{} labelSelector := labels.NewSelector() - req, _ := labels.NewRequirement(common.LabelKeyRepo, selection.Exists, nil) + req, err := labels.NewRequirement(common.LabelKeySecretType, selection.Equals, []string{common.SecretTypeRepository}) + if err != nil { + return nil, err + } labelSelector = labelSelector.Add(*req) listOpts.LabelSelector = labelSelector.String() repoSecrets, err := s.kubeclientset.CoreV1().Secrets(s.ns).List(listOpts) @@ -52,20 +62,28 @@ func (s *Server) List(ctx context.Context, q *RepoQuery) (*appsv1.RespositoryLis // Create creates a repository func (s *Server) Create(ctx context.Context, r *appsv1.Respository) (*appsv1.Respository, error) { repoURL := git.NormalizeGitURL(r.Repo) - cmName := repoURLToSecretName(repoURL) + err := git.TestRepo(repoURL, r.Username, r.Password) + if err != nil { + return nil, err + } + secName := repoURLToSecretName(repoURL) repoSecret := &apiv1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: cmName, + Name: secName, Labels: map[string]string{ - common.LabelKeyRepo: repoURL, + common.LabelKeySecretType: common.SecretTypeRepository, }, }, } repoSecret.StringData = make(map[string]string) + repoSecret.StringData["repository"] = repoURL repoSecret.StringData["username"] = r.Username repoSecret.StringData["password"] = r.Password - repoSecret, err := s.kubeclientset.CoreV1().Secrets(s.ns).Create(repoSecret) + repoSecret, err = s.kubeclientset.CoreV1().Secrets(s.ns).Create(repoSecret) if err != nil { + if apierr.IsAlreadyExists(err) { + return nil, grpc.Errorf(codes.AlreadyExists, "repository '%s' already exists", repoURL) + } return nil, err } return secretToRepo(repoSecret), nil @@ -73,8 +91,8 @@ func (s *Server) Create(ctx context.Context, r *appsv1.Respository) (*appsv1.Res // Get returns a repository by URL func (s *Server) Get(ctx context.Context, q *RepoQuery) (*appsv1.Respository, error) { - cmName := repoURLToSecretName(q.Repo) - repoSecret, err := s.kubeclientset.CoreV1().Secrets(s.ns).Get(cmName, metav1.GetOptions{}) + secName := repoURLToSecretName(q.Repo) + repoSecret, err := s.kubeclientset.CoreV1().Secrets(s.ns).Get(secName, metav1.GetOptions{}) if err != nil { return nil, err } @@ -83,14 +101,21 @@ func (s *Server) Get(ctx context.Context, q *RepoQuery) (*appsv1.Respository, er // Update updates a repository func (s *Server) Update(ctx context.Context, r *appsv1.Respository) (*appsv1.Respository, error) { - cmName := repoURLToSecretName(r.Repo) - repoSecret, err := s.kubeclientset.CoreV1().Secrets(s.ns).Get(cmName, metav1.GetOptions{}) + secName := repoURLToSecretName(r.Repo) + repoSecret, err := s.kubeclientset.CoreV1().Secrets(s.ns).Get(secName, metav1.GetOptions{}) if err != nil { return nil, err } repoSecret.StringData = make(map[string]string) + repoSecret.StringData["repository"] = r.Repo repoSecret.StringData["username"] = r.Username repoSecret.StringData["password"] = r.Password + + err = git.TestRepo(r.Repo, r.Username, r.Password) + if err != nil { + return nil, err + } + repoSecret, err = s.kubeclientset.CoreV1().Secrets(s.ns).Update(repoSecret) if err != nil { return nil, err @@ -105,22 +130,26 @@ func (s *Server) UpdateREST(ctx context.Context, r *RepoUpdateRequest) (*appsv1. // Delete updates a repository func (s *Server) Delete(ctx context.Context, q *RepoQuery) (*RepoResponse, error) { - cmName := repoURLToSecretName(q.Repo) - err := s.kubeclientset.CoreV1().Secrets(s.ns).Delete(cmName, &metav1.DeleteOptions{}) + secName := repoURLToSecretName(q.Repo) + err := s.kubeclientset.CoreV1().Secrets(s.ns).Delete(secName, &metav1.DeleteOptions{}) return &RepoResponse{}, err } -// repoURLToSecretName converts a repo +// repoURLToSecretName hashes repo URL to the secret name using formula +// part of the original repo name is incorporated for debugging purposes func repoURLToSecretName(repo string) string { - repoURL := git.NormalizeGitURL(repo) - return repoURL + repo = git.NormalizeGitURL(repo) + h := fnv.New32a() + _, _ = h.Write([]byte(repo)) + parts := strings.Split(strings.TrimSuffix(repo, ".git"), "/") + return fmt.Sprintf("repo-%s-%v", parts[len(parts)-1], h.Sum32()) } // secretToRepo converts a secret into a repository object func secretToRepo(s *apiv1.Secret) *appsv1.Respository { repo := appsv1.Respository{ - Repo: s.ObjectMeta.Labels[common.LabelKeyRepo], + Repo: string(s.Data["repository"]), Username: string(s.Data["username"]), Password: string(s.Data["password"]), } diff --git a/server/server.go b/server/server.go index e7b69e3b35..610511db2a 100644 --- a/server/server.go +++ b/server/server.go @@ -12,6 +12,7 @@ import ( "github.com/argoproj/argo-cd/server/cluster" "github.com/argoproj/argo-cd/server/repository" "github.com/argoproj/argo-cd/server/version" + grpc_util "github.com/argoproj/argo-cd/util/grpc" jsonutil "github.com/argoproj/argo-cd/util/json" grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus" @@ -71,9 +72,11 @@ func (a *ArgoCDServer) Run() { grpcS := grpc.NewServer( grpc.StreamInterceptor(grpc_middleware.ChainStreamServer( grpc_logrus.StreamServerInterceptor(a.log), + grpc_util.PanicLoggerStreamServerInterceptor(a.log), )), grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer( grpc_logrus.UnaryServerInterceptor(a.log), + grpc_util.PanicLoggerUnaryServerInterceptor(a.log), )), ) version.RegisterVersionServiceServer(grpcS, &version.Server{}) diff --git a/util/cmd/cmd.go b/util/cli/cli.go similarity index 98% rename from util/cmd/cmd.go rename to util/cli/cli.go index 68192a1771..e7dc766c59 100644 --- a/util/cmd/cmd.go +++ b/util/cli/cli.go @@ -1,6 +1,6 @@ // Package cmd provides functionally common to various argo CLIs -package cmd +package cli import ( "fmt" diff --git a/util/git/git.go b/util/git/git.go index 497d94e0a8..b76dcc65fe 100644 --- a/util/git/git.go +++ b/util/git/git.go @@ -1,7 +1,46 @@ package git +import ( + "fmt" + "net/url" + "os" + "os/exec" + "regexp" + "strings" +) + // NormalizeGitURL normalizes a git URL for lookup and storage -func NormalizeGitURL(repoURL string) string { +func NormalizeGitURL(repo string) string { // TODO: implement this - return repoURL + repo = strings.TrimSpace(repo) + return repo +} + +// TestRepo tests if a repo exists and is accessible with the given credentials +func TestRepo(repo, username, password string) error { + repoURL, err := url.ParseRequestURI(repo) + if err != nil { + return err + } + repoURL.User = url.UserPassword(username, password) + cmd := exec.Command("git", "ls-remote", repoURL.String(), "HEAD") + env := os.Environ() + env = append(env, "GIT_ASKPASS=") + cmd.Env = env + _, err = cmd.Output() + if err != nil { + exErr := err.(*exec.ExitError) + errOutput := strings.Split(string(exErr.Stderr), "\n")[0] + errOutput = redactPassword(errOutput, password) + return fmt.Errorf("failed to test %s: %s", repo, errOutput) + } + return nil +} + +func redactPassword(msg string, password string) string { + if password != "" { + passwordRegexp := regexp.MustCompile("\\b" + regexp.QuoteMeta(password) + "\\b") + msg = passwordRegexp.ReplaceAllString(msg, "*****") + } + return msg } diff --git a/util/grpc/grpc.go b/util/grpc/grpc.go new file mode 100644 index 0000000000..fde7e85b5f --- /dev/null +++ b/util/grpc/grpc.go @@ -0,0 +1,36 @@ +package grpc + +import ( + "runtime/debug" + + "github.com/sirupsen/logrus" + "golang.org/x/net/context" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" +) + +// PanicLoggerUnaryServerInterceptor returns a new unary server interceptor for recovering from panics and returning error +func PanicLoggerUnaryServerInterceptor(log *logrus.Entry) grpc.UnaryServerInterceptor { + return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (_ interface{}, err error) { + defer func() { + if r := recover(); r != nil { + log.Errorf("Recovered from panic: %+v\n%s", r, debug.Stack()) + err = grpc.Errorf(codes.Internal, "%s", r) + } + }() + return handler(ctx, req) + } +} + +// PanicLoggerStreamServerInterceptor returns a new streaming server interceptor for recovering from panics and returning error +func PanicLoggerStreamServerInterceptor(log *logrus.Entry) grpc.StreamServerInterceptor { + return func(srv interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) (err error) { + defer func() { + if r := recover(); r != nil { + log.Errorf("Recovered from panic: %+v\n%s", r, debug.Stack()) + err = grpc.Errorf(codes.Internal, "%s", r) + } + }() + return handler(srv, stream) + } +}