fix(server): Cache glob patterns to improve RBAC evaluation performance (#25759)

Signed-off-by: Sinhyeok Seo <sinhyeok@gmail.com>
Signed-off-by: Sinhyeok Seo <44961659+Sinhyeok@users.noreply.github.com>
This commit is contained in:
Sinhyeok Seo 2026-03-17 23:22:23 +09:00 committed by GitHub
parent 8142920ab8
commit 382c507beb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
17 changed files with 410 additions and 6 deletions

View file

@ -34,6 +34,7 @@ import (
"github.com/argoproj/argo-cd/v3/util/dex"
"github.com/argoproj/argo-cd/v3/util/env"
"github.com/argoproj/argo-cd/v3/util/errors"
utilglob "github.com/argoproj/argo-cd/v3/util/glob"
"github.com/argoproj/argo-cd/v3/util/kube"
"github.com/argoproj/argo-cd/v3/util/templates"
"github.com/argoproj/argo-cd/v3/util/tls"
@ -87,6 +88,7 @@ func NewCommand() *cobra.Command {
applicationNamespaces []string
enableProxyExtension bool
webhookParallelism int
globCacheSize int
hydratorEnabled bool
syncWithReplaceAllowed bool
@ -122,6 +124,7 @@ func NewCommand() *cobra.Command {
cli.SetLogFormat(cmdutil.LogFormat)
cli.SetLogLevel(cmdutil.LogLevel)
cli.SetGLogLevel(glogLevel)
utilglob.SetCacheSize(globCacheSize)
// Recover from panic and log the error using the configured logger instead of the default.
defer func() {
@ -326,6 +329,7 @@ func NewCommand() *cobra.Command {
command.Flags().StringSliceVar(&applicationNamespaces, "application-namespaces", env.StringsFromEnv("ARGOCD_APPLICATION_NAMESPACES", []string{}, ","), "List of additional namespaces where application resources can be managed in")
command.Flags().BoolVar(&enableProxyExtension, "enable-proxy-extension", env.ParseBoolFromEnv("ARGOCD_SERVER_ENABLE_PROXY_EXTENSION", false), "Enable Proxy Extension feature")
command.Flags().IntVar(&webhookParallelism, "webhook-parallelism-limit", env.ParseNumFromEnv("ARGOCD_SERVER_WEBHOOK_PARALLELISM_LIMIT", 50, 1, 1000), "Number of webhook requests processed concurrently")
command.Flags().IntVar(&globCacheSize, "glob-cache-size", env.ParseNumFromEnv("ARGOCD_SERVER_GLOB_CACHE_SIZE", utilglob.DefaultGlobCacheSize, 1, math.MaxInt32), "Maximum number of compiled glob patterns to cache for RBAC evaluation")
command.Flags().StringSliceVar(&enableK8sEvent, "enable-k8s-event", env.StringsFromEnv("ARGOCD_ENABLE_K8S_EVENT", argo.DefaultEnableEventList(), ","), "Enable ArgoCD to use k8s event. For disabling all events, set the value as `none`. (e.g --enable-k8s-event=none), For enabling specific events, set the value as `event reason`. (e.g --enable-k8s-event=StatusRefreshed,ResourceCreated)")
command.Flags().BoolVar(&hydratorEnabled, "hydrator-enabled", env.ParseBoolFromEnv("ARGOCD_HYDRATOR_ENABLED", false), "Feature flag to enable Hydrator. Default (\"false\")")
command.Flags().BoolVar(&syncWithReplaceAllowed, "sync-with-replace-allowed", env.ParseBoolFromEnv("ARGOCD_SYNC_WITH_REPLACE_ALLOWED", true), "Whether to allow users to select replace for syncs from UI/CLI")

View file

@ -150,6 +150,8 @@ data:
server.api.content.types: "application/json"
# Number of webhook requests processed concurrently (default 50)
server.webhook.parallelism.limit: "50"
# Maximum number of compiled glob patterns to cache for RBAC evaluation (default 10000)
server.glob.cache.size: "10000"
# Whether to allow sync with replace checked to go through. Resource-level annotation to replace override this setting, i.e. it's only enforced on the API server level.
server.sync.replace.allowed: "true"

View file

@ -253,6 +253,11 @@ spec:
megabytes.
The default value is 200. You might need to increase this for an Argo CD instance that manages 3000+ applications.
* The `server.glob.cache.size` config key in `argocd-cmd-params-cm` (or the `--glob-cache-size` server flag) controls
the maximum number of compiled glob patterns cached for RBAC policy evaluation. Glob pattern compilation is expensive,
and caching significantly improves RBAC performance when many applications are managed. The default value is 10000.
See [RBAC Glob Matching](rbac.md#glob-matching) for more details.
### argocd-dex-server, argocd-redis
The `argocd-dex-server` uses an in-memory database, and two or more instances may have inconsistent data.

View file

@ -321,6 +321,10 @@ When the `example-user` executes the `extensions/DaemonSet/test` action, the fol
3. The value `action/extensions/DaemonSet/test` matches `action/extensions/*`. Note that `/` is not treated as a separator and the use of `**` is not necessary.
4. The value `default/my-app` matches `default/*`.
> [!TIP]
> For performance tuning of glob pattern matching, see the `server.glob.cache.size` config key in
> [High Availability - argocd-server](high_availability.md#argocd-server).
## Using SSO Users/Groups
The `scopes` field controls which OIDC scopes to examine during RBAC enforcement (in addition to `sub` scope).

View file

@ -54,6 +54,7 @@ argocd-server [flags]
--enable-gzip Enable GZIP compression (default true)
--enable-k8s-event none Enable ArgoCD to use k8s event. For disabling all events, set the value as none. (e.g --enable-k8s-event=none), For enabling specific events, set the value as `event reason`. (e.g --enable-k8s-event=StatusRefreshed,ResourceCreated) (default [all])
--enable-proxy-extension Enable Proxy Extension feature
--glob-cache-size int Maximum number of compiled glob patterns to cache for RBAC evaluation (default 10000)
--gloglevel int Set the glog logging level
-h, --help help for argocd-server
--hydrator-enabled Feature flag to enable Hydrator. Default ("false")

2
go.mod
View file

@ -45,6 +45,7 @@ require (
github.com/gogits/go-gogs-client v0.0.0-20210131175652-1d7215cd8d85
github.com/gogo/protobuf v1.3.2
github.com/golang-jwt/jwt/v5 v5.3.1
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8
github.com/golang/protobuf v1.5.4
github.com/google/btree v1.1.3
github.com/google/gnostic-models v0.7.0 // indirect
@ -208,7 +209,6 @@ require (
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
github.com/golang/glog v1.2.5 // indirect
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
github.com/google/go-querystring v1.2.0 // indirect
github.com/google/s2a-go v0.1.9 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect

View file

@ -316,6 +316,12 @@ spec:
name: argocd-cmd-params-cm
key: server.webhook.parallelism.limit
optional: true
- name: ARGOCD_SERVER_GLOB_CACHE_SIZE
valueFrom:
configMapKeyRef:
name: argocd-cmd-params-cm
key: server.glob.cache.size
optional: true
- name: ARGOCD_APPLICATIONSET_CONTROLLER_ENABLE_NEW_GIT_FILE_GLOBBING
valueFrom:
configMapKeyRef:

View file

@ -34058,6 +34058,12 @@ spec:
key: server.webhook.parallelism.limit
name: argocd-cmd-params-cm
optional: true
- name: ARGOCD_SERVER_GLOB_CACHE_SIZE
valueFrom:
configMapKeyRef:
key: server.glob.cache.size
name: argocd-cmd-params-cm
optional: true
- name: ARGOCD_APPLICATIONSET_CONTROLLER_ENABLE_NEW_GIT_FILE_GLOBBING
valueFrom:
configMapKeyRef:

View file

@ -33888,6 +33888,12 @@ spec:
key: server.webhook.parallelism.limit
name: argocd-cmd-params-cm
optional: true
- name: ARGOCD_SERVER_GLOB_CACHE_SIZE
valueFrom:
configMapKeyRef:
key: server.glob.cache.size
name: argocd-cmd-params-cm
optional: true
- name: ARGOCD_APPLICATIONSET_CONTROLLER_ENABLE_NEW_GIT_FILE_GLOBBING
valueFrom:
configMapKeyRef:

View file

@ -3305,6 +3305,12 @@ spec:
key: server.webhook.parallelism.limit
name: argocd-cmd-params-cm
optional: true
- name: ARGOCD_SERVER_GLOB_CACHE_SIZE
valueFrom:
configMapKeyRef:
key: server.glob.cache.size
name: argocd-cmd-params-cm
optional: true
- name: ARGOCD_APPLICATIONSET_CONTROLLER_ENABLE_NEW_GIT_FILE_GLOBBING
valueFrom:
configMapKeyRef:

View file

@ -3135,6 +3135,12 @@ spec:
key: server.webhook.parallelism.limit
name: argocd-cmd-params-cm
optional: true
- name: ARGOCD_SERVER_GLOB_CACHE_SIZE
valueFrom:
configMapKeyRef:
key: server.glob.cache.size
name: argocd-cmd-params-cm
optional: true
- name: ARGOCD_APPLICATIONSET_CONTROLLER_ENABLE_NEW_GIT_FILE_GLOBBING
valueFrom:
configMapKeyRef:

View file

@ -33026,6 +33026,12 @@ spec:
key: server.webhook.parallelism.limit
name: argocd-cmd-params-cm
optional: true
- name: ARGOCD_SERVER_GLOB_CACHE_SIZE
valueFrom:
configMapKeyRef:
key: server.glob.cache.size
name: argocd-cmd-params-cm
optional: true
- name: ARGOCD_APPLICATIONSET_CONTROLLER_ENABLE_NEW_GIT_FILE_GLOBBING
valueFrom:
configMapKeyRef:

View file

@ -32854,6 +32854,12 @@ spec:
key: server.webhook.parallelism.limit
name: argocd-cmd-params-cm
optional: true
- name: ARGOCD_SERVER_GLOB_CACHE_SIZE
valueFrom:
configMapKeyRef:
key: server.glob.cache.size
name: argocd-cmd-params-cm
optional: true
- name: ARGOCD_APPLICATIONSET_CONTROLLER_ENABLE_NEW_GIT_FILE_GLOBBING
valueFrom:
configMapKeyRef:

View file

@ -2273,6 +2273,12 @@ spec:
key: server.webhook.parallelism.limit
name: argocd-cmd-params-cm
optional: true
- name: ARGOCD_SERVER_GLOB_CACHE_SIZE
valueFrom:
configMapKeyRef:
key: server.glob.cache.size
name: argocd-cmd-params-cm
optional: true
- name: ARGOCD_APPLICATIONSET_CONTROLLER_ENABLE_NEW_GIT_FILE_GLOBBING
valueFrom:
configMapKeyRef:

View file

@ -2101,6 +2101,12 @@ spec:
key: server.webhook.parallelism.limit
name: argocd-cmd-params-cm
optional: true
- name: ARGOCD_SERVER_GLOB_CACHE_SIZE
valueFrom:
configMapKeyRef:
key: server.glob.cache.size
name: argocd-cmd-params-cm
optional: true
- name: ARGOCD_APPLICATIONSET_CONTROLLER_ENABLE_NEW_GIT_FILE_GLOBBING
valueFrom:
configMapKeyRef:

View file

@ -1,26 +1,108 @@
package glob
import (
"sync"
"github.com/gobwas/glob"
"github.com/golang/groupcache/lru"
log "github.com/sirupsen/logrus"
"golang.org/x/sync/singleflight"
)
const (
// DefaultGlobCacheSize is the default maximum number of compiled glob patterns to cache.
// This limit prevents memory exhaustion from untrusted RBAC patterns.
// 10,000 patterns should be sufficient for most deployments while limiting
// memory usage to roughly ~10MB (assuming ~1KB per compiled pattern).
DefaultGlobCacheSize = 10000
)
type compileFn func(pattern string, separators ...rune) (glob.Glob, error)
var (
// globCache stores compiled glob patterns using an LRU cache with bounded size.
// This prevents memory exhaustion from potentially untrusted RBAC patterns
// while still providing significant performance benefits.
globCache *lru.Cache
globCacheLock sync.Mutex
compileGroup singleflight.Group
compileGlob compileFn = glob.Compile
)
func init() {
globCache = lru.New(DefaultGlobCacheSize)
}
// SetCacheSize reinitializes the glob cache with the given maximum number of entries.
// This should be called early during process startup, before concurrent access begins.
func SetCacheSize(maxEntries int) {
globCacheLock.Lock()
defer globCacheLock.Unlock()
globCache = lru.New(maxEntries)
}
// globCacheKey uniquely identifies a compiled glob pattern.
// The same pattern compiled with different separators produces different globs,
// so both fields are needed.
type globCacheKey struct {
Pattern string
Separators string
}
func cacheKey(pattern string, separators ...rune) globCacheKey {
return globCacheKey{Pattern: pattern, Separators: string(separators)}
}
// getOrCompile returns a cached compiled glob pattern, compiling and caching it if necessary.
// Cache hits are a brief lock + map lookup. On cache miss, singleflight ensures each
// unique pattern is compiled exactly once even under concurrent access, while unrelated
// patterns compile in parallel.
// lru.Cache.Get() promotes entries (mutating), so a Mutex is used rather than RWMutex.
func getOrCompile(pattern string, compiler compileFn, separators ...rune) (glob.Glob, error) {
key := cacheKey(pattern, separators...)
globCacheLock.Lock()
if cached, ok := globCache.Get(key); ok {
globCacheLock.Unlock()
return cached.(glob.Glob), nil
}
globCacheLock.Unlock()
sfKey := key.Pattern + "\x00" + key.Separators
v, err, _ := compileGroup.Do(sfKey, func() (any, error) {
compiled, err := compiler(pattern, separators...)
if err != nil {
return nil, err
}
globCacheLock.Lock()
globCache.Add(key, compiled)
globCacheLock.Unlock()
return compiled, nil
})
if err != nil {
return nil, err
}
return v.(glob.Glob), nil
}
// Match tries to match a text with a given glob pattern.
// Compiled glob patterns are cached for performance.
func Match(pattern, text string, separators ...rune) bool {
compiledGlob, err := glob.Compile(pattern, separators...)
compiled, err := getOrCompile(pattern, compileGlob, separators...)
if err != nil {
log.Warnf("failed to compile pattern %s due to error %v", pattern, err)
return false
}
return compiledGlob.Match(text)
return compiled.Match(text)
}
// MatchWithError tries to match a text with a given glob pattern.
// returns error if the glob pattern fails to compile.
// Returns error if the glob pattern fails to compile.
// Compiled glob patterns are cached for performance.
func MatchWithError(pattern, text string, separators ...rune) (bool, error) {
compiledGlob, err := glob.Compile(pattern, separators...)
compiled, err := getOrCompile(pattern, compileGlob, separators...)
if err != nil {
return false, err
}
return compiledGlob.Match(text), nil
return compiled.Match(text), nil
}

View file

@ -1,11 +1,57 @@
package glob
import (
"errors"
"fmt"
"sync"
"sync/atomic"
"testing"
extglob "github.com/gobwas/glob"
"github.com/stretchr/testify/require"
)
// Test helpers - these access internal variables for testing purposes
// resetGlobCacheForTest clears the cached glob patterns for testing.
func resetGlobCacheForTest() {
globCacheLock.Lock()
defer globCacheLock.Unlock()
globCache.Clear()
}
// isPatternCached returns true if the pattern (with optional separators) is cached.
func isPatternCached(pattern string, separators ...rune) bool {
globCacheLock.Lock()
defer globCacheLock.Unlock()
_, ok := globCache.Get(cacheKey(pattern, separators...))
return ok
}
// globCacheLen returns the number of cached patterns.
func globCacheLen() int {
globCacheLock.Lock()
defer globCacheLock.Unlock()
return globCache.Len()
}
func matchWithCompiler(pattern, text string, compiler compileFn, separators ...rune) bool {
compiled, err := getOrCompile(pattern, compiler, separators...)
if err != nil {
return false
}
return compiled.Match(text)
}
func countingCompiler() (compileFn, *int32) {
var compileCount int32
compiler := func(pattern string, separators ...rune) (extglob.Glob, error) {
atomic.AddInt32(&compileCount, 1)
return extglob.Compile(pattern, separators...)
}
return compiler, &compileCount
}
func Test_Match(t *testing.T) {
tests := []struct {
name string
@ -86,3 +132,209 @@ func Test_MatchWithError(t *testing.T) {
})
}
}
func Test_GlobCaching(t *testing.T) {
// Clear cache before test
resetGlobCacheForTest()
compiler, compileCount := countingCompiler()
pattern := "test*pattern"
text := "testABCpattern"
// First call should compile and cache
result1 := matchWithCompiler(pattern, text, compiler)
require.True(t, result1)
// Verify pattern is cached
require.True(t, isPatternCached(pattern), "pattern should be cached after first Match call")
// Second call should use cached value
result2 := matchWithCompiler(pattern, text, compiler)
require.True(t, result2)
// Results should be consistent
require.Equal(t, result1, result2)
require.Equal(t, int32(1), atomic.LoadInt32(compileCount), "glob should compile once for the cached pattern")
}
func Test_GlobCachingConcurrent(t *testing.T) {
// Clear cache before test
resetGlobCacheForTest()
compiler, compileCount := countingCompiler()
pattern := "concurrent*test"
text := "concurrentABCtest"
var wg sync.WaitGroup
numGoroutines := 100
errChan := make(chan error, numGoroutines)
for range numGoroutines {
wg.Go(func() {
result := matchWithCompiler(pattern, text, compiler)
if !result {
errChan <- errors.New("expected match to return true")
}
})
}
wg.Wait()
close(errChan)
// Check for any errors from goroutines
for err := range errChan {
t.Error(err)
}
// Verify pattern is cached
require.True(t, isPatternCached(pattern))
require.Equal(t, 1, globCacheLen(), "should only have one cached entry for the pattern")
require.Equal(t, int32(1), atomic.LoadInt32(compileCount), "glob should compile once for the cached pattern")
}
func Test_GlobCacheLRUEviction(t *testing.T) {
// Clear cache before test
resetGlobCacheForTest()
// Fill cache beyond DefaultGlobCacheSize
for i := range DefaultGlobCacheSize + 100 {
pattern := fmt.Sprintf("pattern-%d-*", i)
Match(pattern, "pattern-0-test")
}
// Cache size should be limited to DefaultGlobCacheSize
require.Equal(t, DefaultGlobCacheSize, globCacheLen(), "cache size should be limited to DefaultGlobCacheSize")
// The oldest patterns should be evicted
oldest := fmt.Sprintf("pattern-%d-*", 0)
require.False(t, isPatternCached(oldest), "oldest pattern should be evicted")
// The most recently used patterns should still be cached
require.True(t, isPatternCached(fmt.Sprintf("pattern-%d-*", DefaultGlobCacheSize+99)), "most recent pattern should be cached")
}
func Test_GlobCacheKeyIncludesSeparators(t *testing.T) {
resetGlobCacheForTest()
compiler, compileCount := countingCompiler()
pattern := "a*b"
textWithSlash := "a/b"
// Without separators, '*' matches '/' so "a/b" matches "a*b"
require.True(t, matchWithCompiler(pattern, textWithSlash, compiler))
require.Equal(t, int32(1), atomic.LoadInt32(compileCount))
// With separator '/', '*' does NOT match '/' so "a/b" should NOT match "a*b"
require.False(t, matchWithCompiler(pattern, textWithSlash, compiler, '/'))
require.Equal(t, int32(2), atomic.LoadInt32(compileCount), "same pattern with different separators must compile separately")
// Both entries should be independently cached
require.True(t, isPatternCached(pattern))
require.True(t, isPatternCached(pattern, '/'))
require.Equal(t, 2, globCacheLen())
// Subsequent calls should use cache (no additional compiles)
matchWithCompiler(pattern, textWithSlash, compiler)
matchWithCompiler(pattern, textWithSlash, compiler, '/')
require.Equal(t, int32(2), atomic.LoadInt32(compileCount), "cached patterns should not recompile")
}
func Test_InvalidGlobNotCached(t *testing.T) {
// Clear cache before test
resetGlobCacheForTest()
invalidPattern := "e[[a*"
text := "test"
// Match should return false for invalid pattern
result := Match(invalidPattern, text)
require.False(t, result)
// Invalid patterns should NOT be cached
require.False(t, isPatternCached(invalidPattern), "invalid pattern should not be cached")
// Also test with MatchWithError
_, err := MatchWithError(invalidPattern, text)
require.Error(t, err)
// Still should not be cached after MatchWithError
require.False(t, isPatternCached(invalidPattern), "invalid pattern should not be cached after MatchWithError")
}
func Test_SetCacheSize(t *testing.T) {
resetGlobCacheForTest()
customSize := 5
SetCacheSize(customSize)
defer SetCacheSize(DefaultGlobCacheSize)
for i := range customSize + 3 {
Match(fmt.Sprintf("setsize-%d-*", i), "setsize-0-test")
}
require.Equal(t, customSize, globCacheLen(), "cache size should respect the custom size set via SetCacheSize")
require.False(t, isPatternCached("setsize-0-*"), "oldest pattern should be evicted with custom cache size")
require.True(t, isPatternCached(fmt.Sprintf("setsize-%d-*", customSize+2)), "most recent pattern should be cached")
}
// BenchmarkMatch_WithCache benchmarks Match with caching (cache hit)
func BenchmarkMatch_WithCache(b *testing.B) {
pattern := "proj:*/app-*"
text := "proj:myproject/app-frontend"
// Warm up the cache
Match(pattern, text)
b.ResetTimer()
for i := 0; i < b.N; i++ {
Match(pattern, text)
}
}
// BenchmarkMatch_WithoutCache simulates the OLD behavior (compile every time)
// by calling glob.Compile + Match directly, bypassing the cache entirely.
func BenchmarkMatch_WithoutCache(b *testing.B) {
pattern := "proj:*/app-*"
text := "proj:myproject/app-frontend"
b.ResetTimer()
for i := 0; i < b.N; i++ {
compiled, err := extglob.Compile(pattern)
if err != nil {
b.Fatal(err)
}
compiled.Match(text)
}
}
// BenchmarkGlobCompile measures raw glob.Compile cost
func BenchmarkGlobCompile(b *testing.B) {
pattern := "proj:*/app-*"
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, _ = extglob.Compile(pattern)
}
}
// BenchmarkMatch_RBACSimulation simulates real RBAC evaluation scenario
// 50 policies × 1 app = what happens per application in List
func BenchmarkMatch_RBACSimulation(b *testing.B) {
patterns := make([]string, 50)
for i := range 50 {
patterns[i] = fmt.Sprintf("proj:team-%d/*", i)
}
text := "proj:team-25/my-app"
// With caching: patterns are compiled once
b.ResetTimer()
for i := 0; i < b.N; i++ {
for _, pattern := range patterns {
Match(pattern, text)
}
}
}