diff --git a/kernel/bazaar/bazaar.go b/kernel/bazaar/bazaar.go index b8b6734d2..580190a25 100644 --- a/kernel/bazaar/bazaar.go +++ b/kernel/bazaar/bazaar.go @@ -88,7 +88,10 @@ func buildBazaarPackageWithMetadata(repo *StageRepo, bazaarStats map[string]*baz if stats := bazaarStats[repoURLHash[0]]; nil != stats { // 通过 bazaarStats[owner/repo] 获取单个包的统计数据 pkg.Downloads = stats.Downloads } - packageInstallSizeCache.SetDefault(pkg.RepoURL, pkg.InstallSize) + // TODO 分离本地安装大小和在线 stage 数据的安装大小，不保存到 installSizeCache + bazaarMemMu.Lock() + installSizeCache[pkg.RepoURL] = pkg.InstallSize + bazaarMemMu.Unlock() return &pkg } diff --git a/kernel/bazaar/installed.go b/kernel/bazaar/installed.go index 5b87c7a13..23fd6a462 100644 --- a/kernel/bazaar/installed.go +++ b/kernel/bazaar/installed.go @@ -24,7 +24,6 @@ import ( "github.com/88250/go-humanize" "github.com/88250/gulu" - gcache "github.com/patrickmn/go-cache" "github.com/siyuan-note/filelock" "github.com/siyuan-note/logging" "github.com/siyuan-note/siyuan/kernel/util" @@ -32,9 +31,6 @@ import ( "golang.org/x/sync/singleflight" ) -// packageInstallSizeCache 缓存集市包的安装大小，与 cachedStageIndex 使用相同的缓存时间 -var packageInstallSizeCache = gcache.New(time.Duration(util.RhyCacheDuration)*time.Second, time.Duration(util.RhyCacheDuration)*time.Second/6) // [repoURL]*int64 - // ReadInstalledPackageDirs 读取本地集市包的目录列表 func ReadInstalledPackageDirs(basePath string) ([]os.DirEntry, error) { if !util.IsPathRegularDirOrSymlinkDir(basePath) { @@ -85,12 +81,18 @@ func SetInstalledPackageMetadata(pkg *Package, installPath, baseURLPath, pkgType pkg.HInstallDate = getPackageHInstallDate(pkgType, pkg.Name, installPath) // TODO 本地安装大小的缓存改成 1 分钟有效，打开集市包 README 的时候才遍历集市包文件夹进行统计，异步返回结果到前端显示 https://github.com/siyuan-note/siyuan/issues/16983 // 目前优先使用在线 stage 数据：不耗时，但可能不准确，比如本地旧版本与云端最新版本的安装大小可能不一致；其次使用本地目录大小：耗时，但准确 - if installSize, ok := packageInstallSizeCache.Get(pkg.RepoURL); ok { - pkg.InstallSize = installSize.(int64) + // 需要分离本地安装大小和在线 stage 数据的安装大小 + bazaarMemMu.RLock() + cachedSize, hit := installSizeCache[pkg.RepoURL] + bazaarMemMu.RUnlock() + if hit { + pkg.InstallSize = cachedSize } else { size, _ := util.SizeOfDirectory(installPath) pkg.InstallSize = size - packageInstallSizeCache.SetDefault(pkg.RepoURL, size) + bazaarMemMu.Lock() + installSizeCache[pkg.RepoURL] = size + bazaarMemMu.Unlock() } pkg.HInstallSize = humanize.BytesCustomCeil(uint64(pkg.InstallSize), 2) diff --git a/kernel/bazaar/stage.go b/kernel/bazaar/stage.go index 2d2f56bf2..9d80fa4f0 100644 --- a/kernel/bazaar/stage.go +++ b/kernel/bazaar/stage.go @@ -19,18 +19,38 @@ package bazaar import ( "context" "errors" + "maps" "sync" - "time" - gcache "github.com/patrickmn/go-cache" "github.com/siyuan-note/httpclient" "github.com/siyuan-note/logging" "github.com/siyuan-note/siyuan/kernel/util" "golang.org/x/sync/singleflight" ) -// cachedStageIndex 缓存 stage 索引 -var cachedStageIndex = gcache.New(time.Duration(util.RhyCacheDuration)*time.Second, time.Duration(util.RhyCacheDuration)*time.Second/6) +var ( + bazaarMemMu sync.RWMutex + bazaarCacheRhyHash string // bazaar hash，发生变更时清空以下缓存 + stageIndexCache = make(map[string]*StageIndex) // pkgType -> 集市包索引 + bazaarStatsCache = make(map[string]*bazaarStats) // 集市统计数据 + installSizeCache = make(map[string]int64) // repoURL -> 安装大小 +) + +func applyRhyBazaarHash(ctx context.Context) { + bazaarHash := util.GetRhyBazaarHash(ctx) + if "" == bazaarHash { + return + } + bazaarMemMu.Lock() + defer bazaarMemMu.Unlock() + if bazaarCacheRhyHash != "" && bazaarHash != bazaarCacheRhyHash { + clear(stageIndexCache) + clear(bazaarStatsCache) + clear(installSizeCache) + logging.LogInfof("rhy bazaar hash changed, clearing bazaar caches") + } + bazaarCacheRhyHash = bazaarHash +} type StageBazaarResult struct { StageIndex *StageIndex // stage 索引 @@ -46,7 +66,7 @@ var bazaarStatsFlight singleflight.Group // getStageAndBazaar 获取 stage 索引和 bazaar 索引，相同 pkgType 的并发调用会合并为一次实际请求 (single-flight) func getStageAndBazaar(pkgType string) (result StageBazaarResult) { key := "stageBazaar:" + pkgType - v, err, _ := stageBazaarFlight.Do(key, func() (interface{}, error) { + v, err, _ := stageBazaarFlight.Do(key, func() (any, error) { return getStageAndBazaar0(pkgType), nil }) if err != nil { @@ -58,22 +78,22 @@ func getStageAndBazaar(pkgType string) (result StageBazaarResult) { // getStageAndBazaar0 执行一次 stage 和 bazaar 索引拉取 func getStageAndBazaar0(pkgType string) (result StageBazaarResult) { - stageIndex, stageErr := getStageIndexFromCache(pkgType) - bazaarStats := getBazaarStatsFromCache() - if nil != stageIndex && nil != bazaarStats { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + stageIndex := getStageIndexFromCache(ctx, pkgType) + statsMap := getBazaarStatsFromCache(ctx) + if nil != stageIndex && nil != statsMap { // 两者都从缓存返回，不需要 online 检查 return StageBazaarResult{ StageIndex: stageIndex, - BazaarStats: bazaarStats, + BazaarStats: statsMap, Online: true, - StageErr: stageErr, + StageErr: nil, } } - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() var onlineResult bool onlineDone := make(chan bool, 1) + var stageErr error wg := &sync.WaitGroup{} wg.Go(func() { onlineResult = isBazaarOnline() @@ -83,7 +103,7 @@ func getStageAndBazaar0(pkgType string) (result StageBazaarResult) { stageIndex, stageErr = getStageIndex(ctx, pkgType) }) wg.Go(func() { - bazaarStats = getBazaarStats(ctx) + statsMap = getBazaarStats(ctx) }) <-onlineDone @@ -92,7 +112,7 @@ func getStageAndBazaar0(pkgType string) (result StageBazaarResult) { cancel() return StageBazaarResult{ StageIndex: stageIndex, - BazaarStats: bazaarStats, + BazaarStats: statsMap, Online: false, StageErr: stageErr, } @@ -103,7 +123,7 @@ func getStageAndBazaar0(pkgType string) (result StageBazaarResult) { return StageBazaarResult{ StageIndex: stageIndex, - BazaarStats: bazaarStats, + BazaarStats: statsMap, Online: onlineResult, StageErr: stageErr, } @@ -128,29 +148,27 @@ func isBazaarOnline0() (ret bool) { return } -// getStageIndexFromCache 仅从缓存获取 stage 索引，过期或无缓存时返回 nil -func getStageIndexFromCache(pkgType string) (ret *StageIndex, err error) { - if val, found := cachedStageIndex.Get(pkgType); found { - ret = val.(*StageIndex) - } - return +// getStageIndexFromCache 仅从缓存获取 stage 索引，无缓存时返回 nil（读前根据 util 已同步的 bazaar hash 视情况清理缓存） +func getStageIndexFromCache(ctx context.Context, pkgType string) *StageIndex { + applyRhyBazaarHash(ctx) + bazaarMemMu.RLock() + defer bazaarMemMu.RUnlock() + return stageIndexCache[pkgType] } // getStageIndex 获取 stage 索引 func getStageIndex(ctx context.Context, pkgType string) (ret *StageIndex, err error) { - if cached, cacheErr := getStageIndexFromCache(pkgType); nil != cached { + if cached := getStageIndexFromCache(ctx, pkgType); nil != cached { ret = cached - err = cacheErr return } - var rhyRet map[string]interface{} - rhyRet, err = util.GetRhyResult(ctx, false) - if nil != err { + bazaarHash := util.GetRhyBazaarHash(ctx) + if "" == bazaarHash { + logging.LogErrorf("bazaar hash unavailable (rhy missing or invalid bazaar field)") + err = errors.New("bazaar hash not available") return } - - bazaarHash := rhyRet["bazaar"].(string) ret = &StageIndex{} request := httpclient.NewBrowserRequest() u := util.BazaarOSSServer + "/bazaar@" + bazaarHash + "/stage/" + pkgType + ".json" // pkgType 单词为复数形式 @@ -166,7 +184,9 @@ func getStageIndex(ctx context.Context, pkgType string) (ret *StageIndex, err er return } - cachedStageIndex.SetDefault(pkgType, ret) + bazaarMemMu.Lock() + stageIndexCache[pkgType] = ret + bazaarMemMu.Unlock() return } @@ -190,20 +210,20 @@ type bazaarStats struct { Downloads int `json:"downloads"` // 下载次数 } -// cachedBazaarStats 缓存集市包统计信息 -var cachedBazaarStats = gcache.New(time.Duration(util.RhyCacheDuration)*time.Second, time.Duration(util.RhyCacheDuration)*time.Second/6) - -// getBazaarStatsFromCache 仅从缓存获取集市包统计信息，过期或无缓存时返回 nil -func getBazaarStatsFromCache() (ret map[string]*bazaarStats) { - if val, found := cachedBazaarStats.Get("index"); found { - ret = val.(map[string]*bazaarStats) +// getBazaarStatsFromCache 仅从缓存获取集市包统计信息，无缓存时返回 nil +func getBazaarStatsFromCache(ctx context.Context) (ret map[string]*bazaarStats) { + applyRhyBazaarHash(ctx) + bazaarMemMu.RLock() + defer bazaarMemMu.RUnlock() + if 0 == len(bazaarStatsCache) { + return nil } - return + return bazaarStatsCache } // getBazaarStats 获取集市包统计信息 func getBazaarStats(ctx context.Context) map[string]*bazaarStats { - if cached := getBazaarStatsFromCache(); nil != cached { + if cached := getBazaarStatsFromCache(ctx); nil != cached { return cached } @@ -213,19 +233,24 @@ func getBazaarStats(ctx context.Context) map[string]*bazaarStats { return v.(map[string]*bazaarStats) } -func getBazaarStats0(ctx context.Context) map[string]*bazaarStats { - var result map[string]*bazaarStats +func getBazaarStats0(ctx context.Context) (result map[string]*bazaarStats) { request := httpclient.NewBrowserRequest() u := util.BazaarStatServer + "/bazaar/index.json" resp, reqErr := request.SetContext(ctx).SetSuccessResult(&result).Get(u) if nil != reqErr { logging.LogErrorf("get bazaar stats [%s] failed: %s", u, reqErr) - return result + return } if 200 != resp.StatusCode { logging.LogErrorf("get bazaar stats [%s] failed: %d", u, resp.StatusCode) - return result + return } - cachedBazaarStats.SetDefault("index", result) - return result + if nil == result { + result = make(map[string]*bazaarStats) + } + bazaarMemMu.Lock() + clear(bazaarStatsCache) + maps.Copy(bazaarStatsCache, result) + bazaarMemMu.Unlock() + return } diff --git a/kernel/util/rhy.go b/kernel/util/rhy.go index eb466e5ba..3a04160ac 100644 --- a/kernel/util/rhy.go +++ b/kernel/util/rhy.go @@ -18,7 +18,6 @@ package util import ( "context" - "errors" "fmt" "sync" "time" @@ -31,50 +30,15 @@ import ( var ( RhyCacheDuration = int64(3600 * 6) - cachedRhyResult = map[string]interface{}{} + cachedRhyResult = map[string]any{} rhyResultCacheTime int64 rhyResultLock = sync.Mutex{} rhyResultFlight singleflight.Group + + rhyBazaarHash string + rhyBazaarHashLock sync.RWMutex ) -func GetRhyResult(ctx context.Context, force bool) (map[string]interface{}, error) { - if ContainerDocker == Container { - RhyCacheDuration = int64(3600 * 24) - } - - if RhyCacheDuration >= time.Now().Unix()-rhyResultCacheTime && !force && 0 < len(cachedRhyResult) { - return cachedRhyResult, nil - } - - // 并发调用只执行一次实际请求 - v, err, _ := rhyResultFlight.Do("rhyResult", func() (interface{}, error) { - return getRhyResult0(ctx) - }) - if err != nil { - return nil, err - } - return v.(map[string]interface{}), nil -} - -func getRhyResult0(ctx context.Context) (map[string]interface{}, error) { - rhyResultLock.Lock() - defer rhyResultLock.Unlock() - - request := httpclient.NewCloudRequest30s() - resp, err := request.SetContext(ctx).SetSuccessResult(&cachedRhyResult).Get(GetCloudServer() + "/apis/siyuan/version?ver=" + Ver) - if err != nil { - logging.LogErrorf("get version info failed: %s", err) - return nil, err - } - if 200 != resp.StatusCode { - msg := fmt.Sprintf("get rhy result failed: %d", resp.StatusCode) - logging.LogErrorf(msg) - return nil, errors.New(msg) - } - rhyResultCacheTime = time.Now().Unix() - return cachedRhyResult, nil -} - func RefreshRhyResultJob() { _, err := GetRhyResult(context.TODO(), true) if nil != err { @@ -85,3 +49,75 @@ func RefreshRhyResultJob() { }() } } + +func GetRhyResult(ctx context.Context, force bool) (map[string]any, error) { + if ContainerDocker == Container { + RhyCacheDuration = int64(3600 * 24) + } + + if RhyCacheDuration >= time.Now().Unix()-rhyResultCacheTime && !force && 0 < len(cachedRhyResult) { + return cachedRhyResult, nil + } + + // 并发调用只执行一次实际请求 + v, err, _ := rhyResultFlight.Do("rhyResult", func() (any, error) { + return getRhyResult0(ctx) + }) + if err != nil { + return nil, err + } + ret := v.(map[string]any) + syncRhyBazaarHashFromResult(ret) + return ret, nil +} + +func getRhyResult0(ctx context.Context) (map[string]any, error) { + rhyResultLock.Lock() + defer rhyResultLock.Unlock() + + request := httpclient.NewCloudRequest30s() + resp, err := request.SetContext(ctx).SetSuccessResult(&cachedRhyResult).Get(GetCloudServer() + "/apis/siyuan/version?ver=" + Ver) + if err != nil { + logging.LogErrorf("get version info failed: %s", err) + return nil, err + } + if 200 != resp.StatusCode { + logging.LogErrorf("get rhy result failed: %d", resp.StatusCode) + return nil, fmt.Errorf("get rhy result failed: %d", resp.StatusCode) + } + rhyResultCacheTime = time.Now().Unix() + return cachedRhyResult, nil +} + +func syncRhyBazaarHashFromResult(m map[string]any) { + rhyBazaarHashLock.Lock() + defer rhyBazaarHashLock.Unlock() + if nil == m { + rhyBazaarHash = "" + return + } + v, ok := m["bazaar"] + if !ok || nil == v { + rhyBazaarHash = "" + return + } + s, ok := v.(string) + if !ok || "" == s { + rhyBazaarHash = "" + return + } + rhyBazaarHash = s +} + +func GetRhyBazaarHash(ctx context.Context) string { + rhyBazaarHashLock.RLock() + h := rhyBazaarHash + rhyBazaarHashLock.RUnlock() + if "" != h { + return h + } + _, _ = GetRhyResult(ctx, false) + rhyBazaarHashLock.RLock() + defer rhyBazaarHashLock.RUnlock() + return rhyBazaarHash +}