Elgato_dark/siyuan
1
0
Fork 0
mirror of https://github.com/siyuan-note/siyuan synced 2026-04-21 13:37:52 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

🎨 Bazaar cache invalidates following hash changes (#17320)

Browse source
This commit is contained in:
Jeffrey Chen 2026-03-30 12:16:20 +08:00 committed by GitHub
parent 1bc7f556ff
commit 5a4e6b6483
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 159 additions and 93 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
kernel/bazaar/bazaar.go
View file

@ -88,7 +88,10 @@ func buildBazaarPackageWithMetadata(repo *StageRepo, bazaarStats map[string]*baz
if stats := bazaarStats[repoURLHash[0]]; nil != stats { // 通过 bazaarStats[owner/repo] 获取单个包的统计数据
pkg.Downloads = stats.Downloads
}
packageInstallSizeCache.SetDefault(pkg.RepoURL, pkg.InstallSize)
// TODO 分离本地安装大小和在线 stage 数据的安装大小，不保存到 installSizeCache
bazaarMemMu.Lock()
installSizeCache[pkg.RepoURL] = pkg.InstallSize
bazaarMemMu.Unlock()
return &pkg
}

16
kernel/bazaar/installed.go
View file

@ -24,7 +24,6 @@ import (
"github.com/88250/go-humanize"
"github.com/88250/gulu"
gcache "github.com/patrickmn/go-cache"
"github.com/siyuan-note/filelock"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/util"
@ -32,9 +31,6 @@ import (
"golang.org/x/sync/singleflight"
)
// packageInstallSizeCache 缓存集市包的安装大小，与 cachedStageIndex 使用相同的缓存时间
var packageInstallSizeCache = gcache.New(time.Duration(util.RhyCacheDuration)*time.Second, time.Duration(util.RhyCacheDuration)*time.Second/6) // [repoURL]*int64
// ReadInstalledPackageDirs 读取本地集市包的目录列表
func ReadInstalledPackageDirs(basePath string) ([]os.DirEntry, error) {
if !util.IsPathRegularDirOrSymlinkDir(basePath) {
@ -85,12 +81,18 @@ func SetInstalledPackageMetadata(pkg *Package, installPath, baseURLPath, pkgType
pkg.HInstallDate = getPackageHInstallDate(pkgType, pkg.Name, installPath)
// TODO 本地安装大小的缓存改成 1 分钟有效，打开集市包 README 的时候才遍历集市包文件夹进行统计，异步返回结果到前端显示 https://github.com/siyuan-note/siyuan/issues/16983
// 目前优先使用在线 stage 数据：不耗时，但可能不准确，比如本地旧版本与云端最新版本的安装大小可能不一致；其次使用本地目录大小：耗时，但准确
if installSize, ok := packageInstallSizeCache.Get(pkg.RepoURL); ok {
pkg.InstallSize = installSize.(int64)
// 需要分离本地安装大小和在线 stage 数据的安装大小
bazaarMemMu.RLock()
cachedSize, hit := installSizeCache[pkg.RepoURL]
bazaarMemMu.RUnlock()
if hit {
pkg.InstallSize = cachedSize
} else {
size, _ := util.SizeOfDirectory(installPath)
pkg.InstallSize = size
packageInstallSizeCache.SetDefault(pkg.RepoURL, size)
bazaarMemMu.Lock()
installSizeCache[pkg.RepoURL] = size
bazaarMemMu.Unlock()
}
pkg.HInstallSize = humanize.BytesCustomCeil(uint64(pkg.InstallSize), 2)

115
kernel/bazaar/stage.go
View file

@ -19,18 +19,38 @@ package bazaar
import (
"context"
"errors"
"maps"
"sync"
"time"
gcache "github.com/patrickmn/go-cache"
"github.com/siyuan-note/httpclient"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/util"
"golang.org/x/sync/singleflight"
)
// cachedStageIndex 缓存 stage 索引
var cachedStageIndex = gcache.New(time.Duration(util.RhyCacheDuration)*time.Second, time.Duration(util.RhyCacheDuration)*time.Second/6)
var (
bazaarMemMu sync.RWMutex
bazaarCacheRhyHash string // bazaar hash，发生变更时清空以下缓存
stageIndexCache = make(map[string]*StageIndex) // pkgType -> 集市包索引
bazaarStatsCache = make(map[string]*bazaarStats) // 集市统计数据
installSizeCache = make(map[string]int64) // repoURL -> 安装大小
)
func applyRhyBazaarHash(ctx context.Context) {
bazaarHash := util.GetRhyBazaarHash(ctx)
if "" == bazaarHash {
return
}
bazaarMemMu.Lock()
defer bazaarMemMu.Unlock()
if bazaarCacheRhyHash != "" && bazaarHash != bazaarCacheRhyHash {
clear(stageIndexCache)
clear(bazaarStatsCache)
clear(installSizeCache)
logging.LogInfof("rhy bazaar hash changed, clearing bazaar caches")
}
bazaarCacheRhyHash = bazaarHash
}
type StageBazaarResult struct {
StageIndex *StageIndex // stage 索引
@ -46,7 +66,7 @@ var bazaarStatsFlight singleflight.Group
// getStageAndBazaar 获取 stage 索引和 bazaar 索引，相同 pkgType 的并发调用会合并为一次实际请求 (single-flight)
func getStageAndBazaar(pkgType string) (result StageBazaarResult) {
key := "stageBazaar:" + pkgType
v, err, _ := stageBazaarFlight.Do(key, func() (interface{}, error) {
v, err, _ := stageBazaarFlight.Do(key, func() (any, error) {
return getStageAndBazaar0(pkgType), nil
})
if err != nil {
@ -58,22 +78,22 @@ func getStageAndBazaar(pkgType string) (result StageBazaarResult) {
// getStageAndBazaar0 执行一次 stage 和 bazaar 索引拉取
func getStageAndBazaar0(pkgType string) (result StageBazaarResult) {
stageIndex, stageErr := getStageIndexFromCache(pkgType)
bazaarStats := getBazaarStatsFromCache()
if nil != stageIndex && nil != bazaarStats {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
stageIndex := getStageIndexFromCache(ctx, pkgType)
statsMap := getBazaarStatsFromCache(ctx)
if nil != stageIndex && nil != statsMap {
// 两者都从缓存返回，不需要 online 检查
return StageBazaarResult{
StageIndex: stageIndex,
BazaarStats: bazaarStats,
BazaarStats: statsMap,
Online: true,
StageErr: stageErr,
StageErr: nil,
}
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
var onlineResult bool
onlineDone := make(chan bool, 1)
var stageErr error
wg := &sync.WaitGroup{}
wg.Go(func() {
onlineResult = isBazaarOnline()
@ -83,7 +103,7 @@ func getStageAndBazaar0(pkgType string) (result StageBazaarResult) {
stageIndex, stageErr = getStageIndex(ctx, pkgType)
})
wg.Go(func() {
bazaarStats = getBazaarStats(ctx)
statsMap = getBazaarStats(ctx)
})
<-onlineDone
@ -92,7 +112,7 @@ func getStageAndBazaar0(pkgType string) (result StageBazaarResult) {
cancel()
return StageBazaarResult{
StageIndex: stageIndex,
BazaarStats: bazaarStats,
BazaarStats: statsMap,
Online: false,
StageErr: stageErr,
}
@ -103,7 +123,7 @@ func getStageAndBazaar0(pkgType string) (result StageBazaarResult) {
return StageBazaarResult{
StageIndex: stageIndex,
BazaarStats: bazaarStats,
BazaarStats: statsMap,
Online: onlineResult,
StageErr: stageErr,
}
@ -128,29 +148,27 @@ func isBazaarOnline0() (ret bool) {
return
}
// getStageIndexFromCache 仅从缓存获取 stage 索引，过期或无缓存时返回 nil
func getStageIndexFromCache(pkgType string) (ret *StageIndex, err error) {
if val, found := cachedStageIndex.Get(pkgType); found {
ret = val.(*StageIndex)
}
return
// getStageIndexFromCache 仅从缓存获取 stage 索引，无缓存时返回 nil（读前根据 util 已同步的 bazaar hash 视情况清理缓存）
func getStageIndexFromCache(ctx context.Context, pkgType string) *StageIndex {
applyRhyBazaarHash(ctx)
bazaarMemMu.RLock()
defer bazaarMemMu.RUnlock()
return stageIndexCache[pkgType]
}
// getStageIndex 获取 stage 索引
func getStageIndex(ctx context.Context, pkgType string) (ret *StageIndex, err error) {
if cached, cacheErr := getStageIndexFromCache(pkgType); nil != cached {
if cached := getStageIndexFromCache(ctx, pkgType); nil != cached {
ret = cached
err = cacheErr
return
}
var rhyRet map[string]interface{}
rhyRet, err = util.GetRhyResult(ctx, false)
if nil != err {
bazaarHash := util.GetRhyBazaarHash(ctx)
if "" == bazaarHash {
logging.LogErrorf("bazaar hash unavailable (rhy missing or invalid bazaar field)")
err = errors.New("bazaar hash not available")
return
}
bazaarHash := rhyRet["bazaar"].(string)
ret = &StageIndex{}
request := httpclient.NewBrowserRequest()
u := util.BazaarOSSServer + "/bazaar@" + bazaarHash + "/stage/" + pkgType + ".json" // pkgType 单词为复数形式
@ -166,7 +184,9 @@ func getStageIndex(ctx context.Context, pkgType string) (ret *StageIndex, err er
return
}
cachedStageIndex.SetDefault(pkgType, ret)
bazaarMemMu.Lock()
stageIndexCache[pkgType] = ret
bazaarMemMu.Unlock()
return
}
@ -190,20 +210,20 @@ type bazaarStats struct {
Downloads int `json:"downloads"` // 下载次数
}
// cachedBazaarStats 缓存集市包统计信息
var cachedBazaarStats = gcache.New(time.Duration(util.RhyCacheDuration)*time.Second, time.Duration(util.RhyCacheDuration)*time.Second/6)
// getBazaarStatsFromCache 仅从缓存获取集市包统计信息，过期或无缓存时返回 nil
func getBazaarStatsFromCache() (ret map[string]*bazaarStats) {
if val, found := cachedBazaarStats.Get("index"); found {
ret = val.(map[string]*bazaarStats)
// getBazaarStatsFromCache 仅从缓存获取集市包统计信息，无缓存时返回 nil
func getBazaarStatsFromCache(ctx context.Context) (ret map[string]*bazaarStats) {
applyRhyBazaarHash(ctx)
bazaarMemMu.RLock()
defer bazaarMemMu.RUnlock()
if 0 == len(bazaarStatsCache) {
return nil
}
return
return bazaarStatsCache
}
// getBazaarStats 获取集市包统计信息
func getBazaarStats(ctx context.Context) map[string]*bazaarStats {
if cached := getBazaarStatsFromCache(); nil != cached {
if cached := getBazaarStatsFromCache(ctx); nil != cached {
return cached
}
@ -213,19 +233,24 @@ func getBazaarStats(ctx context.Context) map[string]*bazaarStats {
return v.(map[string]*bazaarStats)
}
func getBazaarStats0(ctx context.Context) map[string]*bazaarStats {
var result map[string]*bazaarStats
func getBazaarStats0(ctx context.Context) (result map[string]*bazaarStats) {
request := httpclient.NewBrowserRequest()
u := util.BazaarStatServer + "/bazaar/index.json"
resp, reqErr := request.SetContext(ctx).SetSuccessResult(&result).Get(u)
if nil != reqErr {
logging.LogErrorf("get bazaar stats [%s] failed: %s", u, reqErr)
return result
return
}
if 200 != resp.StatusCode {
logging.LogErrorf("get bazaar stats [%s] failed: %d", u, resp.StatusCode)
return result
return
}
cachedBazaarStats.SetDefault("index", result)
return result
if nil == result {
result = make(map[string]*bazaarStats)
}
bazaarMemMu.Lock()
clear(bazaarStatsCache)
maps.Copy(bazaarStatsCache, result)
bazaarMemMu.Unlock()
return
}

116
kernel/util/rhy.go
View file

@ -18,7 +18,6 @@ package util
import (
"context"
"errors"
"fmt"
"sync"
"time"
@ -31,50 +30,15 @@ import (
var (
RhyCacheDuration = int64(3600 * 6)
cachedRhyResult = map[string]interface{}{}
cachedRhyResult = map[string]any{}
rhyResultCacheTime int64
rhyResultLock = sync.Mutex{}
rhyResultFlight singleflight.Group
rhyBazaarHash string
rhyBazaarHashLock sync.RWMutex
)
func GetRhyResult(ctx context.Context, force bool) (map[string]interface{}, error) {
if ContainerDocker == Container {
RhyCacheDuration = int64(3600 * 24)
}
if RhyCacheDuration >= time.Now().Unix()-rhyResultCacheTime && !force && 0 < len(cachedRhyResult) {
return cachedRhyResult, nil
}
// 并发调用只执行一次实际请求
v, err, _ := rhyResultFlight.Do("rhyResult", func() (interface{}, error) {
return getRhyResult0(ctx)
})
if err != nil {
return nil, err
}
return v.(map[string]interface{}), nil
}
func getRhyResult0(ctx context.Context) (map[string]interface{}, error) {
rhyResultLock.Lock()
defer rhyResultLock.Unlock()
request := httpclient.NewCloudRequest30s()
resp, err := request.SetContext(ctx).SetSuccessResult(&cachedRhyResult).Get(GetCloudServer() + "/apis/siyuan/version?ver=" + Ver)
if err != nil {
logging.LogErrorf("get version info failed: %s", err)
return nil, err
}
if 200 != resp.StatusCode {
msg := fmt.Sprintf("get rhy result failed: %d", resp.StatusCode)
logging.LogErrorf(msg)
return nil, errors.New(msg)
}
rhyResultCacheTime = time.Now().Unix()
return cachedRhyResult, nil
}
func RefreshRhyResultJob() {
_, err := GetRhyResult(context.TODO(), true)
if nil != err {
@ -85,3 +49,75 @@ func RefreshRhyResultJob() {
}()
}
}
func GetRhyResult(ctx context.Context, force bool) (map[string]any, error) {
if ContainerDocker == Container {
RhyCacheDuration = int64(3600 * 24)
}
if RhyCacheDuration >= time.Now().Unix()-rhyResultCacheTime && !force && 0 < len(cachedRhyResult) {
return cachedRhyResult, nil
}
// 并发调用只执行一次实际请求
v, err, _ := rhyResultFlight.Do("rhyResult", func() (any, error) {
return getRhyResult0(ctx)
})
if err != nil {
return nil, err
}
ret := v.(map[string]any)
syncRhyBazaarHashFromResult(ret)
return ret, nil
}
func getRhyResult0(ctx context.Context) (map[string]any, error) {
rhyResultLock.Lock()
defer rhyResultLock.Unlock()
request := httpclient.NewCloudRequest30s()
resp, err := request.SetContext(ctx).SetSuccessResult(&cachedRhyResult).Get(GetCloudServer() + "/apis/siyuan/version?ver=" + Ver)
if err != nil {
logging.LogErrorf("get version info failed: %s", err)
return nil, err
}
if 200 != resp.StatusCode {
logging.LogErrorf("get rhy result failed: %d", resp.StatusCode)
return nil, fmt.Errorf("get rhy result failed: %d", resp.StatusCode)
}
rhyResultCacheTime = time.Now().Unix()
return cachedRhyResult, nil
}
func syncRhyBazaarHashFromResult(m map[string]any) {
rhyBazaarHashLock.Lock()
defer rhyBazaarHashLock.Unlock()
if nil == m {
rhyBazaarHash = ""
return
}
v, ok := m["bazaar"]
if !ok || nil == v {
rhyBazaarHash = ""
return
}
s, ok := v.(string)
if !ok || "" == s {
rhyBazaarHash = ""
return
}
rhyBazaarHash = s
}
func GetRhyBazaarHash(ctx context.Context) string {
rhyBazaarHashLock.RLock()
h := rhyBazaarHash
rhyBazaarHashLock.RUnlock()
if "" != h {
return h
}
_, _ = GetRhyResult(ctx, false)
rhyBazaarHashLock.RLock()
defer rhyBazaarHashLock.RUnlock()
return rhyBazaarHash
}