mirror of
https://github.com/wavetermdev/waveterm
synced 2026-05-22 16:28:32 +00:00
more tests, cleaning up structure for flushing -- write intentions, dirty, and flushing. simplify/unify some operations
This commit is contained in:
parent
34b91445a9
commit
14772c8f61
3 changed files with 319 additions and 72 deletions
|
|
@ -23,8 +23,9 @@ var partDataSize int64 = DefaultPartDataSize // overridden in tests
|
|||
var stopFlush = &atomic.Bool{}
|
||||
|
||||
var GBS *BlockStore = &BlockStore{
|
||||
Lock: &sync.Mutex{},
|
||||
Cache: make(map[cacheKey]*CacheEntry),
|
||||
Lock: &sync.Mutex{},
|
||||
Cache: make(map[cacheKey]*CacheEntry),
|
||||
NextIntentionId: 1,
|
||||
}
|
||||
|
||||
type FileOptsType struct {
|
||||
|
|
@ -36,15 +37,19 @@ type FileOptsType struct {
|
|||
type FileMeta = map[string]any
|
||||
|
||||
type BlockFile struct {
|
||||
// these fields are static (not updated)
|
||||
BlockId string `json:"blockid"`
|
||||
Name string `json:"name"`
|
||||
Size int64 `json:"size"`
|
||||
CreatedTs int64 `json:"createdts"`
|
||||
ModTs int64 `json:"modts"`
|
||||
Opts FileOptsType `json:"opts"`
|
||||
Meta FileMeta `json:"meta"`
|
||||
CreatedTs int64 `json:"createdts"`
|
||||
|
||||
// these fields are mutable
|
||||
Size int64 `json:"size"`
|
||||
ModTs int64 `json:"modts"`
|
||||
Meta FileMeta `json:"meta"` // only top-level keys can be updated (lower levels are immutable)
|
||||
}
|
||||
|
||||
// this works because lower levels are immutable
|
||||
func copyMeta(meta FileMeta) FileMeta {
|
||||
newMeta := make(FileMeta)
|
||||
for k, v := range meta {
|
||||
|
|
@ -193,40 +198,28 @@ func (s *BlockStore) ListFiles(ctx context.Context, blockId string) ([]*BlockFil
|
|||
}
|
||||
|
||||
func (s *BlockStore) WriteMeta(ctx context.Context, blockId string, name string, meta FileMeta, merge bool) error {
|
||||
file, ok := s.getFileFromCache(blockId, name)
|
||||
if !ok {
|
||||
dbFile, err := dbGetBlockFile(ctx, blockId, name)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error getting file: %v", err)
|
||||
}
|
||||
file = dbFile
|
||||
s.pinCacheEntry(blockId, name)
|
||||
defer s.unpinCacheEntry(blockId, name)
|
||||
_, err := s.loadFileInfo(ctx, blockId, name)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error loading file info: %v", err)
|
||||
}
|
||||
if file == nil {
|
||||
return fmt.Errorf("file not found")
|
||||
}
|
||||
var rtnErr error
|
||||
s.withLock(blockId, name, true, func(entry *CacheEntry) {
|
||||
if entry.Deleted {
|
||||
rtnErr = fmt.Errorf("file is deleted")
|
||||
return
|
||||
}
|
||||
newFileEntry := entry.copyOrCreateFileEntry(file)
|
||||
if merge {
|
||||
for k, v := range meta {
|
||||
if v == nil {
|
||||
delete(newFileEntry.File.Meta, k)
|
||||
continue
|
||||
return s.withLockExists(blockId, name, func(entry *CacheEntry) error {
|
||||
entry.modifyFileData(func(file *BlockFile) {
|
||||
if merge {
|
||||
for k, v := range meta {
|
||||
if v == nil {
|
||||
delete(file.Meta, k)
|
||||
continue
|
||||
}
|
||||
file.Meta[k] = v
|
||||
}
|
||||
newFileEntry.File.Meta[k] = v
|
||||
} else {
|
||||
file.Meta = meta
|
||||
}
|
||||
} else {
|
||||
newFileEntry.File.Meta = meta
|
||||
}
|
||||
entry.FileEntry = newFileEntry
|
||||
entry.FileEntry.File.ModTs = time.Now().UnixMilli()
|
||||
entry.Version++
|
||||
})
|
||||
return nil
|
||||
})
|
||||
return rtnErr
|
||||
}
|
||||
|
||||
func (s *BlockStore) loadFileInfo(ctx context.Context, blockId string, name string) (*BlockFile, error) {
|
||||
|
|
@ -246,6 +239,7 @@ func (s *BlockStore) loadFileInfo(ctx context.Context, blockId string, name stri
|
|||
}
|
||||
var rtnErr error
|
||||
rtnFile := dbFile
|
||||
// cannot use withLockExists because we're setting entry.FileEntry!
|
||||
s.withLock(blockId, name, true, func(entry *CacheEntry) {
|
||||
if entry.Deleted {
|
||||
rtnFile = nil
|
||||
|
|
@ -257,7 +251,11 @@ func (s *BlockStore) loadFileInfo(ctx context.Context, blockId string, name stri
|
|||
rtnFile = entry.FileEntry.File.DeepCopy()
|
||||
return
|
||||
}
|
||||
entry.FileEntry = entry.copyOrCreateFileEntry(dbFile)
|
||||
entry.FileEntry = &FileCacheEntry{
|
||||
Dirty: &atomic.Bool{},
|
||||
Flushing: &atomic.Bool{},
|
||||
File: *dbFile.DeepCopy(), // make a copy since File must be immutable
|
||||
}
|
||||
// returns dbFile, nil
|
||||
})
|
||||
return rtnFile, rtnErr
|
||||
|
|
@ -327,19 +325,20 @@ func (s *BlockStore) loadDataParts(ctx context.Context, blockId string, name str
|
|||
})
|
||||
}
|
||||
|
||||
func (s *BlockStore) writeAt_nolock(entry *CacheEntry, offset int64, data []byte) {
|
||||
func (entry *CacheEntry) writeAtToCache(offset int64, data []byte, replace bool) {
|
||||
endWrite := offset + int64(len(data))
|
||||
entry.writeAt(offset, data)
|
||||
if endWrite > entry.FileEntry.File.Size {
|
||||
entry.FileEntry.File.Size = endWrite
|
||||
}
|
||||
entry.FileEntry.File.ModTs = time.Now().UnixMilli()
|
||||
entry.Version++
|
||||
entry.writeAt(offset, data, replace)
|
||||
entry.modifyFileData(func(file *BlockFile) {
|
||||
if endWrite > file.Size || replace {
|
||||
file.Size = endWrite
|
||||
}
|
||||
file.ModTs = time.Now().UnixMilli()
|
||||
})
|
||||
}
|
||||
|
||||
func (s *BlockStore) appendDataToCache(blockId string, name string, data []byte) error {
|
||||
return s.withLockExists(blockId, name, func(entry *CacheEntry) error {
|
||||
s.writeAt_nolock(entry, entry.FileEntry.File.Size, data)
|
||||
entry.writeAtToCache(entry.FileEntry.File.Size, data, false)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
|
@ -347,6 +346,8 @@ func (s *BlockStore) appendDataToCache(blockId string, name string, data []byte)
|
|||
func (s *BlockStore) AppendData(ctx context.Context, blockId string, name string, data []byte) error {
|
||||
s.pinCacheEntry(blockId, name)
|
||||
defer s.unpinCacheEntry(blockId, name)
|
||||
intentionId := s.setWriteIntention(blockId, name, WriteIntention{Append: true})
|
||||
defer s.clearWriteIntention(blockId, name, intentionId)
|
||||
_, err := s.loadFileInfo(ctx, blockId, name)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error loading file info: %v", err)
|
||||
|
|
@ -366,6 +367,16 @@ func (s *BlockStore) GetAllBlockIds(ctx context.Context) ([]string, error) {
|
|||
return dbGetAllBlockIds(ctx)
|
||||
}
|
||||
|
||||
func incompletePartsFromMap(partMap map[int]int) []int {
|
||||
var incompleteParts []int
|
||||
for partIdx, size := range partMap {
|
||||
if size != int(partDataSize) {
|
||||
incompleteParts = append(incompleteParts, partIdx)
|
||||
}
|
||||
}
|
||||
return incompleteParts
|
||||
}
|
||||
|
||||
// returns a map of partIdx to amount of data to write to that part
|
||||
func (file *BlockFile) computePartMap(startOffset int64, size int64) map[int]int {
|
||||
partMap := make(map[int]int)
|
||||
|
|
@ -388,6 +399,21 @@ func (file *BlockFile) computePartMap(startOffset int64, size int64) map[int]int
|
|||
return partMap
|
||||
}
|
||||
|
||||
func (s *BlockStore) WriteFile(ctx context.Context, blockId string, name string, data []byte) error {
|
||||
s.pinCacheEntry(blockId, name)
|
||||
defer s.unpinCacheEntry(blockId, name)
|
||||
intentionId := s.setWriteIntention(blockId, name, WriteIntention{Replace: true})
|
||||
defer s.clearWriteIntention(blockId, name, intentionId)
|
||||
_, err := s.loadFileInfo(ctx, blockId, name)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error loading file info: %v", err)
|
||||
}
|
||||
return s.withLockExists(blockId, name, func(entry *CacheEntry) error {
|
||||
entry.writeAtToCache(0, data, true)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (s *BlockStore) WriteAt(ctx context.Context, blockId string, name string, offset int64, data []byte) error {
|
||||
s.pinCacheEntry(blockId, name)
|
||||
defer s.unpinCacheEntry(blockId, name)
|
||||
|
|
@ -395,16 +421,34 @@ func (s *BlockStore) WriteAt(ctx context.Context, blockId string, name string, o
|
|||
if err != nil {
|
||||
return fmt.Errorf("error loading file info: %v", err)
|
||||
}
|
||||
startWriteIdx := offset
|
||||
endWriteIdx := offset + int64(len(data))
|
||||
startPartIdx := file.partIdxAtOffset(startWriteIdx)
|
||||
endPartIdx := file.partIdxAtOffset(endWriteIdx)
|
||||
err = s.loadDataParts(ctx, blockId, name, []int{startPartIdx, endPartIdx})
|
||||
if offset < 0 {
|
||||
return fmt.Errorf("offset must be non-negative")
|
||||
}
|
||||
if offset > file.Size {
|
||||
return fmt.Errorf("offset is past the end of the file")
|
||||
}
|
||||
if file.Opts.Circular {
|
||||
startCirFileOffset := file.Size - file.Opts.MaxSize
|
||||
if offset+int64(len(data)) < startCirFileOffset {
|
||||
// write is before the start of the circular file
|
||||
return nil
|
||||
}
|
||||
if offset < startCirFileOffset {
|
||||
amtBeforeStart := startCirFileOffset - offset
|
||||
offset += amtBeforeStart
|
||||
data = data[amtBeforeStart:]
|
||||
}
|
||||
}
|
||||
partMap := file.computePartMap(offset, int64(len(data)))
|
||||
intentionId := s.setWriteIntention(blockId, name, WriteIntention{Parts: partMap})
|
||||
defer s.clearWriteIntention(blockId, name, intentionId)
|
||||
incompleteParts := incompletePartsFromMap(partMap)
|
||||
err = s.loadDataParts(ctx, blockId, name, incompleteParts)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error loading data parts: %v", err)
|
||||
}
|
||||
return s.withLockExists(blockId, name, func(entry *CacheEntry) error {
|
||||
s.writeAt_nolock(entry, offset, data)
|
||||
entry.writeAtToCache(offset, data, false)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
|
@ -487,6 +531,25 @@ func (s *BlockStore) ReadFile(ctx context.Context, blockId string, name string)
|
|||
return s.ReadAt(ctx, blockId, name, 0, file.Size)
|
||||
}
|
||||
|
||||
func (s *BlockStore) FlushCache(ctx context.Context) error {
|
||||
var dirtyCacheKeys []cacheKey
|
||||
s.Lock.Lock()
|
||||
for key, entry := range s.Cache {
|
||||
if entry.FileEntry != nil && entry.FileEntry.Dirty.Load() {
|
||||
dirtyCacheKeys = append(dirtyCacheKeys, key)
|
||||
continue
|
||||
}
|
||||
for _, dataEntry := range entry.DataEntries {
|
||||
if dataEntry != nil && dataEntry.Dirty.Load() {
|
||||
dirtyCacheKeys = append(dirtyCacheKeys, key)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
s.Lock.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func minInt64(a, b int64) int64 {
|
||||
if a < b {
|
||||
return a
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import (
|
|||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
)
|
||||
|
|
@ -16,6 +17,14 @@ type cacheKey struct {
|
|||
Name string
|
||||
}
|
||||
|
||||
// note about "Dirty" and "Flushing" fields:
|
||||
// - Dirty is set to true when the entry is modified
|
||||
// - Flushing is set to true when the entry is being flushed to disk
|
||||
// note these fields can *only* be set to true while holding the store lock
|
||||
// but the flusher may set them to false without the lock (when the flusher no longer will read the entry fields)
|
||||
// the flusher *must* unset Dirty first, then Flushing
|
||||
// other code should test Flushing before Dirty
|
||||
// that means you *cannot* write a field in a cache entry if Flushing.Load() is true (you must make a copy)
|
||||
type DataCacheEntry struct {
|
||||
Dirty *atomic.Bool
|
||||
Flushing *atomic.Bool
|
||||
|
|
@ -24,13 +33,15 @@ type DataCacheEntry struct {
|
|||
}
|
||||
|
||||
type FileCacheEntry struct {
|
||||
Dirty *atomic.Bool
|
||||
File BlockFile
|
||||
Dirty *atomic.Bool
|
||||
Flushing *atomic.Bool
|
||||
File BlockFile
|
||||
}
|
||||
|
||||
type WriteIntention struct {
|
||||
Parts map[int]bool
|
||||
Append bool
|
||||
Parts map[int]int
|
||||
Append bool
|
||||
Replace bool
|
||||
}
|
||||
|
||||
// invariants:
|
||||
|
|
@ -43,10 +54,9 @@ type WriteIntention struct {
|
|||
type CacheEntry struct {
|
||||
BlockId string
|
||||
Name string
|
||||
Version int
|
||||
PinCount int
|
||||
Deleted bool
|
||||
WriteIntentions map[string]*WriteIntention // map from intentionid -> WriteIntention
|
||||
WriteIntentions map[int]WriteIntention // map from intentionid -> WriteIntention
|
||||
FileEntry *FileCacheEntry
|
||||
DataEntries []*DataCacheEntry
|
||||
}
|
||||
|
|
@ -54,7 +64,7 @@ type CacheEntry struct {
|
|||
//lint:ignore U1000 used for testing
|
||||
func (e *CacheEntry) dump() string {
|
||||
var buf bytes.Buffer
|
||||
fmt.Fprintf(&buf, "CacheEntry{\nBlockId: %q, Name: %q, Version: %d, PinCount: %d, Deleted: %v, IW: %v\n", e.BlockId, e.Name, e.Version, e.PinCount, e.Deleted, e.WriteIntentions)
|
||||
fmt.Fprintf(&buf, "CacheEntry{\nBlockId: %q, Name: %q, PinCount: %d, Deleted: %v, IW: %v\n", e.BlockId, e.Name, e.PinCount, e.Deleted, e.WriteIntentions)
|
||||
if e.FileEntry != nil {
|
||||
fmt.Fprintf(&buf, "FileEntry: %v\n", e.FileEntry.File)
|
||||
}
|
||||
|
|
@ -140,7 +150,10 @@ func (dce *DataCacheEntry) writeToPart(offset int64, data []byte) (int64, *DataC
|
|||
return toWrite, dce
|
||||
}
|
||||
|
||||
func (entry *CacheEntry) writeAt(offset int64, data []byte) {
|
||||
func (entry *CacheEntry) writeAt(offset int64, data []byte, replace bool) {
|
||||
if replace {
|
||||
entry.DataEntries = nil
|
||||
}
|
||||
for len(data) > 0 {
|
||||
partIdx := int(offset / partDataSize)
|
||||
if entry.FileEntry.File.Opts.Circular {
|
||||
|
|
@ -157,8 +170,9 @@ func (entry *CacheEntry) writeAt(offset int64, data []byte) {
|
|||
}
|
||||
|
||||
type BlockStore struct {
|
||||
Lock *sync.Mutex
|
||||
Cache map[cacheKey]*CacheEntry
|
||||
Lock *sync.Mutex
|
||||
Cache map[cacheKey]*CacheEntry
|
||||
NextIntentionId int
|
||||
}
|
||||
|
||||
func makeCacheEntry(blockId string, name string) *CacheEntry {
|
||||
|
|
@ -166,7 +180,7 @@ func makeCacheEntry(blockId string, name string) *CacheEntry {
|
|||
BlockId: blockId,
|
||||
Name: name,
|
||||
PinCount: 0,
|
||||
WriteIntentions: make(map[string]*WriteIntention),
|
||||
WriteIntentions: make(map[int]WriteIntention),
|
||||
FileEntry: nil,
|
||||
DataEntries: nil,
|
||||
}
|
||||
|
|
@ -206,12 +220,36 @@ func (s *BlockStore) pinCacheEntry(blockId string, name string) {
|
|||
entry.PinCount++
|
||||
}
|
||||
|
||||
func (s *BlockStore) setWriteIntention(blockId string, name string, intention WriteIntention) int {
|
||||
s.Lock.Lock()
|
||||
defer s.Lock.Unlock()
|
||||
entry := s.Cache[cacheKey{BlockId: blockId, Name: name}]
|
||||
if entry == nil {
|
||||
return 0
|
||||
}
|
||||
intentionId := s.NextIntentionId
|
||||
s.NextIntentionId++
|
||||
entry.WriteIntentions[intentionId] = intention
|
||||
return intentionId
|
||||
}
|
||||
|
||||
func (s *BlockStore) clearWriteIntention(blockId string, name string, intentionId int) {
|
||||
s.Lock.Lock()
|
||||
defer s.Lock.Unlock()
|
||||
entry := s.Cache[cacheKey{BlockId: blockId, Name: name}]
|
||||
if entry == nil {
|
||||
log.Printf("warning: cannot find write intention to clear %q %q", blockId, name)
|
||||
return
|
||||
}
|
||||
delete(entry.WriteIntentions, intentionId)
|
||||
}
|
||||
|
||||
func (s *BlockStore) unpinCacheEntry(blockId string, name string) {
|
||||
s.Lock.Lock()
|
||||
defer s.Lock.Unlock()
|
||||
entry := s.Cache[cacheKey{BlockId: blockId, Name: name}]
|
||||
if entry == nil {
|
||||
// this is not good
|
||||
log.Printf("warning: unpinning non-existent cache entry %q %q", blockId, name)
|
||||
return
|
||||
}
|
||||
entry.PinCount--
|
||||
|
|
@ -236,6 +274,7 @@ func (s *BlockStore) tryDeleteCacheEntry(blockId string, name string) bool {
|
|||
}
|
||||
|
||||
// getFileFromCache returns the file from the cache if it exists
|
||||
// makes a copy, so it can be used by the caller
|
||||
// return (file, cached)
|
||||
func (s *BlockStore) getFileFromCache(blockId string, name string) (*BlockFile, bool) {
|
||||
s.Lock.Lock()
|
||||
|
|
@ -253,17 +292,20 @@ func (s *BlockStore) getFileFromCache(blockId string, name string) (*BlockFile,
|
|||
return entry.FileEntry.File.DeepCopy(), true
|
||||
}
|
||||
|
||||
func (e *CacheEntry) copyOrCreateFileEntry(dbFile *BlockFile) *FileCacheEntry {
|
||||
if e.FileEntry == nil {
|
||||
return &FileCacheEntry{
|
||||
Dirty: &atomic.Bool{},
|
||||
File: *dbFile,
|
||||
func (e *CacheEntry) modifyFileData(fn func(*BlockFile)) {
|
||||
var fileEntry = e.FileEntry
|
||||
if e.FileEntry.Flushing.Load() {
|
||||
// must make a copy
|
||||
fileEntry = &FileCacheEntry{
|
||||
Dirty: &atomic.Bool{},
|
||||
Flushing: &atomic.Bool{},
|
||||
File: *e.FileEntry.File.DeepCopy(),
|
||||
}
|
||||
e.FileEntry = fileEntry
|
||||
}
|
||||
return &FileCacheEntry{
|
||||
Dirty: &atomic.Bool{},
|
||||
File: *e.FileEntry.File.DeepCopy(),
|
||||
}
|
||||
// always set to dirty (we're modifying it)
|
||||
fileEntry.Dirty.Store(true)
|
||||
fn(&fileEntry.File)
|
||||
}
|
||||
|
||||
// also sets Flushing to true
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ package blockstore
|
|||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"testing"
|
||||
"time"
|
||||
|
|
@ -77,10 +78,27 @@ func TestCreate(t *testing.T) {
|
|||
if file.Opts.Circular || file.Opts.IJson || file.Opts.MaxSize != 0 {
|
||||
t.Fatalf("opts not empty")
|
||||
}
|
||||
blockIds, err := GBS.GetAllBlockIds(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("error getting block ids: %v", err)
|
||||
}
|
||||
if len(blockIds) != 1 {
|
||||
t.Fatalf("block id count mismatch")
|
||||
}
|
||||
if blockIds[0] != blockId {
|
||||
t.Fatalf("block id mismatch")
|
||||
}
|
||||
err = GBS.DeleteFile(ctx, blockId, "testfile")
|
||||
if err != nil {
|
||||
t.Fatalf("error deleting file: %v", err)
|
||||
}
|
||||
blockIds, err = GBS.GetAllBlockIds(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("error getting block ids: %v", err)
|
||||
}
|
||||
if len(blockIds) != 0 {
|
||||
t.Fatalf("block id count mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
func containsFile(arr []*BlockFile, name string) bool {
|
||||
|
|
@ -272,6 +290,130 @@ func TestAppend(t *testing.T) {
|
|||
checkFileData(t, ctx, blockId, fileName, "hello world")
|
||||
}
|
||||
|
||||
func TestWriteFile(t *testing.T) {
|
||||
initDb(t)
|
||||
defer cleanupDb(t)
|
||||
|
||||
ctx, cancelFn := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancelFn()
|
||||
blockId := uuid.New().String()
|
||||
fileName := "t3"
|
||||
err := GBS.MakeFile(ctx, blockId, fileName, nil, FileOptsType{})
|
||||
if err != nil {
|
||||
t.Fatalf("error creating file: %v", err)
|
||||
}
|
||||
err = GBS.WriteFile(ctx, blockId, fileName, []byte("hello world!"))
|
||||
if err != nil {
|
||||
t.Fatalf("error writing data: %v", err)
|
||||
}
|
||||
checkFileData(t, ctx, blockId, fileName, "hello world!")
|
||||
err = GBS.WriteFile(ctx, blockId, fileName, []byte("goodbye world!"))
|
||||
if err != nil {
|
||||
t.Fatalf("error writing data: %v", err)
|
||||
}
|
||||
checkFileData(t, ctx, blockId, fileName, "goodbye world!")
|
||||
err = GBS.WriteFile(ctx, blockId, fileName, []byte("hello"))
|
||||
if err != nil {
|
||||
t.Fatalf("error writing data: %v", err)
|
||||
}
|
||||
checkFileData(t, ctx, blockId, fileName, "hello")
|
||||
|
||||
// circular file
|
||||
err = GBS.MakeFile(ctx, blockId, "c1", nil, FileOptsType{Circular: true, MaxSize: 50})
|
||||
if err != nil {
|
||||
t.Fatalf("error creating file: %v", err)
|
||||
}
|
||||
err = GBS.WriteFile(ctx, blockId, "c1", []byte("123456789 123456789 123456789 123456789 123456789 apple"))
|
||||
if err != nil {
|
||||
t.Fatalf("error writing data: %v", err)
|
||||
}
|
||||
checkFileData(t, ctx, blockId, "c1", "6789 123456789 123456789 123456789 123456789 apple")
|
||||
err = GBS.AppendData(ctx, blockId, "c1", []byte(" banana"))
|
||||
if err != nil {
|
||||
t.Fatalf("error appending data: %v", err)
|
||||
}
|
||||
checkFileData(t, ctx, blockId, "c1", "3456789 123456789 123456789 123456789 apple banana")
|
||||
}
|
||||
|
||||
func TestCircularWrites(t *testing.T) {
|
||||
initDb(t)
|
||||
defer cleanupDb(t)
|
||||
ctx, cancelFn := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancelFn()
|
||||
blockId := uuid.New().String()
|
||||
err := GBS.MakeFile(ctx, blockId, "c1", nil, FileOptsType{Circular: true, MaxSize: 50})
|
||||
if err != nil {
|
||||
t.Fatalf("error creating file: %v", err)
|
||||
}
|
||||
err = GBS.WriteFile(ctx, blockId, "c1", []byte("123456789 123456789 123456789 123456789 123456789 "))
|
||||
if err != nil {
|
||||
t.Fatalf("error writing data: %v", err)
|
||||
}
|
||||
checkFileData(t, ctx, blockId, "c1", "123456789 123456789 123456789 123456789 123456789 ")
|
||||
|
||||
err = GBS.AppendData(ctx, blockId, "c1", []byte("apple"))
|
||||
if err != nil {
|
||||
t.Fatalf("error appending data: %v", err)
|
||||
}
|
||||
checkFileData(t, ctx, blockId, "c1", "6789 123456789 123456789 123456789 123456789 apple")
|
||||
err = GBS.WriteAt(ctx, blockId, "c1", 0, []byte("foo"))
|
||||
if err != nil {
|
||||
t.Fatalf("error writing data: %v", err)
|
||||
}
|
||||
// content should be unchanged because write is before the beginning of circular offset
|
||||
checkFileData(t, ctx, blockId, "c1", "6789 123456789 123456789 123456789 123456789 apple")
|
||||
err = GBS.WriteAt(ctx, blockId, "c1", 5, []byte("a"))
|
||||
if err != nil {
|
||||
t.Fatalf("error writing data: %v", err)
|
||||
}
|
||||
checkFileSize(t, ctx, blockId, "c1", 55)
|
||||
checkFileData(t, ctx, blockId, "c1", "a789 123456789 123456789 123456789 123456789 apple")
|
||||
err = GBS.AppendData(ctx, blockId, "c1", []byte(" banana"))
|
||||
if err != nil {
|
||||
t.Fatalf("error appending data: %v", err)
|
||||
}
|
||||
checkFileSize(t, ctx, blockId, "c1", 62)
|
||||
checkFileData(t, ctx, blockId, "c1", "3456789 123456789 123456789 123456789 apple banana")
|
||||
err = GBS.WriteAt(ctx, blockId, "c1", 20, []byte("foo"))
|
||||
if err != nil {
|
||||
t.Fatalf("error writing data: %v", err)
|
||||
}
|
||||
checkFileSize(t, ctx, blockId, "c1", 62)
|
||||
checkFileData(t, ctx, blockId, "c1", "3456789 foo456789 123456789 123456789 apple banana")
|
||||
offset, _, _ := GBS.ReadFile(ctx, blockId, "c1")
|
||||
if offset != 12 {
|
||||
t.Errorf("offset mismatch: expected 12, got %d", offset)
|
||||
}
|
||||
err = GBS.AppendData(ctx, blockId, "c1", []byte(" world"))
|
||||
if err != nil {
|
||||
t.Fatalf("error appending data: %v", err)
|
||||
}
|
||||
checkFileSize(t, ctx, blockId, "c1", 68)
|
||||
offset, _, _ = GBS.ReadFile(ctx, blockId, "c1")
|
||||
if offset != 18 {
|
||||
t.Errorf("offset mismatch: expected 18, got %d", offset)
|
||||
}
|
||||
checkFileData(t, ctx, blockId, "c1", "9 foo456789 123456789 123456789 apple banana world")
|
||||
err = GBS.AppendData(ctx, blockId, "c1", []byte(" 123456789 123456789 123456789 123456789 bar456789 123456789"))
|
||||
if err != nil {
|
||||
t.Fatalf("error appending data: %v", err)
|
||||
}
|
||||
checkFileSize(t, ctx, blockId, "c1", 128)
|
||||
checkFileData(t, ctx, blockId, "c1", " 123456789 123456789 123456789 bar456789 123456789")
|
||||
GBS.withLock(blockId, "c1", false, func(entry *CacheEntry) {
|
||||
if entry == nil {
|
||||
err = fmt.Errorf("entry not found")
|
||||
return
|
||||
}
|
||||
if len(entry.DataEntries) != 1 {
|
||||
err = fmt.Errorf("data entries mismatch: expected 1, got %d", len(entry.DataEntries))
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("error checking data entries: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func makeText(n int) string {
|
||||
var buf bytes.Buffer
|
||||
for i := 0; i < n; i++ {
|
||||
|
|
|
|||
Loading…
Reference in a new issue