Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

store aggregated stats in childEntry #74

Merged
merged 1 commit into from
Nov 13, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions cmd/plakar/subcommands/info/info.go
Original file line number Diff line number Diff line change
Expand Up @@ -458,9 +458,9 @@ func info_vfs(repo *repository.Repository, snapshotPath string) error {
fmt.Printf("ExtendedAttributes: %s\n", dirEntry.ExtendedAttributes)
fmt.Printf("CustomMetadata: %s\n", dirEntry.CustomMetadata)
fmt.Printf("Tags: %s\n", dirEntry.Tags)
fmt.Printf("AggregateDirs: %d\n", dirEntry.AggregateDirs)
fmt.Printf("AggregateFiles: %d\n", dirEntry.AggregateFiles)
fmt.Printf("AggregateSize: %s (%d bytes)\n", humanize.Bytes(uint64(dirEntry.AggregateSize)), dirEntry.AggregateSize)
fmt.Printf("AggregateDirs: %d\n", dirEntry.AggregatedStats.NDirs)
fmt.Printf("AggregateFiles: %d\n", dirEntry.AggregatedStats.NFiles)
fmt.Printf("AggregateSize: %s (%d bytes)\n", humanize.Bytes(uint64(dirEntry.AggregatedStats.Size)), dirEntry.AggregatedStats.Size)
for offset, child := range dirEntry.Children {
fmt.Printf("Child[%d].Checksum: %x\n", offset, child.Checksum)
fmt.Printf("Child[%d].FileInfo.Name(): %s\n", offset, child.FileInfo.Name())
Expand Down
32 changes: 19 additions & 13 deletions snapshot/backup.go
Original file line number Diff line number Diff line change
Expand Up @@ -116,15 +116,15 @@ func (cache *scanCache) RecordChecksum(pathname string, checksum [32]byte) error
return cache.db.Put([]byte(fmt.Sprintf("__checksum__:%s", pathname)), checksum[:], nil)
}

func (cache *scanCache) RecordAggregates(pathname string, files uint64, dirs uint64, size uint64) error {
func (cache *scanCache) RecordAggregates(pathname string, dirs uint64, files uint64, size uint64) error {
pathname = strings.TrimSuffix(pathname, "/")
if pathname == "" {
pathname = "/"
}

buffer := bytes.NewBuffer(make([]byte, 0, 24))
binary.Write(buffer, binary.LittleEndian, files)
binary.Write(buffer, binary.LittleEndian, dirs)
binary.Write(buffer, binary.LittleEndian, files)
binary.Write(buffer, binary.LittleEndian, size)
return cache.db.Put([]byte(fmt.Sprintf("__aggregate__:%s", pathname)), buffer.Bytes(), nil)
}
Expand Down Expand Up @@ -158,10 +158,10 @@ func (cache *scanCache) GetAggregate(pathname string) (uint64, uint64, uint64, e
var files uint64
var dirs uint64
var size uint64
binary.Read(buffer, binary.LittleEndian, &files)
binary.Read(buffer, binary.LittleEndian, &dirs)
binary.Read(buffer, binary.LittleEndian, &files)
binary.Read(buffer, binary.LittleEndian, &size)
return files, dirs, size, nil
return dirs, files, size, nil
}

func (cache *scanCache) EnumerateKeysWithPrefixReverse(prefix string, isDirectory bool) (<-chan importer.ScanRecord, error) {
Expand Down Expand Up @@ -573,21 +573,27 @@ func (snap *Snapshot) Backup(scanDir string, options *PushOptions) error {
continue
}

var childAggregatedStates *vfs.AggregatedStats
if child.IsDir() {
dirEntry.AggregateDirs++
childAggregatedStates = &vfs.AggregatedStats{}
dirEntry.AggregatedStats.NDirs++

files, dirs, size, err := sc.GetAggregate(childpath)
dirs, files, size, err := sc.GetAggregate(childpath)
if err != nil {
continue
}
dirEntry.AggregateFiles += files
dirEntry.AggregateDirs += dirs
dirEntry.AggregateSize += size
childAggregatedStates.NDirs = dirs
childAggregatedStates.NFiles = files
childAggregatedStates.Size = size

dirEntry.AggregatedStats.NDirs += dirs
dirEntry.AggregatedStats.NFiles += files
dirEntry.AggregatedStats.Size += size
} else {
dirEntry.AggregateFiles++
dirEntry.AggregateSize += uint64(child.Size())
dirEntry.AggregatedStats.NFiles++
dirEntry.AggregatedStats.Size += uint64(child.Size())
}
dirEntry.AddChild(value, child)
dirEntry.AddChild(value, child, childAggregatedStates)
}

serialized, err := dirEntry.Serialize()
Expand All @@ -607,7 +613,7 @@ func (snap *Snapshot) Backup(scanDir string, options *PushOptions) error {
if err != nil {
return err
}
err = sc.RecordAggregates(record.Pathname, dirEntry.AggregateFiles, dirEntry.AggregateDirs, dirEntry.AggregateSize)
err = sc.RecordAggregates(record.Pathname, dirEntry.AggregatedStats.NFiles, dirEntry.AggregatedStats.NDirs, dirEntry.AggregatedStats.Size)
if err != nil {
return err
}
Expand Down
22 changes: 14 additions & 8 deletions snapshot/vfs/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -172,9 +172,16 @@ func (f *FileEntry) FileInfo() *objects.FileInfo {
}
}

type AggregatedStats struct {
NFiles uint64 `msgpack:"NFiles,omitempty"` // Total number of files in the directory
NDirs uint64 `msgpack:"NDirs,omitempty"` // Total number of subdirectories in the directory
Size uint64 `msgpack:"Size,omitempty"` // Total size of all files in the directory
}

type ChildEntry struct {
Checksum [32]byte
FileInfo objects.FileInfo
Checksum [32]byte `msgpack:"checksum"`
FileInfo objects.FileInfo `msgpack:"fileInfo"`
AggregatedStats *AggregatedStats `msgpack:"aggregatedStats,omitempty"`
}

// DirEntry represents the comprehensive structure for a directory entry
Expand All @@ -195,9 +202,7 @@ type DirEntry struct {
CustomMetadata []CustomMetadata `msgpack:"customMetadata,omitempty"` // Custom key-value metadata defined by the user (optional)
Tags []string `msgpack:"tags,omitempty"` // List of tags associated with the directory (optional)
ParentPath string `msgpack:"parentPath,omitempty"` // Path to the parent directory (optional)
AggregateFiles uint64 `msgpack:"aggregateFiles,omitempty"` // Total number of files in the directory
AggregateDirs uint64 `msgpack:"aggregateDirs,omitempty"` // Total number of subdirectories in the directory
AggregateSize uint64 `msgpack:"aggregateSize,omitempty"` // Total size of all files in the directory
AggregatedStats AggregatedStats `msgpack:"aggregatedStats,omitempty"`
}

func (*DirEntry) fsEntry() {}
Expand Down Expand Up @@ -240,10 +245,11 @@ func DirEntryFromBytes(serialized []byte) (*DirEntry, error) {
return &d, nil
}

func (d *DirEntry) AddChild(checksum [32]byte, fileInfo objects.FileInfo) {
func (d *DirEntry) AddChild(checksum [32]byte, fileInfo objects.FileInfo, aggregatedStats *AggregatedStats) {
d.Children = append(d.Children, ChildEntry{
Checksum: checksum,
FileInfo: fileInfo,
Checksum: checksum,
FileInfo: fileInfo,
AggregatedStats: aggregatedStats,
})
}

Expand Down
Loading