Skip to content

Commit

Permalink
swap from uint64 -> uint32 for index incremental chunk id
Browse files Browse the repository at this point in the history
  • Loading branch information
@poolpOrg
poolpOrg committed Oct 4, 2023
1 parent a8e3a22 commit 401c4f4
Show file tree
Hide file tree
Showing 2 changed files with 69 additions and 67 deletions.
48 changes: 25 additions & 23 deletions cmd/plakar/cmd_info.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,31 +34,31 @@ type JSONChunk struct {
}

type JSONObject struct {
Chunks []uint64
ContentType uint64
Chunks []uint32
ContentType uint32
}

type JSONIndex struct {

// Pathnames -> Object checksum
Pathnames map[string]uint64
Pathnames map[string]uint32

ContentTypes map[string]uint64
ContentTypes map[string]uint32

// Object checksum -> Object
Objects map[uint64]*JSONObject
Objects map[uint32]*JSONObject

// Chunk checksum -> Chunk
Chunks map[uint64]*JSONChunk
Chunks map[uint32]*JSONChunk

// Chunk checksum -> Object checksums
ChunkToObjects map[uint64][]uint64
ChunkToObjects map[uint32][]uint32

// Object checksum -> Filenames
ObjectToPathnames map[uint64][]uint64
ObjectToPathnames map[uint32][]uint32

// Content Type -> Object checksums
ContentTypeToObjects map[uint64][]uint64
ContentTypeToObjects map[uint32][]uint32
}

func init() {
Expand Down Expand Up @@ -103,13 +103,13 @@ func cmd_info(ctx Plakar, repository *storage.Repository, args []string) int {

for _, index := range indexes {
jindex := JSONIndex{}
jindex.Pathnames = make(map[string]uint64)
jindex.ContentTypes = make(map[string]uint64)
jindex.Objects = make(map[uint64]*JSONObject)
jindex.Chunks = make(map[uint64]*JSONChunk)
jindex.ChunkToObjects = make(map[uint64][]uint64)
jindex.ObjectToPathnames = make(map[uint64][]uint64)
jindex.ContentTypeToObjects = make(map[uint64][]uint64)
jindex.Pathnames = make(map[string]uint32)
jindex.ContentTypes = make(map[string]uint32)
jindex.Objects = make(map[uint32]*JSONObject)
jindex.Chunks = make(map[uint32]*JSONChunk)
jindex.ChunkToObjects = make(map[uint32][]uint32)
jindex.ObjectToPathnames = make(map[uint32][]uint32)
jindex.ContentTypeToObjects = make(map[uint32][]uint32)

for pathname, checksumID := range index.Pathnames {
jindex.Pathnames[pathname] = checksumID
Expand All @@ -121,7 +121,7 @@ func cmd_info(ctx Plakar, repository *storage.Repository, args []string) int {

for checksumID, object := range index.Objects {
jobject := &JSONObject{
Chunks: make([]uint64, 0),
Chunks: make([]uint32, 0),
ContentType: object.ContentType,
}

Expand All @@ -141,19 +141,21 @@ func cmd_info(ctx Plakar, repository *storage.Repository, args []string) int {
jindex.Chunks[checksumID] = jchunk
}

for checksum, objects := range index.ChunkToObjects {
jindex.ChunkToObjects[checksum] = make([]uint64, 0)
for _, objChecksum := range objects {
jindex.ChunkToObjects[checksum] = append(jindex.ChunkToObjects[checksum], objChecksum)
/*
for checksum, objects := range index.ChunkToObjects {
jindex.ChunkToObjects[checksum] = make([]uint32, 0)
for _, objChecksum := range objects {
jindex.ChunkToObjects[checksum] = append(jindex.ChunkToObjects[checksum], objChecksum)
}
}
}
*/

for checksumID, pathnames := range index.ObjectToPathnames {
jindex.ObjectToPathnames[checksumID] = pathnames
}

for contentType, objects := range index.ContentTypeToObjects {
jindex.ContentTypeToObjects[contentType] = make([]uint64, 0)
jindex.ContentTypeToObjects[contentType] = make([]uint32, 0)
for _, objChecksum := range objects {
jindex.ContentTypeToObjects[contentType] = append(jindex.ContentTypeToObjects[contentType], objChecksum)
}
Expand Down
88 changes: 44 additions & 44 deletions index/index.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@ import (
)

type IndexObject struct {
Chunks []uint64
ContentType uint64
Chunks []uint32
ContentType uint32
}

type IndexChunk struct {
Expand All @@ -22,54 +22,54 @@ type IndexChunk struct {

type Index struct {
muChecksums sync.Mutex
Checksums map[[32]byte]uint64
checksumsInverse map[uint64][32]byte
Checksums map[[32]byte]uint32
checksumsInverse map[uint32][32]byte

muPathnames sync.Mutex
Pathnames map[string]uint64
pathnamesInverse map[uint64]string
Pathnames map[string]uint32
pathnamesInverse map[uint32]string

muContentType sync.Mutex
ContentTypes map[string]uint64
contentTypesInverse map[uint64]string
ContentTypes map[string]uint32
contentTypesInverse map[uint32]string

muPathnameToObject sync.Mutex
PathnameToObject map[uint64]uint64
ObjectToPathnames map[uint64][]uint64
PathnameToObject map[uint32]uint32
ObjectToPathnames map[uint32][]uint32

// Object checksum -> Object
muObjects sync.Mutex
Objects map[uint64]IndexObject
Objects map[uint32]IndexObject

// Chunk checksum -> Chunk
muChunks sync.Mutex
Chunks map[uint64]IndexChunk
ChunkToObjects map[uint64][]uint64
Chunks map[uint32]IndexChunk
ChunkToObjects map[uint32][]uint32

// Content Type -> Object checksums
muContentTypeToObjects sync.Mutex
ContentTypeToObjects map[uint64][]uint64
ContentTypeToObjects map[uint32][]uint32
}

func NewIndex() *Index {
return &Index{
Checksums: make(map[[32]byte]uint64),
checksumsInverse: make(map[uint64][32]byte),
Checksums: make(map[[32]byte]uint32),
checksumsInverse: make(map[uint32][32]byte),

Pathnames: make(map[string]uint64),
pathnamesInverse: make(map[uint64]string),
Pathnames: make(map[string]uint32),
pathnamesInverse: make(map[uint32]string),

ContentTypes: make(map[string]uint64),
contentTypesInverse: make(map[uint64]string),
ContentTypes: make(map[string]uint32),
contentTypesInverse: make(map[uint32]string),

PathnameToObject: make(map[uint64]uint64),
ObjectToPathnames: make(map[uint64][]uint64),
PathnameToObject: make(map[uint32]uint32),
ObjectToPathnames: make(map[uint32][]uint32),

Objects: make(map[uint64]IndexObject),
Chunks: make(map[uint64]IndexChunk),
ChunkToObjects: make(map[uint64][]uint64),
Objects: make(map[uint32]IndexObject),
Chunks: make(map[uint32]IndexChunk),
ChunkToObjects: make(map[uint32][]uint32),

ContentTypeToObjects: make(map[uint64][]uint64),
ContentTypeToObjects: make(map[uint32][]uint32),
}
}

Expand All @@ -79,17 +79,17 @@ func NewIndexFromBytes(serialized []byte) (*Index, error) {
return nil, err
}

index.checksumsInverse = make(map[uint64][32]byte)
index.checksumsInverse = make(map[uint32][32]byte)
for checksum, checksumID := range index.Checksums {
index.checksumsInverse[checksumID] = checksum
}

index.pathnamesInverse = make(map[uint64]string)
index.pathnamesInverse = make(map[uint32]string)
for pathname, pathnameID := range index.Pathnames {
index.pathnamesInverse[pathnameID] = pathname
}

index.contentTypesInverse = make(map[uint64]string)
index.contentTypesInverse = make(map[uint32]string)
for contentType, contentTypeID := range index.ContentTypes {
index.contentTypesInverse[contentTypeID] = contentType
}
Expand All @@ -111,21 +111,21 @@ func (index *Index) addChecksum(checksum [32]byte) {
defer index.muChecksums.Unlock()

if _, exists := index.Checksums[checksum]; !exists {
checksumID := uint64(len(index.Checksums))
checksumID := uint32(len(index.Checksums))
index.Checksums[checksum] = checksumID
index.checksumsInverse[checksumID] = checksum
}
}

func (index *Index) ChecksumToId(checksum [32]byte) (uint64, bool) {
func (index *Index) ChecksumToId(checksum [32]byte) (uint32, bool) {
index.muChecksums.Lock()
defer index.muChecksums.Unlock()

checksumID, exists := index.Checksums[checksum]
return checksumID, exists
}

func (index *Index) IdToChecksum(checksumID uint64) ([32]byte, bool) {
func (index *Index) IdToChecksum(checksumID uint32) ([32]byte, bool) {
index.muChecksums.Lock()
defer index.muChecksums.Unlock()

Expand All @@ -139,21 +139,21 @@ func (index *Index) addPathname(pathname string) {
defer index.muPathnames.Unlock()

if _, exists := index.Pathnames[pathname]; !exists {
pathnameID := uint64(len(index.Pathnames))
pathnameID := uint32(len(index.Pathnames))
index.Pathnames[pathname] = pathnameID
index.pathnamesInverse[pathnameID] = pathname
}
}

func (index *Index) getPathnameID(pathname string) (uint64, bool) {
func (index *Index) getPathnameID(pathname string) (uint32, bool) {
index.muPathnames.Lock()
defer index.muPathnames.Unlock()

pathnameID, exists := index.Pathnames[pathname]
return pathnameID, exists
}

func (index *Index) getPathname(pathnameID uint64) (string, bool) {
func (index *Index) getPathname(pathnameID uint32) (string, bool) {
index.muPathnames.Lock()
defer index.muPathnames.Unlock()

Expand All @@ -167,13 +167,13 @@ func (index *Index) addContentType(contentType string) {
defer index.muContentType.Unlock()

if _, exists := index.ContentTypes[contentType]; !exists {
contentTypeID := uint64(len(index.ContentTypes))
contentTypeID := uint32(len(index.ContentTypes))
index.ContentTypes[contentType] = contentTypeID
index.contentTypesInverse[contentTypeID] = contentType
}
}

func (index *Index) getContentTypeID(contentType string) (uint64, bool) {
func (index *Index) getContentTypeID(contentType string) (uint32, bool) {
index.muContentType.Lock()
defer index.muContentType.Unlock()

Expand All @@ -182,30 +182,30 @@ func (index *Index) getContentTypeID(contentType string) (uint64, bool) {
return contentTypeID, exists
}

func (index *Index) getContentType(contentTypeID uint64) (string, bool) {
func (index *Index) getContentType(contentTypeID uint32) (string, bool) {
index.muContentType.Lock()
defer index.muContentType.Unlock()

contentType, exists := index.contentTypesInverse[contentTypeID]
return contentType, exists
}

func (index *Index) linkObjectToContentType(checksumID uint64, contentTypeID uint64) {
func (index *Index) linkObjectToContentType(checksumID uint32, contentTypeID uint32) {
index.muContentTypeToObjects.Lock()
defer index.muContentTypeToObjects.Unlock()

if _, exists := index.ContentTypeToObjects[contentTypeID]; !exists {
index.ContentTypeToObjects[contentTypeID] = make([]uint64, 0)
index.ContentTypeToObjects[contentTypeID] = make([]uint32, 0)
}
index.ContentTypeToObjects[contentTypeID] = append(index.ContentTypeToObjects[contentTypeID], checksumID)
}

func (index *Index) linkChunkToObject(chunkChecksumID uint64, objectChecksumID uint64) {
func (index *Index) linkChunkToObject(chunkChecksumID uint32, objectChecksumID uint32) {
index.muChunks.Lock()
defer index.muChunks.Unlock()

if _, exists := index.ChunkToObjects[chunkChecksumID]; !exists {
index.ChunkToObjects[chunkChecksumID] = make([]uint64, 0)
index.ChunkToObjects[chunkChecksumID] = make([]uint32, 0)
}
index.ChunkToObjects[chunkChecksumID] = append(index.ChunkToObjects[chunkChecksumID], objectChecksumID)
}
Expand Down Expand Up @@ -301,7 +301,7 @@ func (index *Index) AddObject(object *objects.Object) {

index.linkObjectToContentType(objectChecksumID, contentTypeID)

chunks := make([]uint64, 0)
chunks := make([]uint32, 0)
for _, checksum := range object.Chunks {
chunkChecksumID, exists := index.ChecksumToId(checksum)
if !exists {
Expand Down Expand Up @@ -337,7 +337,7 @@ func (index *Index) LinkPathnameToObject(pathname string, object *objects.Object

index.PathnameToObject[pathnameID] = checksumID
if _, exists := index.ObjectToPathnames[checksumID]; !exists {
index.ObjectToPathnames[checksumID] = make([]uint64, 0)
index.ObjectToPathnames[checksumID] = make([]uint32, 0)
}
index.ObjectToPathnames[checksumID] = append(index.ObjectToPathnames[checksumID], pathnameID)
}
Expand Down

0 comments on commit 401c4f4

Please sign in to comment.