use optimized go-fastcdc

go.mod

@@ -3,7 +3,7 @@ module github.com/poolpOrg/plakar
 go 1.18
 
 require (
-	github.com/PlakarLabs/go-fastcdc v0.3.0
+	github.com/PlakarLabs/go-fastcdc v0.4.0
 	github.com/alecthomas/chroma v0.10.0
 	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be
 	github.com/denisbrodbeck/machineid v1.0.1

go.sum

@@ -1,5 +1,5 @@
-github.com/PlakarLabs/go-fastcdc v0.3.0 h1:qm94HT8cnAgpze/Q7JhjB/zNs/orRYeNJo1I2CUmtiE=
-github.com/PlakarLabs/go-fastcdc v0.3.0/go.mod h1:vhDUv02Bos7HbLNyS4onBbOFGu8RoqIq8hCe1UPaGzI=
+github.com/PlakarLabs/go-fastcdc v0.4.0 h1:j6VoYyMDS2tuBZOOxS6TRl7chg1RDeeV4Pz3Vgv8Bm8=
+github.com/PlakarLabs/go-fastcdc v0.4.0/go.mod h1:vhDUv02Bos7HbLNyS4onBbOFGu8RoqIq8hCe1UPaGzI=
 github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek=
 github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s=
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=

snapshot/push.go

@@ -12,8 +12,8 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/gabriel-vasile/mimetype"
 	"github.com/PlakarLabs/go-fastcdc"
+	"github.com/gabriel-vasile/mimetype"
 	"github.com/poolpOrg/plakar/logger"
 	"github.com/poolpOrg/plakar/objects"
 	"github.com/poolpOrg/plakar/progress"
@@ -80,6 +80,7 @@ func chunkify(chunkerOptions *fastcdc.ChunkerOpts, snapshot *Snapshot, pathname
 	}
 
 	firstChunk := true
+	cdcOffset := 0
 	for {
 		cdcChunk, err := chk.Next()
 		if err == io.EOF {
@@ -90,24 +91,25 @@ func chunkify(chunkerOptions *fastcdc.ChunkerOpts, snapshot *Snapshot, pathname
 		}
 		if firstChunk {
 			if object.ContentType == "" {
-				object.ContentType = mimetype.Detect(cdcChunk.Data).String()
+				object.ContentType = mimetype.Detect(cdcChunk).String()
 			}
 			firstChunk = false
 		}
 
-		objectHash.Write(cdcChunk.Data)
+		objectHash.Write(cdcChunk)
 
 		chunkHash := sha256.New()
-		chunkHash.Write(cdcChunk.Data)
+		chunkHash.Write(cdcChunk)
 
 		var t32 [32]byte
 		copy(t32[:], chunkHash.Sum(nil))
 
 		chunk := objects.Chunk{}
 		chunk.Checksum = t32
-		chunk.Start = uint(cdcChunk.Offset)
-		chunk.Length = uint(cdcChunk.Size)
+		chunk.Start = uint(cdcOffset)
+		chunk.Length = uint(len(cdcChunk))
 		object.Chunks = append(object.Chunks, chunk.Checksum)
+		cdcOffset += len(cdcChunk)
 
 		indexChunk := snapshot.Index.LookupChunk(chunk.Checksum)
 		if indexChunk == nil {
@@ -117,7 +119,7 @@ func chunkify(chunkerOptions *fastcdc.ChunkerOpts, snapshot *Snapshot, pathname
 			}
 
 			if !exists {
-				nbytes, err := snapshot.PutChunk(chunk.Checksum, cdcChunk.Data)
+				nbytes, err := snapshot.PutChunk(chunk.Checksum, cdcChunk)
 				if err != nil {
 					return nil, err
 				}