From 5863f6628c331babe1be94670041e9da0662a2df Mon Sep 17 00:00:00 2001 From: Gilles Chehade Date: Mon, 25 Sep 2023 00:49:14 +0200 Subject: [PATCH] Revert "lz4 all the way" This reverts commit 58bb24bb632bac7ced6682821df0d0a8ba51e6cd. --- cmd/plakar/cmd_create.go | 2 +- compression/compression.go | 44 ++--------------------------- compression/compression_test.go | 4 +-- go.mod | 2 -- go.sum | 4 --- snapshot/cacheWrapper.go | 7 ++--- snapshot/snapshot.go | 49 +++++++++++---------------------- storage/fs/fs.go | 8 ++---- 8 files changed, 25 insertions(+), 95 deletions(-) diff --git a/cmd/plakar/cmd_create.go b/cmd/plakar/cmd_create.go index 2056a569..89690a0e 100644 --- a/cmd/plakar/cmd_create.go +++ b/cmd/plakar/cmd_create.go @@ -44,7 +44,7 @@ func cmd_create(ctx Plakar, args []string) int { if opt_nocompression { repositoryConfig.Compression = "" } else { - repositoryConfig.Compression = "lz4" + repositoryConfig.Compression = "gzip" } if !opt_noencryption { diff --git a/compression/compression.go b/compression/compression.go index 5a3882e1..c8449dfb 100644 --- a/compression/compression.go +++ b/compression/compression.go @@ -19,31 +19,10 @@ package compression import ( "bytes" "compress/gzip" - "fmt" "io" - - "github.com/pierrec/lz4/v4" ) -func Deflate(method string, buf []byte) ([]byte, error) { - if method == "gzip" { - return DeflateGzip(buf), nil - } - if method == "lz4" { - return DeflateLZ4(buf), nil - } - return nil, fmt.Errorf("unknown compression method: %s", method) -} - -func DeflateLZ4(buf []byte) []byte { - var b bytes.Buffer - w := lz4.NewWriter(&b) - w.Write(buf) - w.Close() - return b.Bytes() -} - -func DeflateGzip(buf []byte) []byte { +func Deflate(buf []byte) []byte { var b bytes.Buffer w := gzip.NewWriter(&b) w.Write(buf) @@ -51,26 +30,7 @@ func DeflateGzip(buf []byte) []byte { return b.Bytes() } -func Inflate(method string, buf []byte) ([]byte, error) { - if method == "gzip" { - return InflateGzip(buf) - } - if method == "lz4" { - return InflateLZ4(buf) - } - return nil, fmt.Errorf("unknown compression method: %s", method) -} - -func InflateLZ4(buf []byte) ([]byte, error) { - w := lz4.NewReader(bytes.NewBuffer(buf)) - data, err := io.ReadAll(w) - if err != nil { - return nil, err - } - return data, nil -} - -func InflateGzip(buf []byte) ([]byte, error) { +func Inflate(buf []byte) ([]byte, error) { w, err := gzip.NewReader(bytes.NewBuffer(buf)) if err != nil { return nil, err diff --git a/compression/compression_test.go b/compression/compression_test.go index 246994f1..cd88cd06 100644 --- a/compression/compression_test.go +++ b/compression/compression_test.go @@ -25,9 +25,9 @@ import ( func TestCompression(t *testing.T) { token := make([]byte, 65*1024) rand.Read(token) - deflated, _ := Deflate("gzip", token) + deflated := Deflate(token) fmt.Println(deflated) - inflated, err := Inflate("gzip", deflated) + inflated, err := Inflate(deflated) if err != nil { t.Errorf("Inflate(Deflate(%q)) != %q", inflated, token) } diff --git a/go.mod b/go.mod index 211f357c..93984190 100644 --- a/go.mod +++ b/go.mod @@ -40,7 +40,6 @@ require ( github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/rs/xid v1.5.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect @@ -50,5 +49,4 @@ require ( golang.org/x/text v0.12.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.0 // indirect - lukechampine.com/blake3 v1.2.1 // indirect ) diff --git a/go.sum b/go.sum index 9f8fbb42..5d008350 100644 --- a/go.sum +++ b/go.sum @@ -77,8 +77,6 @@ github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= -github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -135,5 +133,3 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA= gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI= -lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= diff --git a/snapshot/cacheWrapper.go b/snapshot/cacheWrapper.go index 1cf5e2d2..0b49ec05 100644 --- a/snapshot/cacheWrapper.go +++ b/snapshot/cacheWrapper.go @@ -49,7 +49,7 @@ func (snapshot *Snapshot) GetCachedObject(pathname string) (*CachedObject, error data = tmp } - data, err = compression.Inflate(snapshot.repository.Configuration().Compression, data) + data, err = compression.Inflate(data) if err != nil { return nil, err } @@ -92,10 +92,7 @@ func (snapshot *Snapshot) PutCachedObject(pathname string, object objects.Object return err } - jobject, err = compression.Deflate(snapshot.repository.Configuration().Compression, jobject) - if err != nil { - return err - } + jobject = compression.Deflate(jobject) if snapshot.repository.Configuration().Encryption != "" { tmp, err := encryption.Encrypt(secret, jobject) if err != nil { diff --git a/snapshot/snapshot.go b/snapshot/snapshot.go index 199ad990..9a206da7 100644 --- a/snapshot/snapshot.go +++ b/snapshot/snapshot.go @@ -186,7 +186,7 @@ func GetMetadata(repository *storage.Repository, indexID uuid.UUID) (*metadata.M } if repository.Configuration().Compression != "" { - tmp, err := compression.Inflate(repository.Configuration().Compression, buffer) + tmp, err := compression.Inflate(buffer) if err != nil { return nil, false, err } @@ -247,7 +247,7 @@ func GetIndex(repository *storage.Repository, indexID uuid.UUID) (*index.Index, } if repository.Configuration().Compression != "" { - tmp, err := compression.Inflate(repository.Configuration().Compression, buffer) + tmp, err := compression.Inflate(buffer) if err != nil { return nil, nil, err } @@ -309,7 +309,7 @@ func GetFilesystem(repository *storage.Repository, indexID uuid.UUID) (*vfs.File } if repository.Configuration().Compression != "" { - tmp, err := compression.Inflate(repository.Configuration().Compression, buffer) + tmp, err := compression.Inflate(buffer) if err != nil { return nil, nil, err } @@ -342,12 +342,8 @@ func (snapshot *Snapshot) PutChunk(checksum [32]byte, data []byte) (int, error) secret := snapshot.repository.GetSecret() buffer := data - var err error if snapshot.repository.Configuration().Compression != "" { - buffer, err = compression.Deflate(snapshot.repository.Configuration().Compression, buffer) - if err != nil { - return 0, err - } + buffer = compression.Deflate(buffer) } if secret != nil { @@ -358,7 +354,7 @@ func (snapshot *Snapshot) PutChunk(checksum [32]byte, data []byte) (int, error) buffer = tmp } - err = snapshot.repository.PutChunk(checksum, buffer) + err := snapshot.repository.PutChunk(checksum, buffer) if err != nil { return 0, err } @@ -381,10 +377,7 @@ func (snapshot *Snapshot) PutObject(object *objects.Object) (int, error) { buffer := data if snapshot.repository.Configuration().Compression != "" { - buffer, err = compression.Deflate(snapshot.repository.Configuration().Compression, buffer) - if err != nil { - return 0, err - } + buffer = compression.Deflate(buffer) } if secret != nil { @@ -412,12 +405,9 @@ func (snapshot *Snapshot) PutMetadata(data []byte) (int, error) { secret := snapshot.repository.GetSecret() buffer := data - var err error + if snapshot.repository.Configuration().Compression != "" { - buffer, err = compression.Deflate(snapshot.repository.Configuration().Compression, buffer) - if err != nil { - return 0, err - } + buffer = compression.Deflate(buffer) } if secret != nil { @@ -432,7 +422,7 @@ func (snapshot *Snapshot) PutMetadata(data []byte) (int, error) { cache.PutMetadata(snapshot.repository.Configuration().RepositoryID.String(), snapshot.Metadata.GetIndexID().String(), buffer) } - err = snapshot.transaction.PutMetadata(buffer) + err := snapshot.transaction.PutMetadata(buffer) if err != nil { return 0, err } @@ -451,12 +441,9 @@ func (snapshot *Snapshot) PutIndex(data []byte) (int, error) { secret := snapshot.repository.GetSecret() buffer := data - var err error + if snapshot.repository.Configuration().Compression != "" { - buffer, err = compression.Deflate(snapshot.repository.Configuration().Compression, buffer) - if err != nil { - return 0, err - } + buffer = compression.Deflate(buffer) } if secret != nil { @@ -471,7 +458,7 @@ func (snapshot *Snapshot) PutIndex(data []byte) (int, error) { cache.PutIndex(snapshot.repository.Configuration().RepositoryID.String(), snapshot.Metadata.GetIndexID().String(), buffer) } - err = snapshot.transaction.PutIndex(buffer) + err := snapshot.transaction.PutIndex(buffer) if err != nil { return 0, err } @@ -489,13 +476,9 @@ func (snapshot *Snapshot) PutFilesystem(data []byte) (int, error) { secret := snapshot.repository.GetSecret() buffer := data - var err error if snapshot.repository.Configuration().Compression != "" { - buffer, err = compression.Deflate(snapshot.repository.Configuration().Compression, buffer) - if err != nil { - return 0, err - } + buffer = compression.Deflate(buffer) } if secret != nil { @@ -510,7 +493,7 @@ func (snapshot *Snapshot) PutFilesystem(data []byte) (int, error) { cache.PutFilesystem(snapshot.repository.Configuration().RepositoryID.String(), snapshot.Metadata.GetIndexID().String(), buffer) } - err = snapshot.transaction.PutFilesystem(buffer) + err := snapshot.transaction.PutFilesystem(buffer) if err != nil { return 0, err } @@ -538,7 +521,7 @@ func (snapshot *Snapshot) GetChunk(checksum [32]byte) ([]byte, error) { } if snapshot.repository.Configuration().Compression != "" { - tmp, err := compression.Inflate(snapshot.repository.Configuration().Compression, buffer) + tmp, err := compression.Inflate(buffer) if err != nil { return nil, err } @@ -581,7 +564,7 @@ func (snapshot *Snapshot) GetObject(checksum [32]byte) (*objects.Object, error) } if snapshot.repository.Configuration().Compression != "" { - tmp, err := compression.Inflate(snapshot.repository.Configuration().Compression, buffer) + tmp, err := compression.Inflate(buffer) if err != nil { return nil, err } diff --git a/storage/fs/fs.go b/storage/fs/fs.go index f544267c..a3e244e9 100644 --- a/storage/fs/fs.go +++ b/storage/fs/fs.go @@ -115,11 +115,7 @@ func (repository *FSRepository) Create(location string, config storage.Repositor return err } - compressedConfig, err := compression.Deflate("lz4", jconfig) - if err != nil { - return err - } - _, err = f.Write(compressedConfig) + _, err = f.Write(compression.Deflate(jconfig)) if err != nil { return err } @@ -141,7 +137,7 @@ func (repository *FSRepository) Open(location string) error { return err } - jconfig, err := compression.Inflate("lz4", compressed) + jconfig, err := compression.Inflate(compressed) if err != nil { return err }