Skip to content

Commit 7f4dc1f

Browse files
dschegdscheg
committed
* Pin stream buffers only for a short period of time * libzstd 1.4.5
* Use ZSTD_compressStream2 instead of ZSTD_flushStream and ZSTD_endStream in CompressionStream * libzstd 1.4.5 cross-compiled using `(i686|x86_64)-w64-mingw32-gcc -DZSTD_MULTITHREAD -DZSTD_LEGACY_SUPPORT=0 -pthread -s` instead of msvc for performance reasons
1 parent aaee00a commit 7f4dc1f

17 files changed

+270
-144
lines changed

‎LICENSE

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ BSD License
22

33
For ZstdNet software
44

5-
Copyright (c) 2016-2018, SKB Kontur. All rights reserved.
5+
Copyright (c) 2016-present, SKB Kontur. All rights reserved.
66

77
Redistribution and use in source and binary forms, with or without modification,
88
are permitted provided that the following conditions are met:

‎README.md

+5-5
Original file line numberDiff line numberDiff line change
@@ -36,12 +36,12 @@ Reference
3636

3737
### Requirements
3838

39-
*ZstdNet* requires *Zstdlib* >= v1.0.0. Both 32-bit and 64-bit versions are supported.
40-
The corresponding DLLs (compiled from v1.3.3 using Visual C++) are included in this repository.
39+
*ZstdNet* requires *libzstd* >= v1.0.0. Both 32-bit and 64-bit versions are supported.
40+
The corresponding DLLs (cross-compiled using gcc-mingw-w64) are included in this repository.
4141

4242
### Exceptions
4343

44-
The wrapper throws `ZstdException` in case of malformed data or an error inside *Zstdlib*.
44+
The wrapper throws `ZstdException` in case of malformed data or an error inside *libzstd*.
4545
If the given destination buffer is too small, `InsufficientMemoryException` is thrown away.
4646

4747
### Compressor class
@@ -152,7 +152,7 @@ Allocates buffers for performing decompression. Instances of this class are **no
152152
the size of the destination buffer will be checked before actual decompression.
153153

154154
Note that if this field is malformed (and is less than actual decompressed data size),
155-
*Zstdlib* still doesn't allow a buffer overflow to happen during decompression.
155+
*libzstd* still doesn't allow a buffer overflow to happen during decompression.
156156

157157
* `static ulong GetDecompressedSize(byte[] src)`
158158

@@ -198,6 +198,6 @@ performance and memory overhead.
198198
Wrapper Authors
199199
---------------
200200

201-
Copyright (c) 2016-2017 [SKB Kontur](https://kontur.ru/eng/about)
201+
Copyright (c) 2016-present [SKB Kontur](https://kontur.ru/eng/about)
202202

203203
*ZstdNet* is distributed under [BSD 3-Clause License](LICENSE).

‎ZstdNet.Benchmarks/CompressionBenchmarks.cs

+34-8
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,56 @@
11
using System;
2+
using System.IO;
23
using BenchmarkDotNet.Attributes;
34

45
namespace ZstdNet.Benchmarks
56
{
67
[MemoryDiagnoser]
78
public class CompressionOverheadBenchmarks
89
{
9-
private const int TestSize = 1024;
10+
[Params(1024)] public static int DataSize;
1011

11-
private readonly byte[] UncompressedData = new byte[TestSize];
12+
private readonly byte[] Data;
1213
private readonly byte[] CompressedData;
14+
private readonly byte[] CompressedStreamData;
1315

14-
private readonly byte[] Buffer = new byte[Compressor.GetCompressBound(TestSize)];
16+
private readonly byte[] Buffer;
1517

16-
private readonly Compressor Compressor = new Compressor(new CompressionOptions(1));
18+
private readonly Compressor Compressor = new Compressor(CompressionOptions.Default);
1719
private readonly Decompressor Decompressor = new Decompressor();
1820

1921
public CompressionOverheadBenchmarks()
2022
{
2123
var r = new Random(0);
22-
r.NextBytes(UncompressedData);
2324

24-
CompressedData = Compressor.Wrap(UncompressedData);
25+
Buffer = new byte[Math.Max(DataSize, Compressor.GetCompressBound(DataSize))];
26+
Data = new byte[DataSize];
27+
r.NextBytes(Data);
28+
29+
CompressedData = Compressor.Wrap(Data);
30+
31+
using var tempStream = new MemoryStream();
32+
using var compressionStream = new CompressionStream(tempStream);
33+
new MemoryStream(Data).CopyTo(compressionStream);
34+
CompressedStreamData = tempStream.ToArray();
35+
}
36+
37+
[Benchmark] public void Compress() => Compressor.Wrap(Data, Buffer, 0);
38+
[Benchmark] public void Decompress() => Decompressor.Unwrap(CompressedData, Buffer, 0);
39+
40+
[Benchmark]
41+
[Arguments(7, 13)]
42+
public void CompressStream(int zstdBufferSize, int copyBufferSize)
43+
{
44+
using var compressionStream = new CompressionStream(Stream.Null, CompressionOptions.Default, zstdBufferSize);
45+
new MemoryStream(Data).CopyTo(compressionStream, copyBufferSize);
2546
}
2647

27-
[Benchmark] public void Compress1KBRandom() => Compressor.Wrap(UncompressedData, Buffer, 0);
28-
[Benchmark] public void Decompress1KBRandom() => Decompressor.Unwrap(CompressedData, Buffer, 0);
48+
[Benchmark]
49+
[Arguments(7, 13)]
50+
public void DecompressStream(int zstdBufferSize, int copyBufferSize)
51+
{
52+
using var decompressionStream = new DecompressionStream(new MemoryStream(CompressedStreamData), zstdBufferSize);
53+
decompressionStream.CopyTo(Stream.Null, copyBufferSize);
54+
}
2955
}
3056
}

‎ZstdNet.Benchmarks/Main.cs

+4-2
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,11 @@ namespace ZstdNet.Benchmarks
44
{
55
class Program
66
{
7-
static void Main()
7+
static void Main(string[] args)
88
{
9-
BenchmarkRunner.Run<CompressionOverheadBenchmarks>();
9+
BenchmarkSwitcher
10+
.FromTypes(new[] {typeof(CompressionOverheadBenchmarks)})
11+
.Run(args);
1012
}
1113
}
1214
}

‎ZstdNet.Benchmarks/ZstdNet.Benchmarks.csproj

+3-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,9 @@
22

33
<PropertyGroup>
44
<OutputType>Exe</OutputType>
5-
<TargetFramework>netcoreapp3.1</TargetFramework>
5+
<TargetFrameworks>net48;netcoreapp3.1</TargetFrameworks>
6+
<PlatformTarget>AnyCPU</PlatformTarget>
7+
<LangVersion>8.0</LangVersion>
68
</PropertyGroup>
79

810
<ItemGroup>

‎ZstdNet.Tests/Binding_Tests.cs

+54-3
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
using System;
22
using System.Linq;
33
using System.Text;
4+
using System.Threading;
45
using NUnit.Framework;
56

67
namespace ZstdNet.Tests
@@ -184,6 +185,34 @@ public void Compress_canRead_fromArraySegment([Values(false, true)] bool useDict
184185
CollectionAssert.AreEqual(segment, decompressed);
185186
}
186187

188+
[Test]
189+
public void CompressAndDecompress_workCorrectly_spans([Values(false, true)] bool useDictionary)
190+
{
191+
var buffer = GenerateSample();
192+
193+
var data = new ReadOnlySpan<byte>(buffer, 1, buffer.Length - 1);
194+
var dict = useDictionary ? BuildDictionary() : null;
195+
196+
Span<byte> compressed = stackalloc byte[Compressor.GetCompressBound(data.Length)];
197+
using(var options = new CompressionOptions(dict))
198+
using(var compressor = new Compressor(options))
199+
{
200+
var size = compressor.Wrap(data, compressed);
201+
compressed = compressed.Slice(0, size);
202+
}
203+
204+
Span<byte> decompressed = stackalloc byte[data.Length + 1];
205+
using(var options = new DecompressionOptions(dict))
206+
using(var decompressor = new Decompressor(options))
207+
{
208+
var size = decompressor.Unwrap(compressed, decompressed);
209+
Assert.AreEqual(data.Length, size);
210+
decompressed = decompressed.Slice(0, size);
211+
}
212+
213+
CollectionAssert.AreEqual(data.ToArray(), decompressed.ToArray());
214+
}
215+
187216
[Test]
188217
public void Decompress_canRead_fromArraySegment([Values(false, true)] bool useDictionary)
189218
{
@@ -364,16 +393,38 @@ public void CompressAndDecompress_workCorrectly_ifDifferentInstancesRunInDiffere
364393
});
365394
}
366395

396+
[Test, Explicit("stress")]
397+
public void CompressAndDecompress_workCorrectly_stress([Values(false, true)] bool useDictionary)
398+
{
399+
long i = 0L;
400+
var data = GenerateBuffer(65536);
401+
var dict = useDictionary ? BuildDictionary() : null;
402+
using(var compressionOptions = new CompressionOptions(dict))
403+
using(var decompressionOptions = new DecompressionOptions(dict))
404+
Enumerable.Range(0, 10000)
405+
.AsParallel().WithDegreeOfParallelism(100)
406+
.ForAll(_ =>
407+
{
408+
using(var compressor = new Compressor(compressionOptions))
409+
using(var decompressor = new Decompressor(decompressionOptions))
410+
{
411+
var decompressed = decompressor.Unwrap(compressor.Wrap(data));
412+
if(Interlocked.Increment(ref i) % 100 == 0)
413+
GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced, true, true);
414+
CollectionAssert.AreEqual(data, decompressed);
415+
}
416+
});
417+
}
418+
367419
private static byte[] BuildDictionary()
368420
{
369-
return DictBuilder.TrainFromBuffer(Enumerable.Range(0, 5).Select(_ => GenerateSample()).ToArray(), 1024);
421+
return DictBuilder.TrainFromBuffer(Enumerable.Range(0, 8).Select(_ => GenerateSample()).ToArray(), 1024);
370422
}
371423

372424
private static byte[] GenerateSample()
373425
{
374426
return Enumerable.Range(0, 10)
375-
.SelectMany(_ => Encoding.ASCII.GetBytes(string.Format("['a': 'constant_field', 'b': '{0}', 'c': {1}, 'd': '{2} constant field']",
376-
Random.Next(), Random.Next(), Random.Next(1) == 1 ? "almost" : "sometimes")))
427+
.SelectMany(_ => Encoding.ASCII.GetBytes($"['a': 'constant_field', 'b': '{Random.Next()}', 'c': {Random.Next()}, 'd': '{(Random.Next(1) == 1 ? "almost" : "sometimes")} constant field']"))
377428
.ToArray();
378429
}
379430

‎ZstdNet.Tests/SteamingCompressionTests.cs

+47
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
using System;
22
using System.IO;
3+
using System.Linq;
4+
using System.Threading;
35
using NUnit.Framework;
46

57
namespace ZstdNet.Tests
@@ -69,6 +71,7 @@ public void StreamingCompressionSingleWrite(byte[] data, int offset, int count)
6971
[TestCase(2)]
7072
[TestCase(3)]
7173
[TestCase(5)]
74+
[TestCase(9)]
7275
[TestCase(10)]
7376
public void StreamingDecompressionSingleRead(int readCount)
7477
{
@@ -206,5 +209,49 @@ public void RoundTrip_StreamingToStreaming(
206209

207210
Assert.AreEqual(testStream.ToArray(), resultStream.ToArray());
208211
}
212+
213+
[Test, Explicit("stress")]
214+
public void RoundTrip_StreamingToStreaming_Stress()
215+
{
216+
long i = 0;
217+
Enumerable.Range(0, 10000)
218+
.AsParallel()
219+
.WithDegreeOfParallelism(100)
220+
.ForAll(_ =>
221+
{
222+
var buffer = new byte[13];
223+
var testStream = DataGenerator.GetSmallStream(DataFill.Random);
224+
225+
var tempStream = new MemoryStream();
226+
using(var compressionStream = new CompressionStream(tempStream, 512))
227+
{
228+
int bytesRead;
229+
int offset = (int)(Interlocked.Read(ref i) % buffer.Length);
230+
while((bytesRead = testStream.Read(buffer, offset, buffer.Length - offset)) > 0)
231+
{
232+
compressionStream.Write(buffer, offset, bytesRead);
233+
if(Interlocked.Increment(ref i) % 100 == 0)
234+
GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced, true, true);
235+
}
236+
}
237+
238+
tempStream.Seek(0, SeekOrigin.Begin);
239+
240+
var resultStream = new MemoryStream();
241+
using(var decompressionStream = new DecompressionStream(tempStream, 512))
242+
{
243+
int bytesRead;
244+
int offset = (int)(Interlocked.Read(ref i) % buffer.Length);
245+
while((bytesRead = decompressionStream.Read(buffer, offset, buffer.Length - offset)) > 0)
246+
{
247+
resultStream.Write(buffer, offset, bytesRead);
248+
if(Interlocked.Increment(ref i) % 100 == 0)
249+
GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced, true, true);
250+
}
251+
}
252+
253+
Assert.AreEqual(testStream.ToArray(), resultStream.ToArray());
254+
});
255+
}
209256
}
210257
}

‎ZstdNet.Tests/ZstdNet.Tests.csproj

+1
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
<PropertyGroup>
44
<TargetFramework>net461</TargetFramework>
55
<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
6+
<Platforms>AnyCPU;x64;x86</Platforms>
67
</PropertyGroup>
78

89
<ItemGroup>

‎ZstdNet.sln

+18-18
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11

22
Microsoft Visual Studio Solution File, Format Version 12.00
3-
# Visual Studio 15
4-
VisualStudioVersion = 15.0.27130.2010
3+
# Visual Studio Version 16
4+
VisualStudioVersion = 16.0.30523.141
55
MinimumVisualStudioVersion = 10.0.40219.1
66
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ZstdNet", "ZstdNet\ZstdNet.csproj", "{8ADBEB19-A508-471D-87A0-7443EE086E9B}"
77
EndProject
@@ -21,28 +21,28 @@ Global
2121
GlobalSection(ProjectConfigurationPlatforms) = postSolution
2222
{8ADBEB19-A508-471D-87A0-7443EE086E9B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
2323
{8ADBEB19-A508-471D-87A0-7443EE086E9B}.Debug|Any CPU.Build.0 = Debug|Any CPU
24-
{8ADBEB19-A508-471D-87A0-7443EE086E9B}.Debug|x64.ActiveCfg = Debug|Any CPU
25-
{8ADBEB19-A508-471D-87A0-7443EE086E9B}.Debug|x64.Build.0 = Debug|Any CPU
26-
{8ADBEB19-A508-471D-87A0-7443EE086E9B}.Debug|x86.ActiveCfg = Debug|Any CPU
27-
{8ADBEB19-A508-471D-87A0-7443EE086E9B}.Debug|x86.Build.0 = Debug|Any CPU
24+
{8ADBEB19-A508-471D-87A0-7443EE086E9B}.Debug|x64.ActiveCfg = Debug|x64
25+
{8ADBEB19-A508-471D-87A0-7443EE086E9B}.Debug|x64.Build.0 = Debug|x64
26+
{8ADBEB19-A508-471D-87A0-7443EE086E9B}.Debug|x86.ActiveCfg = Debug|x86
27+
{8ADBEB19-A508-471D-87A0-7443EE086E9B}.Debug|x86.Build.0 = Debug|x86
2828
{8ADBEB19-A508-471D-87A0-7443EE086E9B}.Release|Any CPU.ActiveCfg = Release|Any CPU
2929
{8ADBEB19-A508-471D-87A0-7443EE086E9B}.Release|Any CPU.Build.0 = Release|Any CPU
30-
{8ADBEB19-A508-471D-87A0-7443EE086E9B}.Release|x64.ActiveCfg = Release|Any CPU
31-
{8ADBEB19-A508-471D-87A0-7443EE086E9B}.Release|x64.Build.0 = Release|Any CPU
32-
{8ADBEB19-A508-471D-87A0-7443EE086E9B}.Release|x86.ActiveCfg = Release|Any CPU
33-
{8ADBEB19-A508-471D-87A0-7443EE086E9B}.Release|x86.Build.0 = Release|Any CPU
30+
{8ADBEB19-A508-471D-87A0-7443EE086E9B}.Release|x64.ActiveCfg = Release|x64
31+
{8ADBEB19-A508-471D-87A0-7443EE086E9B}.Release|x64.Build.0 = Release|x64
32+
{8ADBEB19-A508-471D-87A0-7443EE086E9B}.Release|x86.ActiveCfg = Release|x86
33+
{8ADBEB19-A508-471D-87A0-7443EE086E9B}.Release|x86.Build.0 = Release|x86
3434
{8DD3694E-9532-4659-AA9F-BB01CBB9341B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
3535
{8DD3694E-9532-4659-AA9F-BB01CBB9341B}.Debug|Any CPU.Build.0 = Debug|Any CPU
36-
{8DD3694E-9532-4659-AA9F-BB01CBB9341B}.Debug|x64.ActiveCfg = Debug|Any CPU
37-
{8DD3694E-9532-4659-AA9F-BB01CBB9341B}.Debug|x64.Build.0 = Debug|Any CPU
38-
{8DD3694E-9532-4659-AA9F-BB01CBB9341B}.Debug|x86.ActiveCfg = Debug|Any CPU
39-
{8DD3694E-9532-4659-AA9F-BB01CBB9341B}.Debug|x86.Build.0 = Debug|Any CPU
36+
{8DD3694E-9532-4659-AA9F-BB01CBB9341B}.Debug|x64.ActiveCfg = Debug|x64
37+
{8DD3694E-9532-4659-AA9F-BB01CBB9341B}.Debug|x64.Build.0 = Debug|x64
38+
{8DD3694E-9532-4659-AA9F-BB01CBB9341B}.Debug|x86.ActiveCfg = Debug|x86
39+
{8DD3694E-9532-4659-AA9F-BB01CBB9341B}.Debug|x86.Build.0 = Debug|x86
4040
{8DD3694E-9532-4659-AA9F-BB01CBB9341B}.Release|Any CPU.ActiveCfg = Release|Any CPU
4141
{8DD3694E-9532-4659-AA9F-BB01CBB9341B}.Release|Any CPU.Build.0 = Release|Any CPU
42-
{8DD3694E-9532-4659-AA9F-BB01CBB9341B}.Release|x64.ActiveCfg = Release|Any CPU
43-
{8DD3694E-9532-4659-AA9F-BB01CBB9341B}.Release|x64.Build.0 = Release|Any CPU
44-
{8DD3694E-9532-4659-AA9F-BB01CBB9341B}.Release|x86.ActiveCfg = Release|Any CPU
45-
{8DD3694E-9532-4659-AA9F-BB01CBB9341B}.Release|x86.Build.0 = Release|Any CPU
42+
{8DD3694E-9532-4659-AA9F-BB01CBB9341B}.Release|x64.ActiveCfg = Release|x64
43+
{8DD3694E-9532-4659-AA9F-BB01CBB9341B}.Release|x64.Build.0 = Release|x64
44+
{8DD3694E-9532-4659-AA9F-BB01CBB9341B}.Release|x86.ActiveCfg = Release|x86
45+
{8DD3694E-9532-4659-AA9F-BB01CBB9341B}.Release|x86.Build.0 = Release|x86
4646
{6ACD682F-6E1B-4C7E-B60E-66302AD9E2E3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
4747
{6ACD682F-6E1B-4C7E-B60E-66302AD9E2E3}.Debug|Any CPU.Build.0 = Debug|Any CPU
4848
{6ACD682F-6E1B-4C7E-B60E-66302AD9E2E3}.Debug|x64.ActiveCfg = Debug|Any CPU

‎ZstdNet/ArrayHandle.cs

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
using System;
2+
using System.Runtime.InteropServices;
3+
4+
namespace ZstdNet
5+
{
6+
internal struct ArrayHandle : IDisposable
7+
{
8+
public ArrayHandle(byte[] array)
9+
: this(array, 0)
10+
{}
11+
12+
public ArrayHandle(byte[] array, int offset)
13+
{
14+
this.array = array;
15+
this.offset = offset;
16+
handle = GCHandle.Alloc(array, GCHandleType.Pinned);
17+
}
18+
19+
public static implicit operator IntPtr(ArrayHandle pinned)
20+
=> Marshal.UnsafeAddrOfPinnedArrayElement(pinned.array, pinned.offset);
21+
22+
public void Dispose() => handle.Free();
23+
24+
private readonly byte[] array;
25+
private readonly int offset;
26+
private GCHandle handle;
27+
}
28+
}

0 commit comments

Comments
 (0)