diff --git a/CachedQuickLz/ArrayPool.cs b/CachedQuickLz/ArrayPool.cs old mode 100644 new mode 100755 index 0daa8dd..6b70b40 --- a/CachedQuickLz/ArrayPool.cs +++ b/CachedQuickLz/ArrayPool.cs @@ -1,50 +1,67 @@ -using System; -using System.Collections.Concurrent; - -namespace CachedQuickLz -{ - public class ArrayPool - { - private static readonly ConcurrentDictionary> Bins; - - static ArrayPool() - { - Bins = new ConcurrentDictionary> - { - [0] = new ConcurrentStack() - }; - - for (var i = 0; i < 32; i++) - { - Bins[1 << i] = new ConcurrentStack(); - } - } - - internal static T[] Spawn(int minLength) - { - var count = NextPowerOfTwo(minLength); - return Bins[count].TryPop(out var array) ? array : new T[count]; - } - - internal static void Recycle(T[] array) - { - Array.Clear(array, 0, array.Length); - var binKey = NextPowerOfTwo(array.Length + 1) / 2; - - Bins[binKey].Push(array); - } - - private static int NextPowerOfTwo(int value) - { - var result = value; - - result |= result >> 1; - result |= result >> 2; - result |= result >> 4; - result |= result >> 8; - result |= result >> 16; - - return result + 1; - } - } -} +using System; +using System.Collections.Concurrent; + +namespace CachedQuickLz +{ + public class ArrayPool where T : struct //Value types only! + { + private static readonly ConcurrentDictionary> Bins; + + public static int Size + { + get + { + var result = 0; + foreach (var bin in Bins.Values) + { + foreach (var array in bin) + { + result += array.Length; + } + } + return result; + } + } + + static ArrayPool() + { + Bins = new ConcurrentDictionary>(); + + for (var i = 0; i < 32; i++) + { + Bins[1 << i] = new ConcurrentStack(); + } + } + + internal static T[] Spawn(int minLength) + { + var count = NextPowerOfTwo(minLength); + return Bins[count].TryPop(out var array) ? array : new T[count]; + } + + internal static void Recycle(T[] array) + { + if (array.Length != NextPowerOfTwo(array.Length)) throw new InvalidOperationException("Trying to recycle an array that doesn't fit a bin. Memory leak. Please use arrays made with ArrayPool.Spawn(int)."); + + Array.Clear(array, 0, array.Length); + var binKey = array.Length; + + Bins[binKey].Push(array); + } + + private static int NextPowerOfTwo(int value) + { + if (value <= 0) return 1; + + var result = value - 1; + + result |= result >> 1; + result |= result >> 2; + result |= result >> 4; + result |= result >> 8; + result |= result >> 16; + + return result + 1; + } + } +} diff --git a/CachedQuickLz/CachedQlzDecompress.cs b/CachedQuickLz/CachedQlzDecompress.cs old mode 100644 new mode 100755 index 07f8355..8d8d45a --- a/CachedQuickLz/CachedQlzDecompress.cs +++ b/CachedQuickLz/CachedQlzDecompress.cs @@ -1,209 +1,209 @@ -using System; - -namespace CachedQuickLz -{ - public static partial class CachedQlz - { - /// - /// Decompresses the given array and return the contents into the data parameter. - /// Caution! As the arrays are cached the size of it might be bigger than it's contents. - /// Use to check the array length - /// - /// Data to decompress. The results will be written into this array - /// Length of the decompressed array - public static void Decompress(ref byte[] data, out int length) - { - //When decompressing an empty array, return the original empty array. Otherwise, we'll fail trying to access source[0] later. - if (data.Length == 0) - { - length = 0; - return; - } - - var level = (data[0] >> 2) & 0x3; - if (level != 1 && level != 3) - { - throw new ArgumentException("C# version only supports level 1 and 3"); - } - - length = SizeDecompressed(data); - var src = HeaderLen(data); - var dst = 0; - uint cwordVal = 1; - var destination = ArrayPool.Spawn(length); - var hashtable = ArrayPool.Spawn(4096); - var hashCounter = ArrayPool.Spawn(4096); - var lastMatchstart = length - QlzConstants.UnconditionalMatchlen - QlzConstants.UncompressedEnd - 1; - var lastHashed = -1; - uint fetch = 0; - - if ((data[0] & 1) != 1) - { - Array.Copy(data, HeaderLen(data), destination, 0, length); - ArrayPool.Recycle(hashtable); - ArrayPool.Recycle(hashCounter); - ArrayPool.Recycle(data); - - data = destination; - return; - } - - for (; ; ) - { - if (cwordVal == 1) - { - cwordVal = (uint)(data[src] | (data[src + 1] << 8) | (data[src + 2] << 16) | (data[src + 3] << 24)); - src += 4; - if (dst <= lastMatchstart) - { - if (level == 1) - fetch = (uint)(data[src] | (data[src + 1] << 8) | (data[src + 2] << 16)); - else - fetch = (uint)(data[src] | (data[src + 1] << 8) | (data[src + 2] << 16) | (data[src + 3] << 24)); - } - } - - int hash; - if ((cwordVal & 1) == 1) - { - uint matchlen; - uint offset2; - - cwordVal = cwordVal >> 1; - - if (level == 1) - { - hash = ((int)fetch >> 4) & 0xfff; - offset2 = (uint)hashtable[hash]; - - if ((fetch & 0xf) != 0) - { - matchlen = (fetch & 0xf) + 2; - src += 2; - } - else - { - matchlen = data[src + 2]; - src += 3; - } - } - else - { - uint offset; - if ((fetch & 3) == 0) - { - offset = (fetch & 0xff) >> 2; - matchlen = 3; - src++; - } - else if ((fetch & 2) == 0) - { - offset = (fetch & 0xffff) >> 2; - matchlen = 3; - src += 2; - } - else if ((fetch & 1) == 0) - { - offset = (fetch & 0xffff) >> 6; - matchlen = ((fetch >> 2) & 15) + 3; - src += 2; - } - else if ((fetch & 127) != 3) - { - offset = (fetch >> 7) & 0x1ffff; - matchlen = ((fetch >> 2) & 0x1f) + 2; - src += 3; - } - else - { - offset = fetch >> 15; - matchlen = ((fetch >> 7) & 255) + 3; - src += 4; - } - offset2 = (uint)(dst - offset); - } - - destination[dst + 0] = destination[offset2 + 0]; - destination[dst + 1] = destination[offset2 + 1]; - destination[dst + 2] = destination[offset2 + 2]; - - for (var i = 3; i < matchlen; i += 1) - { - destination[dst + i] = destination[offset2 + i]; - } - - dst += (int)matchlen; - - if (level == 1) - { - fetch = (uint)(destination[lastHashed + 1] | (destination[lastHashed + 2] << 8) | (destination[lastHashed + 3] << 16)); - while (lastHashed < dst - matchlen) - { - lastHashed++; - hash = (int)(((fetch >> 12) ^ fetch) & (QlzConstants.HashValues - 1)); - hashtable[hash] = lastHashed; - hashCounter[hash] = 1; - fetch = (uint)(fetch >> 8 & 0xffff | destination[lastHashed + 3] << 16); - } - fetch = (uint)(data[src] | (data[src + 1] << 8) | (data[src + 2] << 16)); - } - else - { - fetch = (uint)(data[src] | (data[src + 1] << 8) | (data[src + 2] << 16) | (data[src + 3] << 24)); - } - lastHashed = dst - 1; - } - else - { - if (dst <= lastMatchstart) - { - destination[dst] = data[src]; - dst += 1; - src += 1; - cwordVal = cwordVal >> 1; - - if (level == 1) - { - while (lastHashed < dst - 3) - { - lastHashed++; - var fetch2 = destination[lastHashed] | (destination[lastHashed + 1] << 8) | (destination[lastHashed + 2] << 16); - hash = ((fetch2 >> 12) ^ fetch2) & (QlzConstants.HashValues - 1); - hashtable[hash] = lastHashed; - hashCounter[hash] = 1; - } - fetch = (uint)(fetch >> 8 & 0xffff | data[src + 2] << 16); - } - else - { - fetch = (uint)(fetch >> 8 & 0xffff | data[src + 2] << 16 | data[src + 3] << 24); - } - } - else - { - while (dst <= length - 1) - { - if (cwordVal == 1) - { - src += QlzConstants.CwordLen; - cwordVal = 0x80000000; - } - - destination[dst] = data[src]; - dst++; - src++; - cwordVal = cwordVal >> 1; - } - - ArrayPool.Recycle(hashtable); - ArrayPool.Recycle(hashCounter); - break; - } - } - } - - ArrayPool.Recycle(data); - data = destination; - } - } +using System; + +namespace CachedQuickLz +{ + public static partial class CachedQlz + { + /// + /// Decompresses the given array and return the contents into the data parameter. + /// Caution! As the arrays are cached the size of it might be bigger than it's contents. + /// Use to check the array length + /// + /// Data to decompress. The results will be written into this array + /// Length of the decompressed array + public static void Decompress(ref byte[] data, out int length) + { + //When decompressing an empty array, return the original empty array. Otherwise, we'll fail trying to access source[0] later. + if (data.Length == 0) + { + length = 0; + return; + } + + var level = (data[0] >> 2) & 0x3; + if (level != 1 && level != 3) + { + throw new ArgumentException("C# version only supports level 1 and 3"); + } + + length = SizeDecompressed(data); + var src = HeaderLen(data); + var dst = 0; + uint cwordVal = 1; + var destination = ArrayPool.Spawn(length); + var hashtable = ArrayPool.Spawn(4096); + var hashCounter = ArrayPool.Spawn(4096); + var lastMatchstart = length - QlzConstants.UnconditionalMatchlen - QlzConstants.UncompressedEnd - 1; + var lastHashed = -1; + uint fetch = 0; + + if ((data[0] & 1) != 1) + { + Array.Copy(data, HeaderLen(data), destination, 0, length); + ArrayPool.Recycle(hashtable); + ArrayPool.Recycle(hashCounter); + ArrayPool.Recycle(data); + + data = destination; + return; + } + + for (; ; ) + { + if (cwordVal == 1) + { + cwordVal = (uint)(data[src] | (data[src + 1] << 8) | (data[src + 2] << 16) | (data[src + 3] << 24)); + src += 4; + if (dst <= lastMatchstart) + { + if (level == 1) + fetch = (uint)(data[src] | (data[src + 1] << 8) | (data[src + 2] << 16)); + else + fetch = (uint)(data[src] | (data[src + 1] << 8) | (data[src + 2] << 16) | (data[src + 3] << 24)); + } + } + + int hash; + if ((cwordVal & 1) == 1) + { + uint matchlen; + uint offset2; + + cwordVal = cwordVal >> 1; + + if (level == 1) + { + hash = ((int)fetch >> 4) & 0xfff; + offset2 = (uint)hashtable[hash]; + + if ((fetch & 0xf) != 0) + { + matchlen = (fetch & 0xf) + 2; + src += 2; + } + else + { + matchlen = data[src + 2]; + src += 3; + } + } + else + { + uint offset; + if ((fetch & 3) == 0) + { + offset = (fetch & 0xff) >> 2; + matchlen = 3; + src++; + } + else if ((fetch & 2) == 0) + { + offset = (fetch & 0xffff) >> 2; + matchlen = 3; + src += 2; + } + else if ((fetch & 1) == 0) + { + offset = (fetch & 0xffff) >> 6; + matchlen = ((fetch >> 2) & 15) + 3; + src += 2; + } + else if ((fetch & 127) != 3) + { + offset = (fetch >> 7) & 0x1ffff; + matchlen = ((fetch >> 2) & 0x1f) + 2; + src += 3; + } + else + { + offset = fetch >> 15; + matchlen = ((fetch >> 7) & 255) + 3; + src += 4; + } + offset2 = (uint)(dst - offset); + } + + destination[dst + 0] = destination[offset2 + 0]; + destination[dst + 1] = destination[offset2 + 1]; + destination[dst + 2] = destination[offset2 + 2]; + + for (var i = 3; i < matchlen; i += 1) + { + destination[dst + i] = destination[offset2 + i]; + } + + dst += (int)matchlen; + + if (level == 1) + { + fetch = (uint)(destination[lastHashed + 1] | (destination[lastHashed + 2] << 8) | (destination[lastHashed + 3] << 16)); + while (lastHashed < dst - matchlen) + { + lastHashed++; + hash = (int)(((fetch >> 12) ^ fetch) & (QlzConstants.HashValues - 1)); + hashtable[hash] = lastHashed; + hashCounter[hash] = 1; + fetch = (uint)(fetch >> 8 & 0xffff | destination[lastHashed + 3] << 16); + } + fetch = (uint)(data[src] | (data[src + 1] << 8) | (data[src + 2] << 16)); + } + else + { + fetch = (uint)(data[src] | (data[src + 1] << 8) | (data[src + 2] << 16) | (data[src + 3] << 24)); + } + lastHashed = dst - 1; + } + else + { + if (dst <= lastMatchstart) + { + destination[dst] = data[src]; + dst += 1; + src += 1; + cwordVal = cwordVal >> 1; + + if (level == 1) + { + while (lastHashed < dst - 3) + { + lastHashed++; + var fetch2 = destination[lastHashed] | (destination[lastHashed + 1] << 8) | (destination[lastHashed + 2] << 16); + hash = ((fetch2 >> 12) ^ fetch2) & (QlzConstants.HashValues - 1); + hashtable[hash] = lastHashed; + hashCounter[hash] = 1; + } + fetch = (uint)(fetch >> 8 & 0xffff | data[src + 2] << 16); + } + else + { + fetch = (uint)(fetch >> 8 & 0xffff | data[src + 2] << 16 | data[src + 3] << 24); + } + } + else + { + while (dst <= length - 1) + { + if (cwordVal == 1) + { + src += QlzConstants.CwordLen; + cwordVal = 0x80000000; + } + + destination[dst] = data[src]; + dst++; + src++; + cwordVal = cwordVal >> 1; + } + + ArrayPool.Recycle(hashtable); + ArrayPool.Recycle(hashCounter); + break; + } + } + } + + ArrayPool.Recycle(data); + data = destination; + } + } } \ No newline at end of file diff --git a/CachedQuickLzTests/ArrayPoolTests.cs b/CachedQuickLzTests/ArrayPoolTests.cs old mode 100644 new mode 100755 index c58b4db..0f7506d --- a/CachedQuickLzTests/ArrayPoolTests.cs +++ b/CachedQuickLzTests/ArrayPoolTests.cs @@ -1,30 +1,79 @@ -using CachedQuickLz; -using Microsoft.VisualStudio.TestTools.UnitTesting; -using System; - -namespace CachedQuickLzTests -{ - [TestClass] - public class ArrayPoolTests - { - [TestMethod] - public void RequestArray() - { - var array = ArrayPool.Spawn(4); - Assert.AreEqual(8, array.Length); - } - - [TestMethod] - public void RecycleArray() - { - var array = ArrayPool.Spawn(4); - ArrayPool.Recycle(array); - - var memBefore = GC.GetTotalMemory(true); - ArrayPool.Spawn(4); - var memAfter = GC.GetTotalMemory(true); - - Assert.IsTrue(memAfter <= memBefore); - } - } -} +using CachedQuickLz; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; + +namespace CachedQuickLzTests +{ + [TestClass] + public class ArrayPoolTests + { + [TestMethod] + public void RequestArray() + { + var array = ArrayPool.Spawn(4); + Assert.AreEqual(4, array.Length); + } + + [TestMethod] + public void RecycleArray() + { + var array = ArrayPool.Spawn(4); + ArrayPool.Recycle(array); + + var memBefore = GC.GetTotalMemory(true); + ArrayPool.Spawn(4); + var memAfter = GC.GetTotalMemory(true); + + Assert.IsTrue(memAfter <= memBefore); + } + + [TestMethod] + public void RecycleArrayActuallyRecycles() + { + var before = ArrayPool.Spawn(4); + ArrayPool.Recycle(before); + + var after = ArrayPool.Spawn(4); + ArrayPool.Recycle(after); + + Assert.IsTrue(before == after); //reference compare is technically enough + + //But since it's 3am and we know Javascript, we are superstitious + var after2 = ArrayPool.Spawn(4); + before[1] = 123; + Assert.AreEqual(after[1], 123); + Assert.AreEqual(after2[1], 123); //CAVEAT: This also shows there is a dangerous side effect of recycling an array that was given as a parameter. The old owner might keep using it! + } + + [TestMethod] + public void RecycleArrayRejectsMissizedArray() + { + Assert.ThrowsException(() => ArrayPool.Recycle(new byte[10])); + } + + [TestMethod] + public void SizingIsAccurate() + { + var test = ArrayPool.Spawn(0); + Assert.IsTrue(test.Length == 1); + + test = ArrayPool.Spawn(1); + Assert.IsTrue(test.Length == 1); + + test = ArrayPool.Spawn(2); + Assert.IsTrue(test.Length == 2); + + test = ArrayPool.Spawn(3); + Assert.IsTrue(test.Length == 4); + + test = ArrayPool.Spawn(4); + Assert.IsTrue(test.Length == 4); + + test = ArrayPool.Spawn(5); + Assert.IsTrue(test.Length == 8); + + test = ArrayPool.Spawn(65535); + Assert.IsTrue(test.Length == 65536); + } + } +} diff --git a/CachedQuickLzTests/CompressionTests.cs b/CachedQuickLzTests/CompressionTests.cs old mode 100644 new mode 100755 index 02043a0..e1cf83d --- a/CachedQuickLzTests/CompressionTests.cs +++ b/CachedQuickLzTests/CompressionTests.cs @@ -2,6 +2,7 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using System; using System.Text; +using System.Threading; using System.Threading.Tasks; namespace CachedQuickLzTests @@ -12,7 +13,7 @@ public class CompressionTests [TestMethod] public void CompressData_ImpossibleToCompress() { - var originalLength = 100; + var originalLength = 128; var numBytes = originalLength; var data = new byte[numBytes]; @@ -25,7 +26,7 @@ public void CompressData_ImpossibleToCompress() [TestMethod] public void CompressData_NoIssues() { - var originalLength = 5000; + var originalLength = 4096; var numBytes = originalLength; var text = Encoding.ASCII.GetBytes(TestCommon.RandomString(numBytes)); @@ -35,32 +36,10 @@ public void CompressData_NoIssues() } [TestMethod] - public void CompressDataReuseArrays() - { - var numBytes = 4500; - - //Compress a text that uses 4500 bytes - var text = Encoding.ASCII.GetBytes(TestCommon.RandomString(numBytes)); - CachedQlz.Compress(ref text, ref numBytes); - - numBytes = 5500; - - //Now compress another text that uses 5500 bytes. As it has the same - //"next exponential value of 2", it should reuse the array - text = Encoding.ASCII.GetBytes(TestCommon.RandomString(numBytes)); - - var memBefore = GC.GetTotalMemory(true); - CachedQlz.Compress(ref text, ref numBytes); - var memAfter = GC.GetTotalMemory(true); - - Assert.IsTrue(memAfter <= memBefore); - } - - [TestMethod] - public void CompressThreadSafe() + public void CompressThreadSafe() //CAVEAT: Very vague test. Not every threading issue causes an exception. Not every test run will cause these two threads to interleave. This may be more of a static analysis task. { const int iterations = 1000; - const int originalLength = 100000; + const int originalLength = 1024*32; var task1Ok = true; var task1 = Task.Run(() => @@ -89,7 +68,7 @@ public void CompressThreadSafe() { var numBytes = originalLength; var text = Encoding.ASCII.GetBytes(TestCommon.RandomString(numBytes)); - CachedQlz.Compress(ref text, ref numBytes); + CachedQlz.Compress(ref text, ref numBytes); } } catch (Exception) diff --git a/CachedQuickLzTests/DecompressionTests.cs b/CachedQuickLzTests/DecompressionTests.cs old mode 100644 new mode 100755 index f57e773..e9d4958 --- a/CachedQuickLzTests/DecompressionTests.cs +++ b/CachedQuickLzTests/DecompressionTests.cs @@ -12,7 +12,7 @@ public class DecompressionTests [TestMethod] public void DecompressData_ImpossibleToCompress() { - const int originalLength = 100; + const int originalLength = 128; var numBytes = originalLength; var data = new byte[numBytes]; @@ -32,11 +32,39 @@ public void DecompressData_ImpossibleToCompress() } Assert.IsTrue(sequenceEqual); } + + /* + * Because Compress will almost guaranteed create some arrays, I don't know whether this test makes sense. + * + [TestMethod] + public void CompressDecompressMemoryInvariant() + { + const int originalLength = 4096; + var numBytes = originalLength; + + var text = TestCommon.RandomString(numBytes); + var data = ArrayPool.Spawn(numBytes); + + Array.Copy(data, Encoding.ASCII.GetBytes(text), numBytes); + + CachedQlz.Compress(ref data, ref numBytes); + CachedQlz.Decompress(ref data, out var _); + + var before = ArrayPool.Size; + + CachedQlz.Compress(ref data, ref numBytes); + CachedQlz.Decompress(ref data, out var _); + + var after = ArrayPool.Size; + + Assert.AreEqual(before, after); + } + */ [TestMethod] public void DecompressData_NoIssues() { - const int originalLength = 5000; + const int originalLength = 4096; var numBytes = originalLength; var text = TestCommon.RandomString(numBytes); @@ -56,33 +84,12 @@ public void DecompressData_NoIssues() } Assert.IsTrue(sequenceEqual); } - + [TestMethod] - public void DecompressDataReuseArrays() - { - var numBytes = 4500; - - var text = Encoding.ASCII.GetBytes(TestCommon.RandomString(numBytes)); - CachedQlz.Compress(ref text, ref numBytes); - CachedQlz.Decompress(ref text, out _); - - numBytes = 5500; - - text = Encoding.ASCII.GetBytes(TestCommon.RandomString(numBytes)); - CachedQlz.Compress(ref text, ref numBytes); - - var memBefore = GC.GetTotalMemory(true); - CachedQlz.Decompress(ref text, out _); - var memAfter = GC.GetTotalMemory(true); - - Assert.IsTrue(memAfter <= memBefore); - } - - [TestMethod] - public void DecompressThreadSafe() - { - const int iterations = 1000; - var length = 100000; + public void DecompressThreadSafe() //FIXME: Very vague test. Not every threading issue causes an exception. Not every test run will cause these two threads to interleave. This may be more of a static analysis task. + { + const int iterations = 1024; + var length = 1024*32; var data = Encoding.ASCII.GetBytes(TestCommon.RandomString(length)); CachedQlz.Compress(ref data, ref length); diff --git a/CachedQuickLzTests/TestCommon.cs b/CachedQuickLzTests/TestCommon.cs old mode 100644 new mode 100755 index 36df971..54caa16 --- a/CachedQuickLzTests/TestCommon.cs +++ b/CachedQuickLzTests/TestCommon.cs @@ -1,32 +1,32 @@ -using System; -using System.Linq; - -namespace CachedQuickLzTests -{ - public class TestCommon - { - public static string RandomString(int length) - { - var random = new Random(); - const string chars = "ABCDEF"; - return new string(Enumerable.Repeat(chars, length) - .Select(s => s[random.Next(s.Length)]).ToArray()); - } - - public static T[] CloneArray(T[] sourceArray) - { - var clone = new T[sourceArray.Length]; - Array.Copy(sourceArray, clone, sourceArray.Length); - - return clone; - } - - public static T[] CloneArray(T[] sourceArray, int length) - { - var clone = new T[sourceArray.Length]; - Array.Copy(sourceArray, clone, length); - - return clone; - } - } -} +using System; +using System.Linq; + +namespace CachedQuickLzTests +{ + public class TestCommon + { + public static string RandomString(int length) + { + var random = new Random(); + const string chars = "ABCDEF"; + return new string(Enumerable.Repeat(chars, length) + .Select(s => s[random.Next(s.Length)]).ToArray()); + } + + public static T[] CloneArray(T[] sourceArray) + { + var clone = new T[sourceArray.Length]; + Array.Copy(sourceArray, clone, sourceArray.Length); + + return clone; + } + + public static T[] CloneArray(T[] sourceArray, int length) + { + var clone = new T[sourceArray.Length]; + Array.Copy(sourceArray, clone, length); + + return clone; + } + } +}