From 8cfcee658d1a172cc98f34691b246c0f072e38a3 Mon Sep 17 00:00:00 2001 From: David Hall Date: Mon, 8 Apr 2019 12:03:20 -0600 Subject: [PATCH] Added much improved native memory stream implementation over MarshalingStream. Use NativeMemoryStream moving forward. It is saver and more function rich. --- Core/InteropServices/NativeMemoryStream.cs | 497 +++++++++++++++++++++++++++++ 1 file changed, 497 insertions(+) create mode 100644 Core/InteropServices/NativeMemoryStream.cs diff --git a/Core/InteropServices/NativeMemoryStream.cs b/Core/InteropServices/NativeMemoryStream.cs new file mode 100644 index 00000000..f06a6cde --- /dev/null +++ b/Core/InteropServices/NativeMemoryStream.cs @@ -0,0 +1,497 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using Vanara.Extensions; + +namespace Vanara.InteropServices +{ + /// A derivative for working with unmanaged memory. + /// + public class NativeMemoryStream : Stream + { + private const long DefaultCapacity = 256; + + private readonly FileAccess access; + private readonly long chunkSize = DefaultCapacity; + private readonly SafeAllocatedMemoryHandle hmem; + private readonly List references = new List(); + private long capacity, length, position, preflushPos; + + /// Initializes a new instance of the class. + /// The initial capacity. + /// Size of additional blocks of memory to add when capacity is exceeded. + /// The maximum capacity. + /// The mode of file access to the native memory stream. + public NativeMemoryStream(long capacity = DefaultCapacity, long addCapacitySize = DefaultCapacity, long maxCapacity = long.MaxValue, FileAccess access = FileAccess.ReadWrite) : + this(new SafeCoTaskMemHandle((int)capacity), addCapacitySize, maxCapacity, access) + { + } + + /// Initializes a new instance of the class. + /// The memory allocator used to create and extend the native memory. + /// Size of additional blocks of memory to add when capacity is exceeded. + /// The maximum capacity. + /// The mode of file access to the native memory stream. + /// memoryAllocator + public NativeMemoryStream(SafeAllocatedMemoryHandle memoryAllocator, long addCapacitySize = DefaultCapacity, long maxCapacity = long.MaxValue, FileAccess access = FileAccess.ReadWrite) + { + hmem = memoryAllocator ?? throw new ArgumentNullException(nameof(memoryAllocator)); + capacity = memoryAllocator.Size; + Pointer = hmem.DangerousGetHandle(); + chunkSize = addCapacitySize; + MaxCapacity = maxCapacity; + this.access = access; + } + + /// Initializes a new instance of the class with a pointer and allows for dynamic growth. + /// The pointer to unmanaged, preallocated memory. + /// The bytes allocated to . + public NativeMemoryStream(IntPtr unmanagedPtr, long bytesAllocated) + { + access = FileAccess.Read; + chunkSize = 0; + Pointer = unmanagedPtr; + MaxCapacity = capacity = bytesAllocated; + } + + /// Gets a value indicating whether the current stream supports reading. + public override bool CanRead => !IsDisposed && access != FileAccess.Write; + + /// Gets a value indicating whether the current stream supports seeking. + public override bool CanSeek => !IsDisposed; + + /// Gets a value indicating whether the current stream supports writing. + public override bool CanWrite => !IsDisposed && access != FileAccess.Read; + + /// Gets or sets the capacity of the underlying buffer. + /// The capacity. + public virtual long Capacity + { + get => ThrowIfDisposed(capacity); + protected set + { + ThrowIfDisposed(); + if (hmem is null) + throw new InvalidOperationException("This constructed instance does not allow changing the capacity."); + if (value < 0 || value > MaxCapacity || (!IsDisposed && value < Length)) + throw new ArgumentOutOfRangeException(nameof(value)); + if (capacity < value) + { + hmem.Size = (int)value; + Pointer = hmem.DangerousGetHandle(); + capacity = value; + } + } + } + + /// Gets or sets the character set used when processing strings. + /// The character set. + public CharSet CharSet { get; set; } = CharSet.Auto; + + /// Gets the length in bytes of the stream. + public override long Length => ThrowIfDisposed(length); + + /// Gets or sets the maximum capacity. + /// The maximum capacity. + public long MaxCapacity { get; protected set; } + + /// Gets the pointer to the underlying buffer. + public IntPtr Pointer { get; private set; } + + /// Gets or sets the position within the current stream. + public override long Position + { + get => ThrowIfDisposed(position); + set => Seek(value, SeekOrigin.Begin); + } + + /// Gets the position PTR. + /// The position PTR. + protected IntPtr PositionPtr => Pointer.Offset(Position); + + private bool IsDisposed => Pointer == IntPtr.Zero || (hmem != null && hmem.IsClosed); + + /// Ensures the allocated buffer is large enough for the supplied capacity. + /// The new capacity. + public virtual void EnsureCapacity(long value) + { + if (value < 0) throw new ArgumentOutOfRangeException(nameof(value)); + Debug.WriteLine($"EnsureCap({value}); Capacity={Capacity}; chunk={chunkSize}; max={MaxCapacity}"); + if (value <= Capacity) return; + Capacity = capacity + Math.Max(value - capacity, chunkSize); + Debug.WriteLine($">> NewCapacity={Capacity}"); + } + + /// Clears all buffers for this stream and causes any buffered data to be written to the underlying device. + public override void Flush() + { + preflushPos = Position; + foreach (var r in references) + { + Marshal.WriteIntPtr(Pointer.Offset(r.Offset), PositionPtr); + var prelen = length; + WriteObject(r.Value); + length = prelen; + } + } + + /// + /// Reads a sequence of bytes from the current stream and advances the position within the stream by the number of bytes read. + /// + /// + /// An array of bytes. When this method returns, the buffer contains the specified byte array with the values between + /// and ( + - 1) replaced by the bytes read from the + /// current source. + /// + /// + /// The zero-based byte offset in at which to begin storing the data read from the current stream. + /// + /// The maximum number of bytes to be read from the current stream. + /// + /// The total number of bytes read into the buffer. This can be less than the number of bytes requested if that many bytes are not + /// currently available, or zero (0) if the end of the stream has been reached. + /// + /// buffer + /// + /// + public override int Read(byte[] buffer, int offset, int count) + { + if (buffer == null) throw new ArgumentNullException(nameof(buffer)); + if (offset + count > buffer.Length) throw new ArgumentException(); + if (count < 0) throw new ArgumentOutOfRangeException(nameof(count)); + if (offset < 0) throw new ArgumentOutOfRangeException(nameof(offset)); + ThrowIfDisposed(); + if (!CanRead) throw new NotSupportedException(); + if (Position + count > Capacity) throw new ArgumentOutOfRangeException(); + if (count > 0) + { + Marshal.Copy(PositionPtr, buffer, offset, count); + Position += count; + } + return count; + } + + /// + /// Reads a blittable type from the current stream and advances the position within the stream by the number of bytes read. + /// + /// The type of the object to read. + /// The character set. + /// An object of type . + /// Type to be read must be blittable. - T + /// + public T Read(CharSet charSet = CharSet.Auto) => (T)Read(typeof(T), charSet); + + /// + /// Reads a blittable type from the current stream and advances the position within the stream by the number of bytes read. + /// + /// The type of the object to read. + /// The character set. + /// An object of type . + /// Type to be read must be blittable. - T + /// + public object Read(Type typeToRead, CharSet charSet = CharSet.Auto) + { + if (typeToRead == null) throw new ArgumentNullException(nameof(typeToRead)); + ThrowIfDisposed(); + if (!CanRead) throw new NotSupportedException(); + var ret = PositionPtr.Convert((uint)Capacity - (uint)Position, typeToRead, charSet); + Position = position + GetSize(ret); + return ret; + } + + /// Reads the array. + /// Type of the array element. + /// The number of elements in the array. + /// if set to , get the values by reference. + /// An array of length with values of type . + public IEnumerable ReadArray(int fsize, bool byRef) => ReadArray(typeof(T), fsize, byRef).Cast(); + + /// Reads the array. + /// Type of the array element. + /// The number of elements in the array. + /// if set to , get the values by reference. + /// An array of length with values of type . + public Array ReadArray(Type elemType, int fsize, bool byRef = false) + { + if (elemType == null) throw new ArgumentNullException(nameof(elemType)); + if (fsize < 0) throw new ArgumentOutOfRangeException(nameof(fsize)); + ThrowIfDisposed(); + if (!CanRead) throw new NotSupportedException(); + if (fsize == 0) return Array.CreateInstance(elemType, 0); + var elemSize = elemType == typeof(string) || byRef ? IntPtr.Size : Marshal.SizeOf(elemType); + var blockSize = elemSize * fsize; + if (Position + blockSize > Capacity) throw new ArgumentOutOfRangeException(); + Array ret; + if (elemType == typeof(string)) + { + if (byRef) + ret = PositionPtr.ToStringEnum(fsize, CharSet).ToArray(); + else + { + ret = PositionPtr.ToStringEnum(CharSet).ToArray(); + if (ret?.GetLength(0) != fsize) throw new ArgumentOutOfRangeException(nameof(fsize)); + blockSize = GetSize(ret) + StringHelper.GetCharSize(CharSet); + } + } + else + { + if (byRef) + { + ret = Array.CreateInstance(elemType, fsize); + var ptrs = PositionPtr.ToArray(fsize); + for (var index = 0; index < ptrs.Length; index++) + { + var ptr = ptrs[index]; + ret.SetValue(Marshal.PtrToStructure(ptr, elemType), index); + } + } + else + ret = PositionPtr.ToArray(elemType, fsize); + } + Position = position + blockSize; + return ret; + } + + /// + /// Reads a type reference from the current stream and advances the position within the stream by the number of bytes read. + /// + /// The type of the object to read. + /// The character set. + /// An object of type . + public T ReadReference(CharSet charSet = CharSet.Auto) => Read().Convert(uint.MaxValue, charSet == CharSet.Auto ? CharSet : charSet); + + /// Sets the position within the current stream. + /// A byte offset relative to the parameter. + /// + /// A value of type indicating the reference point used to obtain the new position. + /// + /// The new position within the current stream. + /// + public override long Seek(long offset, SeekOrigin origin) + { + ThrowIfDisposed(); + var startPos = origin == SeekOrigin.Begin ? 0L : (origin == SeekOrigin.Current ? Position : Capacity); + var reqPos = startPos + offset; + if (reqPos < 0 || reqPos > Capacity) + throw new ArgumentException(); + return position = reqPos; + } + + /// Sets the length of the current stream. + /// The desired length of the current stream in bytes. + /// + public override void SetLength(long value) + { + if (value < 0) throw new ArgumentOutOfRangeException(nameof(value)); + ThrowIfDisposed(); + length = value; + EnsureCapacity(value); + } + + /// + /// Writes a sequence of bytes to the current stream and advances the current position within this stream by the number of bytes written. + /// + /// + /// An array of bytes. This method copies bytes from to the current stream. + /// + /// + /// The zero-based byte offset in at which to begin copying bytes to the current stream. + /// + /// The number of bytes to be written to the current stream. + /// buffer + /// + /// + public override void Write(byte[] buffer, int offset, int count) + { + if (buffer == null) throw new ArgumentNullException(nameof(buffer)); + if (offset + count > buffer.Length) throw new ArgumentException(); + if (offset < 0) throw new ArgumentOutOfRangeException(nameof(offset)); + if (count < 0) throw new ArgumentOutOfRangeException(nameof(count)); + ThrowIfDisposed(); + if (access == FileAccess.Read) throw new NotSupportedException(); + EnsureCapacity(Position + count); + Marshal.Copy(buffer, offset, PositionPtr, count); + position += count; + length += count; + } + + /// Writes the specified value into the stream. + /// The type of the value. + /// The value. + public void Write(in T value) where T : unmanaged => WriteObject(value); + + /// Writes the specified string into the stream. + /// The string value. + /// The character set override. + public void Write(string value, CharSet charSetOverride) + { + var bytes = value.GetBytes(true, charSetOverride); + Write(bytes, 0, bytes.Length); + } + + /// Writes the specified string into the stream. + /// The string value. + public void Write(string value) => Write(value, CharSet); + + /// Writes the specified array into the stream. + /// The type of the array item. + /// The items. + /// Write values as a referenced array. + public void Write(IEnumerable items, bool byRef = false) where T : unmanaged + { + if (access == FileAccess.Read) throw new NotSupportedException(); + if (items == null) return; + ResetIfFlushed(); + foreach (var i in items) + { + if (byRef) + WriteReferenceObject(i); + else + WriteObject(i); + } + } + + /// Writes the specified array into the stream. + /// The items. + /// The packing method for the strings. + public void Write(IEnumerable items, StringListPackMethod method = StringListPackMethod.Concatenated) + { + if (access == FileAccess.Read) throw new NotSupportedException(); + if (items == null) return; + if (method == StringListPackMethod.Concatenated) + { + items.MarshalToPtr(method, i => { EnsureCapacity(Position + i); return PositionPtr; }, out var sz, CharSet); + position += sz; + length += sz; + } + else + { + foreach (var s in items) + WriteReference(s); + } + } + + /// Writes the specified value into the stream. This function should fail if the object cannot be blitted. + /// The value to write. + public virtual void WriteObject(object value) + { + if (access == FileAccess.Read) throw new NotSupportedException(); + if (value is null) return; + if (value is string s) + Write(s); + else + { + var stSize = GetSize(value); + EnsureCapacity(stSize + Position); + ResetIfFlushed(); + if (value is IntPtr p) + Marshal.WriteIntPtr(PositionPtr, p); + else + Marshal.StructureToPtr(value, PositionPtr, false); + position += stSize; + length += stSize; + } + } + + /// + /// Writes a reference to the object (memory address as IntPtr) into the stream and then appends the object to the stream when closed + /// or flushed. + /// + /// The value. + public void WriteReference(T value) where T : unmanaged => WriteReferenceObject(value); + + /// + /// Writes a reference to the string (memory address as IntPtr) into the stream and then appends the string to the stream when closed + /// or flushed. + /// + /// The string value. + public void WriteReference(string value) => WriteReferenceObject(value); + + /// Writes the specified value into the stream. This function should fail if the object cannot be blitted. + /// The value to write. + public virtual void WriteReferenceObject(object value) + { + if (access == FileAccess.Read) throw new NotSupportedException(); + var sz = GetSize(value); + if (sz == 0) + { + WriteObject(IntPtr.Zero); + return; + } + EnsureCapacity(sz + IntPtr.Size + Length); + length += sz; + references.Add(new Reference(Position, value)); + WriteObject(IntPtr.Zero); + } + + /// + /// Releases the unmanaged resources used by the and optionally releases the managed resources. + /// + /// true to release both managed and unmanaged resources; false to release only unmanaged resources. + protected override void Dispose(bool disposing) + { + Pointer = IntPtr.Zero; + base.Dispose(disposing); + } + + /// Gets the size of the object in bytes. + /// The object to check. + /// The character set. + /// The size, in bytes, of the object. + protected virtual int GetSize(object obj, CharSet charSet = CharSet.None) + { + if (charSet == CharSet.None) charSet = CharSet; + switch (obj) + { + case null: + return 0; + + case string s: + return s.GetByteCount(true, charSet); + + case IntPtr p: + return IntPtr.Size; + + case IEnumerable es: + return es.Sum(s => s.GetByteCount(true, charSet)); + + case IEnumerable eo: + return eo.Sum(o => o is null ? 0 : Marshal.SizeOf(o)); + + default: + return Marshal.SizeOf(obj); + } + } + + private int GetRefSize() => references.Sum(e => GetSize(e.Value)); + + private void ResetIfFlushed() + { + if (preflushPos == 0) return; + position = preflushPos; + Pointer.Offset(preflushPos).FillMemory(0, GetRefSize()); + preflushPos = 0; + } + + private void ThrowIfDisposed() + { + if (IsDisposed) throw new ObjectDisposedException(nameof(NativeMemoryStream)); + } + + private T ThrowIfDisposed(T value) => !IsDisposed ? value : throw new ObjectDisposedException(nameof(NativeMemoryStream)); + + private class Reference + { + public long Offset; + public object Value; + + public Reference(long offset, object val) + { + Offset = offset; Value = val; + } + } + } +} \ No newline at end of file