using System; using System.Collections; using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Linq; using System.Runtime.InteropServices; using Vanara.Extensions; using Vanara.Extensions.Reflection; namespace Vanara.InteropServices { /// A derivative for working with unmanaged memory. /// public class NativeMemoryStream : Stream { private const long DefaultCapacity = 256; private readonly FileAccess access; private readonly long chunkSize = DefaultCapacity; private readonly SafeAllocatedMemoryHandle hmem; private readonly List references = new(); private long capacity, length, position, preflushPos; /// Initializes a new instance of the class. /// The initial capacity. /// Size of additional blocks of memory to add when capacity is exceeded. /// The maximum capacity. /// The mode of file access to the native memory stream. public NativeMemoryStream(long capacity = DefaultCapacity, long addCapacitySize = DefaultCapacity, long maxCapacity = long.MaxValue, FileAccess access = FileAccess.ReadWrite) : this(new SafeCoTaskMemHandle((int)capacity), addCapacitySize, maxCapacity, access) { } /// Initializes a new instance of the class. /// The memory allocator used to create and extend the native memory. /// Size of additional blocks of memory to add when capacity is exceeded. /// The maximum capacity. /// The mode of file access to the native memory stream. /// memoryAllocator public NativeMemoryStream(SafeAllocatedMemoryHandle memoryAllocator, long addCapacitySize = DefaultCapacity, long maxCapacity = long.MaxValue, FileAccess access = FileAccess.ReadWrite) { hmem = memoryAllocator ?? throw new ArgumentNullException(nameof(memoryAllocator)); capacity = memoryAllocator.Size; Pointer = hmem.DangerousGetHandle(); chunkSize = addCapacitySize; MaxCapacity = maxCapacity; this.access = access; } /// Initializes a new instance of the class with a pointer and allows for dynamic growth. /// The pointer to unmanaged, preallocated memory. /// The bytes allocated to . /// The mode of file access to the native memory stream. public NativeMemoryStream(IntPtr unmanagedPtr, long bytesAllocated, FileAccess access = FileAccess.Read) { this.access = access; chunkSize = 0; Pointer = unmanagedPtr; MaxCapacity = capacity = bytesAllocated; } /// Gets a value indicating whether the current stream supports reading. public override bool CanRead => !IsDisposed && access != FileAccess.Write; /// Gets a value indicating whether the current stream supports seeking. public override bool CanSeek => !IsDisposed; /// Gets a value indicating whether the current stream supports writing. public override bool CanWrite => !IsDisposed && access != FileAccess.Read; /// Gets or sets the capacity of the underlying buffer. /// The capacity. public virtual long Capacity { get => ThrowIfDisposed(capacity); protected set { ThrowIfDisposed(); if (hmem is null) throw new InvalidOperationException("This constructed instance does not allow changing the capacity."); if (value < 0 || value > MaxCapacity || (!IsDisposed && value < Length)) throw new ArgumentOutOfRangeException(nameof(value)); if (capacity < value) { hmem.Size = (int)value; Pointer = hmem.DangerousGetHandle(); capacity = value; } } } /// Gets or sets the character set used when processing strings. /// The character set. public CharSet CharSet { get; set; } = CharSet.Auto; /// Gets the length in bytes of the stream. public override long Length => ThrowIfDisposed(length); /// Gets or sets the maximum capacity. /// The maximum capacity. public long MaxCapacity { get; protected set; } /// Gets the pointer to the underlying buffer. public IntPtr Pointer { get; private set; } /// Gets or sets the position within the current stream. public override long Position { get => ThrowIfDisposed(position); set => Seek(value, SeekOrigin.Begin); } /// Gets the position PTR. /// The position PTR. protected IntPtr PositionPtr => Pointer.Offset(Position); private bool IsDisposed => Pointer == IntPtr.Zero || (hmem != null && hmem.IsClosed); /// Ensures the allocated buffer is large enough for the supplied capacity. /// The new capacity. public virtual void EnsureCapacity(long value) { if (value < 0) throw new ArgumentOutOfRangeException(nameof(value)); Debug.WriteLine($"EnsureCap({value}); Capacity={Capacity}; chunk={chunkSize}; max={MaxCapacity}"); if (value <= Capacity) return; Capacity = capacity + Math.Max(value - capacity, chunkSize); Debug.WriteLine($">> NewCapacity={Capacity}"); } /// Clears all buffers for this stream and causes any buffered data to be written to the underlying device. public override void Flush() { preflushPos = Position; foreach (var r in references) { Marshal.WriteIntPtr(Pointer.Offset(r.Offset), PositionPtr); var prelen = length; WriteObject(r.Value); length = prelen; } } /// /// Reads a sequence of bytes from the current stream and advances the position within the stream by the number of bytes read. /// /// /// An array of bytes. When this method returns, the buffer contains the specified byte array with the values between /// and ( + - 1) replaced by the bytes read from the /// current source. /// /// /// The zero-based byte offset in at which to begin storing the data read from the current stream. /// /// The maximum number of bytes to be read from the current stream. /// /// The total number of bytes read into the buffer. This can be less than the number of bytes requested if that many bytes are not /// currently available, or zero (0) if the end of the stream has been reached. /// /// buffer /// /// public override int Read(byte[] buffer, int offset, int count) { if (buffer == null) throw new ArgumentNullException(nameof(buffer)); if (offset + count > buffer.Length) throw new ArgumentException(); if (count < 0) throw new ArgumentOutOfRangeException(nameof(count)); if (offset < 0) throw new ArgumentOutOfRangeException(nameof(offset)); ThrowIfDisposed(); if (!CanRead) throw new NotSupportedException(); if (Position + count > Capacity) throw new ArgumentOutOfRangeException(); if (count > 0) { Marshal.Copy(PositionPtr, buffer, offset, count); Position += count; } return count; } /// /// Reads a blittable type from the current stream and advances the position within the stream by the number of bytes read. /// /// The type of the object to read. /// The character set. /// An object of type . /// Type to be read must be blittable. - T /// public T Read(CharSet charSet = CharSet.Auto) => (T)Read(typeof(T), charSet); /// /// Reads a blittable type from the current stream and advances the position within the stream by the number of bytes read. /// /// The type of the object to read. /// The character set. /// An object of type . /// Type to be read must be blittable. - T /// public object Read(Type typeToRead, CharSet charSet = CharSet.Auto) { if (typeToRead == null) throw new ArgumentNullException(nameof(typeToRead)); ThrowIfDisposed(); if (!CanRead) throw new NotSupportedException(); var ret = PositionPtr.Convert((uint)Capacity - (uint)Position, typeToRead, charSet); Position = position + GetSize(ret); return ret; } /// Reads the array. /// Type of the array element. /// The number of elements in the array. /// if set to , get the values by reference. /// An array of length with values of type . public IEnumerable ReadArray(int fsize, bool byRef) => ReadArray(typeof(T), fsize, byRef).Cast(); /// Reads the array. /// Type of the array element. /// The number of elements in the array. /// if set to , get the values by reference. /// An array of length with values of type . public Array ReadArray(Type elemType, int fsize, bool byRef = false) { if (elemType == null) throw new ArgumentNullException(nameof(elemType)); if (fsize < 0) throw new ArgumentOutOfRangeException(nameof(fsize)); ThrowIfDisposed(); if (!CanRead) throw new NotSupportedException(); if (fsize == 0) return Array.CreateInstance(elemType, 0); var elemSize = elemType == typeof(string) || byRef ? IntPtr.Size : (int)InteropExtensions.SizeOf(elemType); var blockSize = elemSize * fsize; if (Position + blockSize > Capacity) throw new ArgumentOutOfRangeException(); Array ret; if (elemType == typeof(string)) { if (byRef) ret = PositionPtr.ToStringEnum(fsize, CharSet).ToArray(); else { ret = PositionPtr.ToStringEnum(CharSet).ToArray(); if (ret?.GetLength(0) != fsize) throw new ArgumentOutOfRangeException(nameof(fsize)); blockSize = GetSize(ret) + StringHelper.GetCharSize(CharSet); } } else { if (byRef) { ret = Array.CreateInstance(elemType, fsize); var ptrs = PositionPtr.ToArray(fsize); for (var index = 0; index < ptrs.Length; index++) { var ptr = ptrs[index]; ret.SetValue(ptr.Convert((uint)Capacity - (uint)Position, elemType, CharSet), index); } } else ret = PositionPtr.ToArray(elemType, fsize); } Position = position + blockSize; return ret; } /// /// Reads a type reference from the current stream and advances the position within the stream by the number of bytes read. /// /// The type of the object to read. /// The character set. /// An object of type . public T ReadReference(CharSet charSet = CharSet.Auto) => Read().Convert(uint.MaxValue, charSet == CharSet.Auto ? CharSet : charSet); /// /// Reads a blittable type from the current stream and advances the position within the stream by the number of bytes read. /// /// The type of the object to read. /// An object of type . /// Type to be read must be blittable. - T /// public T? ReadNullableReference() where T : struct { var p = Read(); return p != IntPtr.Zero ? (T?)(T)p.Convert((uint)Capacity - (uint)Position, typeof(T), CharSet.Auto) : null; } /// Copies a specified number of bytes from the stream to a memory location. /// /// The pointer to the memory location that will recieve the bytes. The caller must ensure that sufficient memory has been allocated /// to this pointer. /// /// The number of bytes to read. public void ReadToPtr(IntPtr ptr, long bytesToRead) { if (ptr == IntPtr.Zero) throw new ArgumentNullException(nameof(ptr)); if (bytesToRead < 0) throw new ArgumentOutOfRangeException(nameof(bytesToRead)); ThrowIfDisposed(); if (!CanRead) throw new NotSupportedException(); if (bytesToRead == 0) return; if (Position + bytesToRead > Capacity) throw new ArgumentOutOfRangeException(nameof(bytesToRead)); PositionPtr.CopyTo(ptr, bytesToRead); Position = position + bytesToRead; } /// Sets the position within the current stream. /// A byte offset relative to the parameter. /// /// A value of type indicating the reference point used to obtain the new position. /// /// The new position within the current stream. /// public override long Seek(long offset, SeekOrigin origin) { ThrowIfDisposed(); var startPos = origin == SeekOrigin.Begin ? 0L : (origin == SeekOrigin.Current ? Position : Capacity); var reqPos = startPos + offset; if (reqPos < 0 || reqPos > Capacity) throw new ArgumentException(); return position = reqPos; } /// Sets the length of the current stream. /// The desired length of the current stream in bytes. /// public override void SetLength(long value) { if (value < 0) throw new ArgumentOutOfRangeException(nameof(value)); ThrowIfDisposed(); length = value; EnsureCapacity(value); } /// /// Writes a sequence of bytes to the current stream and advances the current position within this stream by the number of bytes written. /// /// /// An array of bytes. This method copies bytes from to the current stream. /// /// /// The zero-based byte offset in at which to begin copying bytes to the current stream. /// /// The number of bytes to be written to the current stream. /// buffer /// /// public override void Write(byte[] buffer, int offset, int count) { if (buffer == null) throw new ArgumentNullException(nameof(buffer)); if (offset + count > buffer.Length) throw new ArgumentException(); if (offset < 0) throw new ArgumentOutOfRangeException(nameof(offset)); if (count < 0) throw new ArgumentOutOfRangeException(nameof(count)); ThrowIfDisposed(); if (access == FileAccess.Read) throw new NotSupportedException(); EnsureCapacity(Position + count); Marshal.Copy(buffer, offset, PositionPtr, count); position += count; length += count; } /// Writes the specified value into the stream. /// The type of the value. /// The value. public void Write(in T value) where T : struct => WriteObject(value); /// Writes the specified string into the stream. /// The string value. /// The character set override. public void Write(string value, CharSet charSetOverride) { var bytes = value.GetBytes(true, charSetOverride); Write(bytes, 0, bytes.Length); } /// Writes the specified string into the stream. /// The string value. public void Write(string value) => Write(value, CharSet); /// Writes the specified array into the stream. /// The type of the array item. /// The items. /// Write values as a referenced array. public void Write(IEnumerable items, bool byRef = false) where T : struct { if (access == FileAccess.Read) throw new NotSupportedException(); if (items == null) return; ResetIfFlushed(); foreach (var i in items) { if (byRef) WriteReferenceObject(i); else WriteObject(i); } } /// Writes the specified array into the stream. /// The items. /// The packing method for the strings. public void Write(IEnumerable items, StringListPackMethod method = StringListPackMethod.Concatenated) { if (access == FileAccess.Read) throw new NotSupportedException(); if (items == null) return; if (method == StringListPackMethod.Concatenated) { items.MarshalToPtr(method, i => { EnsureCapacity(Position + i); return PositionPtr; }, out var sz, CharSet); position += sz; length += sz; } else { foreach (var s in items) WriteReference(s); } } /// Writes bytes from a memory location into the stream. /// /// The pointer to the memory location from which to retrieve the bytes to write. The caller must ensure that this memory location /// has at least of allocated memory. /// /// The number of bytes to write from . public void WriteFromPtr(IntPtr ptr, long bytesToWrite) { if (ptr == IntPtr.Zero) throw new ArgumentNullException(nameof(ptr)); if (bytesToWrite < 0) throw new ArgumentOutOfRangeException(nameof(bytesToWrite)); ThrowIfDisposed(); if (access == FileAccess.Read) throw new NotSupportedException(); EnsureCapacity(Position + bytesToWrite); ptr.CopyTo(PositionPtr, bytesToWrite); position += bytesToWrite; length += bytesToWrite; } /// Writes the specified value into the stream. This function should fail if the object cannot be blitted. /// The value to write. public virtual void WriteObject(object value) { if (access == FileAccess.Read) throw new NotSupportedException(); if (value is null) return; if (value is string s) Write(s); else { ThrowIfDisposed(); var stSize = GetSize(value); EnsureCapacity(stSize + Position + 64); // Added some padding //ResetIfFlushed(); if (value is IntPtr p) Marshal.WriteIntPtr(PositionPtr, p); else stSize = InteropExtensions.WriteNoChecks(PositionPtr, value, 0, Capacity - Position); position += stSize; length += stSize; } } /// /// Writes a reference to the object (memory address as IntPtr) into the stream and then appends the object to the stream when closed /// or flushed. /// /// The value. public void WriteReference(T value) where T : unmanaged => WriteReferenceObject(value); /// /// Writes a reference to the object (memory address as IntPtr) into the stream and then appends the object to the stream when closed /// or flushed. /// /// The value. public void WriteReference(T? value) where T : unmanaged => WriteReferenceObject(value.HasValue ? (object)value.Value : null); /// /// Writes a reference to the string (memory address as IntPtr) into the stream and then appends the string to the stream when closed /// or flushed. /// /// The string value. public void WriteReference(string value) => WriteReferenceObject(value); /// Writes the specified value into the stream. This function should fail if the object cannot be blitted. /// The value to write. public virtual void WriteReferenceObject(object value) { if (access == FileAccess.Read) throw new NotSupportedException(); var sz = GetSize(value); if (sz > 0) { EnsureCapacity(sz + IntPtr.Size + Length + 64); length += sz; references.Add(new Reference(Position, value)); } WriteObject(IntPtr.Zero); } /// /// Releases the unmanaged resources used by the and optionally releases the managed resources. /// /// true to release both managed and unmanaged resources; false to release only unmanaged resources. protected override void Dispose(bool disposing) { Pointer = IntPtr.Zero; base.Dispose(disposing); } /// Gets the size of the object in bytes. /// The object to check. /// The character set. /// The size, in bytes, of the object. protected virtual int GetSize(object obj, CharSet charSet = CharSet.None) { if (charSet == CharSet.None) charSet = CharSet; return obj switch { null => 0, string s => s.GetByteCount(true, charSet), IntPtr p => IntPtr.Size, IEnumerable b => b.Count(), IEnumerable es => es.Sum(s => s.GetByteCount(true, charSet)), IEnumerable eo => eo.Sum(o => o is null ? 0 : InteropExtensions.SizeOf(o)), _ => InteropExtensions.SizeOf(obj), }; } private int GetRefSize() => references.Sum(e => GetSize(e.Value)); private void ResetIfFlushed() { if (preflushPos == 0) return; position = preflushPos; Pointer.Offset(preflushPos).FillMemory(0, GetRefSize()); preflushPos = 0; } private void ThrowIfDisposed() { if (IsDisposed) throw new ObjectDisposedException(nameof(NativeMemoryStream)); } private T ThrowIfDisposed(T value) => !IsDisposed ? value : throw new ObjectDisposedException(nameof(NativeMemoryStream)); private class Reference { public long Offset; public object Value; public Reference(long offset, object val) { Offset = offset; Value = val; } } } }