How to create fast and efficient stream recordings on large sparse files

Question

I have an application that writes large files in several segments. I use FileStream.Seek to host each wirte. It seems that when I call FileStream.Write in a deep position in a sparse file, the write starts the backfill operation (write 0) to all previous bytes, which is slow.

Is there a better way to deal with this situation?

The code below demonstrates the problem. Initial recording takes about 370 ms on my machine.

public void WriteToStream() { DateTime dt; using (FileStream fs = File.Create("C:\\testfile.file")) { fs.SetLength(1024 * 1024 * 100); fs.Seek(-1, SeekOrigin.End); dt = DateTime.Now; fs.WriteByte(255); } Console.WriteLine(@"WRITE MS: " + DateTime.Now.Subtract(dt).TotalMilliseconds.ToString()); } 

Answer 1

NTFS supports Sparse Files , however there is no way to do this in .net without p / calling some native methods.

It is not so difficult to mark a file as sparse, just know how the file is marked as a sparse file, which it can never be converted back to a non-sparse file, except that it copies the entire file to a new non-sparse file.

Usage example

 class Program { [DllImport("Kernel32.dll", SetLastError = true, CharSet = CharSet.Auto)] private static extern bool DeviceIoControl( SafeFileHandle hDevice, int dwIoControlCode, IntPtr InBuffer, int nInBufferSize, IntPtr OutBuffer, int nOutBufferSize, ref int pBytesReturned, [In] ref NativeOverlapped lpOverlapped ); static void MarkAsSparseFile(SafeFileHandle fileHandle) { int bytesReturned = 0; NativeOverlapped lpOverlapped = new NativeOverlapped(); bool result = DeviceIoControl( fileHandle, 590020, //FSCTL_SET_SPARSE, IntPtr.Zero, 0, IntPtr.Zero, 0, ref bytesReturned, ref lpOverlapped); if(result == false) throw new Win32Exception(); } static void Main() { //Use stopwatch when benchmarking, not DateTime Stopwatch stopwatch = new Stopwatch(); stopwatch.Start(); using (FileStream fs = File.Create(@"e:\Test\test.dat")) { MarkAsSparseFile(fs.SafeFileHandle); fs.SetLength(1024 * 1024 * 100); fs.Seek(-1, SeekOrigin.End); fs.WriteByte(255); } stopwatch.Stop(); //Returns 2 for sparse files and 1127 for non sparse Console.WriteLine(@"WRITE MS: " + stopwatch.ElapsedMilliseconds); } } 

---

Once a file has been marked as sparse, it now behaves just like you, except that it also behaves in the comments. You do not need to write bytes to mark a file in a set size.

 static void Main() { string filename = @"e:\Test\test.dat"; using (FileStream fs = new FileStream(filename, FileMode.Create)) { MarkAsSparseFile(fs.SafeFileHandle); fs.SetLength(1024 * 1024 * 25); } } 

Answer 2

Here are some codes for using sparse files:

 using System; using System.ComponentModel; using System.IO; using System.Runtime.InteropServices; using System.Text; using System.Threading; using Microsoft.Win32.SafeHandles; public static class SparseFiles { private const int FILE_SUPPORTS_SPARSE_FILES = 64; private const int FSCTL_SET_SPARSE = 0x000900c4; private const int FSCTL_SET_ZERO_DATA = 0x000980c8; public static void MakeSparse(this FileStream fileStream) { var bytesReturned = 0; var lpOverlapped = new NativeOverlapped(); var result = DeviceIoControl( fileStream.SafeFileHandle, FSCTL_SET_SPARSE, IntPtr.Zero, 0, IntPtr.Zero, 0, ref bytesReturned, ref lpOverlapped); if (!result) { throw new Win32Exception(); } } public static void SetSparseRange(this FileStream fileStream, long fileOffset, long length) { var fzd = new FILE_ZERO_DATA_INFORMATION(); fzd.FileOffset = fileOffset; fzd.BeyondFinalZero = fileOffset + length; var lpOverlapped = new NativeOverlapped(); var dwTemp = 0; var result = DeviceIoControl( fileStream.SafeFileHandle, FSCTL_SET_ZERO_DATA, ref fzd, Marshal.SizeOf(typeof(FILE_ZERO_DATA_INFORMATION)), IntPtr.Zero, 0, ref dwTemp, ref lpOverlapped); if (!result) { throw new Win32Exception(); } } public static bool SupportedOnVolume(string filename) { var targetVolume = Path.GetPathRoot(filename); var fileSystemName = new StringBuilder(300); var volumeName = new StringBuilder(300); uint lpFileSystemFlags; uint lpVolumeSerialNumber; uint lpMaxComponentLength; var result = GetVolumeInformationW( targetVolume, volumeName, (uint)volumeName.Capacity, out lpVolumeSerialNumber, out lpMaxComponentLength, out lpFileSystemFlags, fileSystemName, (uint)fileSystemName.Capacity); if (!result) { throw new Win32Exception(); } return (lpFileSystemFlags & FILE_SUPPORTS_SPARSE_FILES) == FILE_SUPPORTS_SPARSE_FILES; } [DllImport("Kernel32.dll", SetLastError = true, CharSet = CharSet.Auto)] [return: MarshalAs(UnmanagedType.Bool)] private static extern bool DeviceIoControl( SafeFileHandle hDevice, int dwIoControlCode, IntPtr InBuffer, int nInBufferSize, IntPtr OutBuffer, int nOutBufferSize, ref int pBytesReturned, [In] ref NativeOverlapped lpOverlapped); [DllImport("Kernel32.dll", SetLastError = true, CharSet = CharSet.Auto)] [return: MarshalAs(UnmanagedType.Bool)] private static extern bool DeviceIoControl( SafeFileHandle hDevice, int dwIoControlCode, ref FILE_ZERO_DATA_INFORMATION InBuffer, int nInBufferSize, IntPtr OutBuffer, int nOutBufferSize, ref int pBytesReturned, [In] ref NativeOverlapped lpOverlapped); [DllImport("kernel32.dll", EntryPoint = "GetVolumeInformationW")] [return: MarshalAs(UnmanagedType.Bool)] private static extern bool GetVolumeInformationW( [In] [MarshalAs(UnmanagedType.LPWStr)] string lpRootPathName, [Out] [MarshalAs(UnmanagedType.LPWStr)] StringBuilder lpVolumeNameBuffer, uint nVolumeNameSize, out uint lpVolumeSerialNumber, out uint lpMaximumComponentLength, out uint lpFileSystemFlags, [Out] [MarshalAs(UnmanagedType.LPWStr)] StringBuilder lpFileSystemNameBuffer, uint nFileSystemNameSize); [StructLayout(LayoutKind.Sequential)] private struct FILE_ZERO_DATA_INFORMATION { public long FileOffset; public long BeyondFinalZero; } } 

And a sample code to test the specified class.

 class Program { static void Main(string[] args) { using (var fileStream = new FileStream("test", FileMode.Create, FileAccess.ReadWrite, FileShare.None)) { fileStream.SetLength(1024 * 1024 * 128); fileStream.MakeSparse(); fileStream.SetSparseRange(0, fileStream.Length); } } } 

Hope this helps