feat: add a builder setting for a storage file that cannot be recovered

CHANGELOG.md

@@ -8,9 +8,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 ### Changed
 - Storage files are now written on a shared background thread. Changes are still serialized on the calling thread, but `Set` and auto-save no longer wait for the disk. `Save()` and `Dispose()` still block until the data has reached the disk.
+- `Build()` no longer throws when neither the storage file nor its backup can be read: the storage starts empty and logs the `StorageFileCorruptedException`. Every read error in the file now counts as a corrupted file, while an error on opening it still reaches the caller.
 
 ### Added
 - `SaveOnBackgroundThread(bool)` on the builder. Pass `false` to write the file before every change returns, as before.
+- `SetCorruptedFileBehaviour(CorruptedFileBehaviour)` on the builder. Pass `ThrowException` to get the exception from `Build()`, or `ResetToEmpty` to start empty without a log.
 
 ### Removed
 - `IsDirty`. With auto-save on it was almost always `false`, and with the background writer it meant "no change is waiting to be handed over", not "everything is on disk". Use `Save()` when you need the data on disk.

README.md

@@ -95,6 +95,7 @@ using var storage = BinaryStorage.Construct(path)
     .SupportDictionariesOf<string, int>()                 // dictionary of supported types
     .SetMissingKeyBehaviour(MissingKeyBehavior.ReturnDefaultValueOnly)
     .SetTypeMismatchBehaviour(TypeMismatchBehaviour.OverrideValueAndType)
+    .SetCorruptedFileBehaviour(CorruptedFileBehaviour.ResetToEmptyWithError)
     .EnableAutoSaveOnChange()
     .SaveOnBackgroundThread(true)                          // on by default, false writes the file before every change returns
     .Build(KeyLoadFailedBehaviour.IgnoreWithWarning);
@@ -227,6 +228,13 @@ Passed to `Build`; controls what happens when a key fails to deserialize on load
 - `Ignore` - skip the bad key silently.
 - `IgnoreWithWarning` - skip the bad key and log a warning (default).
 
+<!-- omit from toc -->
+### CorruptedFileBehaviour
+Controls what `Build` does when neither the storage file nor its backup can be read. In every mode the unreadable files are removed, so the next start is clean. An error on opening the file (no access, the file is locked) is not a corrupted file: it always reaches the caller and the files stay.
+- `ThrowException` - throw a `StorageFileCorruptedException`.
+- `ResetToEmpty` - start with an empty storage.
+- `ResetToEmptyWithError` - start with an empty storage and log the exception (default).
+
 Per-call overrides are available too: `Get<T>(key, default, overrideMissingKeyBehavior)` and `Set<T>(key, value, overrideTypeMismatchBehaviour)`.
 
 ## License

src/Runtime/BinaryStorage.Builder.cs

@@ -50,6 +50,7 @@ public class Builder
             private bool _saveJsonForDebug;
             private MissingKeyBehavior _missingKeyBehavior = MissingKeyBehavior.InitializeWithDefaultValue;
             private TypeMismatchBehaviour _typeMismatchBehaviour = TypeMismatchBehaviour.ThrowException;
+            private CorruptedFileBehaviour _corruptedFileBehaviour = CorruptedFileBehaviour.ResetToEmptyWithError;
 
             internal Builder(string filePath)
             {
@@ -105,6 +106,16 @@ public Builder SetTypeMismatchBehaviour(TypeMismatchBehaviour behavior)
                 return this;
             }
 
+            /// <summary> Specifies the behavior when neither the storage file nor its backup can be read on load. </summary>
+            /// <remarks> In every mode the unreadable files are removed, so the next load starts clean. An error while the file is opened is not a corrupted file: it always reaches the caller and leaves the files untouched. </remarks>
+            /// <param name="behavior">The corrupted file behavior.</param>
+            /// <returns>The current <see cref="Builder"/> instance for method chaining.</returns>
+            public Builder SetCorruptedFileBehaviour(CorruptedFileBehaviour behavior)
+            {
+                _corruptedFileBehaviour = behavior;
+                return this;
+            }
+
             /// <summary> Adds serializers for primitive types to the storage configuration. </summary>
             /// <returns>The current <see cref="Builder"/> instance for method chaining.</returns>
             public Builder AddPrimitiveTypes()
@@ -248,7 +259,7 @@ public Builder SupportDictionariesOf<TKey, TValue>()
             /// <returns>The configured <see cref="BinaryStorage"/> instance.</returns>
             /// <exception cref="ObjectDisposedException">Thrown if the storage is disposed.</exception>
             /// <exception cref="IOException"> An I/O error occurred </exception>
-            /// <exception cref="StorageFileCorruptedException"> The storage file structure is corrupted (bad header, truncated framing, or a duplicate key). </exception>
+            /// <exception cref="StorageFileCorruptedException"> Neither the storage file nor its backup can be read and the corrupted file behavior is <see cref="CorruptedFileBehaviour.ThrowException"/>. </exception>
             /// <exception cref="KeyLoadFailedException"> A key failed to load and <paramref name="keyLoadFailedBehaviour"/> is <see cref="KeyLoadFailedBehaviour.ThrowException"/>. </exception>
             public BinaryStorage Build(KeyLoadFailedBehaviour keyLoadFailedBehaviour = KeyLoadFailedBehaviour.IgnoreWithWarning)
             {
@@ -257,7 +268,7 @@ public BinaryStorage Build(KeyLoadFailedBehaviour keyLoadFailedBehaviour = KeyLo
                 storage.SaveJsonCopyForDebug = _saveJsonForDebug;
                 storage.MissingKeyBehavior = _missingKeyBehavior;
                 storage.TypeMismatchBehaviour = _typeMismatchBehaviour;
-                storage.LoadDataFromDisk(keyLoadFailedBehaviour);
+                storage.LoadDataFromDisk(keyLoadFailedBehaviour, _corruptedFileBehaviour);
                 LockFilePathInEditor(_filePath);
                 return storage;
             }

src/Runtime/BinaryStorage.cs

@@ -698,12 +698,12 @@ public virtual void Dispose()
         /// <summary> Loads the data from disk into memory. </summary>
         /// <exception cref="ObjectDisposedException">Thrown if the storage is disposed.</exception>
         /// <exception cref="IOException"> An I/O error occurred </exception>
-        /// <exception cref="StorageFileCorruptedException"> The file structure is corrupted (bad header, truncated framing, or a duplicate key). </exception>
+        /// <exception cref="StorageFileCorruptedException"> Neither the file nor its backup can be read and <paramref name="corruptedFileBehaviour"/> is <see cref="CorruptedFileBehaviour.ThrowException"/>. </exception>
         /// <exception cref="KeyLoadFailedException"> A key failed to load and <paramref name="keyLoadFailedBehaviour"/> is <see cref="KeyLoadFailedBehaviour.ThrowException"/>. </exception>
-        private void LoadDataFromDisk(KeyLoadFailedBehaviour keyLoadFailedBehaviour)
+        private void LoadDataFromDisk(KeyLoadFailedBehaviour keyLoadFailedBehaviour, CorruptedFileBehaviour corruptedFileBehaviour)
         {
             ThrowIfDisposed();
-            _persistence.Load(_data, keyLoadFailedBehaviour);
+            _persistence.Load(_data, keyLoadFailedBehaviour, corruptedFileBehaviour);
             foreach (var pair in _data)
             {
                 TrackCollectionOf(pair.Value, pair.Key);

src/Runtime/Persistence/StorageFormat.cs

@@ -65,8 +65,18 @@ internal static void Write(BinaryWriter writer, IReadOnlyList<BinarySection> sec
 
         internal static void ReadFile(string storageFilePath, IReadOnlyList<BinarySection> sections, Dictionary<string, Record> data, KeyLoadFailedBehaviour keyLoadFailedBehaviour)
         {
-            using var stream = new FileStream(storageFilePath, FileMode.Open, FileAccess.Read, FileShare.Read, ReadBufferSize);
-            using var reader = new BinaryReader(stream, Encoding.UTF8);
+            using var stream = OpenFile(storageFilePath);
+            Read(storageFilePath, stream, sections, data, keyLoadFailedBehaviour);
+        }
+
+        internal static FileStream OpenFile(string storageFilePath)
+        {
+            return new FileStream(storageFilePath, FileMode.Open, FileAccess.Read, FileShare.Read, ReadBufferSize);
+        }
+
+        internal static void Read(string storageFilePath, FileStream stream, IReadOnlyList<BinarySection> sections, Dictionary<string, Record> data, KeyLoadFailedBehaviour keyLoadFailedBehaviour)
+        {
+            using var reader = new BinaryReader(stream, Encoding.UTF8, true);
 
             var fileSections = ReadHeader(storageFilePath, sections, reader, out var recordCount);
             var fileLength = stream.Length;

src/Runtime/Persistence/StoragePersistence.cs

@@ -20,10 +20,28 @@ public StoragePersistence(string filePath, IReadOnlyList<BinarySection> sections
             _writer = saveOnBackgroundThread ? new BackgroundStorageWriter(_file) : new ImmediateStorageWriter(_file);
         }
 
-        public void Load(Dictionary<string, Record> data, KeyLoadFailedBehaviour keyLoadFailedBehaviour)
+        public void Load(Dictionary<string, Record> data, KeyLoadFailedBehaviour keyLoadFailedBehaviour, CorruptedFileBehaviour corruptedFileBehaviour)
         {
             _serializer.Clear(data);
-            _file.Load((string filePath, out StorageFileCorruptedException failure) => _serializer.TryDeserialize(filePath, data, keyLoadFailedBehaviour, out failure));
+            try
+            {
+                _file.Load((string filePath, out StorageFileCorruptedException failure) => _serializer.TryDeserialize(filePath, data, keyLoadFailedBehaviour, out failure));
+            }
+            catch (StorageFileCorruptedException exception)
+            {
+                switch (corruptedFileBehaviour)
+                {
+                    case CorruptedFileBehaviour.ThrowException:
+                        throw;
+                    case CorruptedFileBehaviour.ResetToEmpty:
+                        break;
+                    case CorruptedFileBehaviour.ResetToEmptyWithError:
+                        Debug.LogException(exception);
+                        break;
+                    default:
+                        throw new UnexpectedEnumException(typeof(CorruptedFileBehaviour), corruptedFileBehaviour);
+                }
+            }
         }
 
         public void Save(Dictionary<string, Record> data, bool waitForDisk)

src/Runtime/Persistence/StorageSerializer.cs

@@ -1,3 +1,4 @@
+using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Text;
@@ -41,16 +42,17 @@ public StorageSnapshot Serialize(Dictionary<string, Record> data)
 
         public bool TryDeserialize(string filePath, Dictionary<string, Record> data, KeyLoadFailedBehaviour keyLoadFailedBehaviour, out StorageFileCorruptedException failure)
         {
+            using var stream = StorageFormat.OpenFile(filePath);
             try
             {
-                StorageFormat.ReadFile(filePath, _sections, data, keyLoadFailedBehaviour);
+                StorageFormat.Read(filePath, stream, _sections, data, keyLoadFailedBehaviour);
                 failure = null;
                 return true;
             }
-            catch (StorageFileCorruptedException exception)
+            catch (Exception exception) when (exception is not KeyLoadFailedException)
             {
                 Clear(data);
-                failure = exception;
+                failure = exception as StorageFileCorruptedException ?? new StorageFileCorruptedException(filePath, exception.Message, exception);
                 return false;
             }
         }

src/Runtime/Settings/CorruptedFileBehaviour.cs

@@ -0,0 +1,23 @@
+namespace Appegy.Storage
+{
+    /// <summary>
+    /// Specifies the behavior when neither the storage file nor its backup can be read on load.
+    /// </summary>
+    public enum CorruptedFileBehaviour
+    {
+        /// <summary>
+        /// Throws a <see cref="StorageFileCorruptedException"/> from <c>Build</c>.
+        /// </summary>
+        ThrowException,
+
+        /// <summary>
+        /// Starts with an empty storage.
+        /// </summary>
+        ResetToEmpty,
+
+        /// <summary>
+        /// Starts with an empty storage and logs the exception.
+        /// </summary>
+        ResetToEmptyWithError,
+    }
+}

src/Tests/BaseStorageTests.cs

@@ -19,11 +19,12 @@ public void CleanStorageBetweenTests()
             BinaryStorage.Delete(StoragePath);
         }
 
-        protected BinaryStorage Open(bool autoSave = false, bool saveOnBackgroundThread = true)
+        protected BinaryStorage Open(bool autoSave = false, bool saveOnBackgroundThread = true, CorruptedFileBehaviour corruptedFileBehaviour = CorruptedFileBehaviour.ResetToEmptyWithError)
         {
             var builder = BinaryStorage.Construct(StoragePath)
                 .AddPrimitiveTypes()
-                .SaveOnBackgroundThread(saveOnBackgroundThread);
+                .SaveOnBackgroundThread(saveOnBackgroundThread)
+                .SetCorruptedFileBehaviour(corruptedFileBehaviour);
             if (autoSave)
             {
                 builder = builder.EnableAutoSaveOnChange();

src/Tests/Persistence/BackupRecoveryTests.cs

@@ -168,7 +168,7 @@ public void WhenStorageIsCorruptedAndBackupIsCut_ThenThrowsAndRemovesBoth()
             var generations = WriteGenerations(2);
             PutFiles(Array.Empty<byte>(), null, Cut(generations[0]));
 
-            FluentActions.Invoking(() => Open().Dispose()).Should().Throw<StorageFileCorruptedException>();
+            FluentActions.Invoking(() => Open(corruptedFileBehaviour: CorruptedFileBehaviour.ThrowException).Dispose()).Should().Throw<StorageFileCorruptedException>();
 
             File.Exists(StoragePath).Should().BeFalse();
             File.Exists(BackupPath).Should().BeFalse();
@@ -179,7 +179,7 @@ public void WhenStorageAndBackupAreBothCorrupted_ThenTempIsKept()
         {
             PutFiles(Array.Empty<byte>(), new byte[] { 1, 2, 3 }, Array.Empty<byte>());
 
-            FluentActions.Invoking(() => Open().Dispose()).Should().Throw<StorageFileCorruptedException>();
+            FluentActions.Invoking(() => Open(corruptedFileBehaviour: CorruptedFileBehaviour.ThrowException).Dispose()).Should().Throw<StorageFileCorruptedException>();
 
             File.Exists(TempPath).Should().BeTrue();
         }
@@ -191,7 +191,7 @@ public void WhenStorageAndBackupAreBothCorrupted_ThenThrowsAndRemovesBoth()
             File.WriteAllBytes(StoragePath, Array.Empty<byte>());
             File.WriteAllBytes(BackupPath, Array.Empty<byte>());
 
-            FluentActions.Invoking(() => Open().Dispose()).Should().Throw<StorageFileCorruptedException>();
+            FluentActions.Invoking(() => Open(corruptedFileBehaviour: CorruptedFileBehaviour.ThrowException).Dispose()).Should().Throw<StorageFileCorruptedException>();
 
             File.Exists(StoragePath).Should().BeFalse();
             File.Exists(BackupPath).Should().BeFalse();
@@ -203,7 +203,7 @@ public void WhenRestartedAfterTotalFailure_ThenStartsCleanWithoutThrowing()
             WriteTwoGenerations();
             File.WriteAllBytes(StoragePath, Array.Empty<byte>());
             File.WriteAllBytes(BackupPath, Array.Empty<byte>());
-            FluentActions.Invoking(() => Open().Dispose()).Should().Throw<StorageFileCorruptedException>();
+            FluentActions.Invoking(() => Open(corruptedFileBehaviour: CorruptedFileBehaviour.ThrowException).Dispose()).Should().Throw<StorageFileCorruptedException>();
 
             using var storage = Open();