Dynamic table growth and data region shifting support

Implement dynamic table growth in the storage engine by tracking reserved record capacity per table and shifting subsequent tables' data regions when a table exceeds its reserved space. Add schema validation for record size consistency. Introduce a comprehensive test to verify data integrity across insert, update, delete, and reload cycles with table growth. Ensures robust handling of tables exceeding initial capacity without data loss.

Author: niltor

src/Perigon.MiniDb/StorageManager.cs

@@ -13,6 +13,7 @@ public class TableMetadata
     public int RecordCount { get; set; }
     public int RecordSize { get; set; }
     public long DataStartOffset { get; set; }
+    public int ReservedRecordCount { get; set; }
     /// <summary>
     /// The index of this table in the file header metadata section.
     /// This ensures consistent offset calculations across reloads.
@@ -88,13 +89,14 @@ private void CreateDatabase(Dictionary<string, Type> tableTypes)
                 RecordCount = 0,
                 RecordSize = metadata.RecordSize,
                 DataStartOffset = currentOffset,
+                ReservedRecordCount = INCREASE_RECORD_SIZE,
                 TableIndex = tableIndex
             };
             _tables[kvp.Key] = tableMetadata;
 
             WriteTableMetadata(writer, tableMetadata);
             tableIndex++;
-            currentOffset += metadata.RecordSize * INCREASE_RECORD_SIZE;
+            currentOffset += metadata.RecordSize * tableMetadata.ReservedRecordCount;
         }
 
         // Freeze the metadata cache after initialization
@@ -117,9 +119,10 @@ private void WriteTableMetadata(BinaryWriter writer, TableMetadata metadata)
         writer.Write(metadata.RecordCount);
         writer.Write(metadata.RecordSize);
         writer.Write(metadata.DataStartOffset);
+        writer.Write(metadata.ReservedRecordCount);
         writer.Write(metadata.TableIndex);
 
-        Span<byte> reserved = stackalloc byte[44];
+        Span<byte> reserved = stackalloc byte[40];
         reserved.Clear();
         writer.Write(reserved);
     }
@@ -149,15 +152,17 @@ private void LoadDatabase()
             var recordCount = reader.ReadInt32();
             var recordSize = reader.ReadInt32();
             var dataStartOffset = reader.ReadInt64();
+            var reservedRecordCount = reader.ReadInt32();
             var tableIndex = reader.ReadInt32();
-            reader.ReadBytes(44); // Skip reserved
+            reader.ReadBytes(40); // Skip reserved
 
             _tables[tableName] = new TableMetadata
             {
                 TableName = tableName,
                 RecordCount = recordCount,
                 RecordSize = recordSize,
                 DataStartOffset = dataStartOffset,
+                ReservedRecordCount = reservedRecordCount > 0 ? reservedRecordCount : INCREASE_RECORD_SIZE,
                 TableIndex = tableIndex
             };
         }
@@ -173,6 +178,11 @@ private void LoadDatabase()
             return result;
 
         var entityMetadata = GetOrCreateEntityMetadata(typeof(T));
+        if (tableMetadata.RecordSize != entityMetadata.RecordSize)
+        {
+            throw new InvalidDataException(
+                $"Schema mismatch for table '{tableName}': file RecordSize={tableMetadata.RecordSize}, expected RecordSize={entityMetadata.RecordSize} for entity '{typeof(T).FullName}'.");
+        }
         byte[]? rentedBuffer = null;
 
         try
@@ -222,13 +232,19 @@ private async Task SaveChangesInternalAsync<T>(string tableName, List<T> added,
     {
         var tableMetadata = _tables[tableName];
         var entityMetadata = GetOrCreateEntityMetadata(typeof(T));
+        if (tableMetadata.RecordSize != entityMetadata.RecordSize)
+        {
+            throw new InvalidDataException(
+                $"Schema mismatch for table '{tableName}': file RecordSize={tableMetadata.RecordSize}, expected RecordSize={entityMetadata.RecordSize} for entity '{typeof(T).FullName}'.");
+        }
 
         await using var file = new FileStream(_filePath, FileMode.Open, FileAccess.ReadWrite, FileShare.Read,
             bufferSize: 4096, useAsync: true);
 
         // Handle added records
         foreach (var entity in added)
         {
+            await EnsureCapacityForAppendAsync(tableName, file, cancellationToken);
             var buffer = SerializeRecord(entity, entityMetadata);
             file.Seek(tableMetadata.DataStartOffset + (tableMetadata.RecordCount * tableMetadata.RecordSize), SeekOrigin.Begin);
             await file.WriteAsync(buffer, cancellationToken);
@@ -274,6 +290,85 @@ private async Task SaveChangesInternalAsync<T>(string tableName, List<T> added,
         await file.FlushAsync(cancellationToken);
     }
 
+    private async Task EnsureCapacityForAppendAsync(string tableName, FileStream file, CancellationToken cancellationToken)
+    {
+        var tableMetadata = _tables[tableName];
+        if (tableMetadata.RecordCount < tableMetadata.ReservedRecordCount)
+        {
+            return;
+        }
+
+        int growByRecords = Math.Max(INCREASE_RECORD_SIZE, tableMetadata.ReservedRecordCount);
+        long growByBytes = (long)growByRecords * tableMetadata.RecordSize;
+
+        var ordered = _tables.Values
+            .OrderBy(t => t.DataStartOffset)
+            .ToArray();
+
+        int tableIndex = Array.FindIndex(ordered, t => string.Equals(t.TableName, tableName, StringComparison.Ordinal));
+        if (tableIndex < 0)
+        {
+            throw new InvalidOperationException($"Table '{tableName}' not found.");
+        }
+
+        // Shift subsequent tables (data + offsets) backwards from end to avoid overwrite.
+        for (int i = ordered.Length - 1; i > tableIndex; i--)
+        {
+            var t = ordered[i];
+            if (t.RecordCount <= 0)
+            {
+                t.DataStartOffset += growByBytes;
+                continue;
+            }
+
+            long srcStart = t.DataStartOffset;
+            long bytesToMove = (long)t.RecordCount * t.RecordSize;
+            long srcEndExclusive = srcStart + bytesToMove;
+            long destStart = srcStart + growByBytes;
+
+            await MoveRegionAsync(file, srcStart, srcEndExclusive, destStart, cancellationToken);
+            t.DataStartOffset = destStart;
+        }
+
+        tableMetadata.ReservedRecordCount += growByRecords;
+
+        foreach (var t in ordered)
+        {
+            await UpdateTableMetadataAsync(t.TableName, file, cancellationToken);
+        }
+    }
+
+    private static async Task MoveRegionAsync(FileStream file, long srcStart, long srcEndExclusive, long destStart, CancellationToken cancellationToken)
+    {
+        const int bufferSize = 64 * 1024;
+        byte[] buffer = ArrayPool<byte>.Shared.Rent(bufferSize);
+        try
+        {
+            long remaining = srcEndExclusive - srcStart;
+            while (remaining > 0)
+            {
+                int toRead = (int)Math.Min(buffer.Length, remaining);
+                long readPos = srcStart + (remaining - toRead);
+                long writePos = destStart + (remaining - toRead);
+
+                file.Seek(readPos, SeekOrigin.Begin);
+                int read = await file.ReadAsync(buffer.AsMemory(0, toRead), cancellationToken);
+                if (read != toRead)
+                {
+                    throw new EndOfStreamException($"Unexpected EOF while moving data region. Expected {toRead} bytes, got {read}.");
+                }
+
+                file.Seek(writePos, SeekOrigin.Begin);
+                await file.WriteAsync(buffer.AsMemory(0, toRead), cancellationToken);
+                remaining -= toRead;
+            }
+        }
+        finally
+        {
+            ArrayPool<byte>.Shared.Return(buffer);
+        }
+    }
+
     private async Task UpdateTableMetadataAsync(string tableName, FileStream file, CancellationToken cancellationToken = default)
     {
         var tableMetadata = _tables[tableName];

tests/Perigon.MiniDb.Tests/ComplexDataAccessTests.cs

@@ -205,6 +205,134 @@ public async ValueTask DisposeAsync()
         }
     }
 
+    [Fact]
+    public async Task WhenGrowingTablesThenDataRemainsConsistentAcrossInsertUpdateDeleteAndReload()
+    {
+        // Cycle 1: Insert enough records to exceed initial reserved capacity (10) for multiple tables
+        var db1 = new ComplexDbContext();
+        await using (db1)
+        {
+            for (int i = 0; i < 25; i++)
+            {
+                db1.Solutions.Add(new Solution
+                {
+                    Name = $"Solution-{i}",
+                    DisplayName = $"Solution Display {i}",
+                    Path = $"C:\\Solutions\\Solution-{i}\\Deep\\Path",
+                    Version = "1.0.0",
+                    SolutionType = (SolutionType)((i % 4) + 1),
+                    ConfigJsonString = JsonSerializer.Serialize(new { index = i, value = new string('X', 100) })
+                });
+
+                db1.Projects.Add(new Project
+                {
+                    ProjectName = $"Project-{i}",
+                    ProjectPath = $"C:\\Projects\\Project-{i}",
+                    FrameworkVersion = "net10.0",
+                    ProjectType = (ProjectType)((i % 4) + 1),
+                    Status = ProjectStatus.Active,
+                    CreatedAt = DateTime.UtcNow,
+                    SolutionId = (i % 25) + 1
+                });
+
+                db1.Configurations.Add(new AppConfiguration
+                {
+                    ConfigKey = $"Key-{i}",
+                    ConfigValue = new string('V', 100),
+                    ConfigType = (ConfigType)((i % 4) + 1),
+                    IsEncrypted = i % 2 == 0,
+                    UpdatedAt = DateTime.UtcNow
+                });
+
+                db1.ApiDocumentations.Add(new ApiDocumentation
+                {
+                    ApiName = $"Api-{i}",
+                    Endpoint = $"/api/{i}",
+                    MethodType = (ApiMethodType)((i % 5) + 1),
+                    JsonSchema = JsonSerializer.Serialize(new { type = "object", i }),
+                    IsPublished = true,
+                    CreatedAt = DateTime.UtcNow,
+                    ProjectId = (i % 25) + 1
+                });
+            }
+
+            await db1.SaveChangesAsync();
+            Assert.Equal(25, db1.Solutions.Count);
+            Assert.Equal(25, db1.Projects.Count);
+            Assert.Equal(25, db1.Configurations.Count);
+            Assert.Equal(25, db1.ApiDocumentations.Count);
+        }
+
+        await ComplexDbContext.ReleaseSharedCacheAsync(_testDbPath);
+        await Task.Delay(10);
+
+        // Cycle 2: Update some records and delete some records
+        var db2 = new ComplexDbContext();
+        await using (db2)
+        {
+            // Update first 5 in each table
+            foreach (var s in db2.Solutions.OrderBy(s => s.Id).Take(5))
+            {
+                s.DisplayName = $"Updated-{s.DisplayName}";
+                db2.Solutions.Update(s);
+            }
+
+            foreach (var p in db2.Projects.OrderBy(p => p.Id).Take(5))
+            {
+                p.Status = ProjectStatus.Archived;
+                db2.Projects.Update(p);
+            }
+
+            foreach (var c in db2.Configurations.OrderBy(c => c.Id).Take(5))
+            {
+                c.ConfigValue = "Updated";
+                db2.Configurations.Update(c);
+            }
+
+            foreach (var a in db2.ApiDocumentations.OrderBy(a => a.Id).Take(5))
+            {
+                a.IsPublished = false;
+                db2.ApiDocumentations.Update(a);
+            }
+
+            // Delete last 7 in each table
+            foreach (var s in db2.Solutions.OrderByDescending(s => s.Id).Take(7).ToArray())
+                db2.Solutions.Remove(s);
+            foreach (var p in db2.Projects.OrderByDescending(p => p.Id).Take(7).ToArray())
+                db2.Projects.Remove(p);
+            foreach (var c in db2.Configurations.OrderByDescending(c => c.Id).Take(7).ToArray())
+                db2.Configurations.Remove(c);
+            foreach (var a in db2.ApiDocumentations.OrderByDescending(a => a.Id).Take(7).ToArray())
+                db2.ApiDocumentations.Remove(a);
+
+            await db2.SaveChangesAsync();
+        }
+
+        await ComplexDbContext.ReleaseSharedCacheAsync(_testDbPath);
+        await Task.Delay(10);
+
+        // Cycle 3: Reload and verify counts and Id uniqueness are correct
+        var db3 = new ComplexDbContext();
+        await using (db3)
+        {
+            Assert.Equal(18, db3.Solutions.Count);
+            Assert.Equal(18, db3.Projects.Count);
+            Assert.Equal(18, db3.Configurations.Count);
+            Assert.Equal(18, db3.ApiDocumentations.Count);
+
+            Assert.Equal(db3.Solutions.Count, db3.Solutions.Select(x => x.Id).Distinct().Count());
+            Assert.Equal(db3.Projects.Count, db3.Projects.Select(x => x.Id).Distinct().Count());
+            Assert.Equal(db3.Configurations.Count, db3.Configurations.Select(x => x.Id).Distinct().Count());
+            Assert.Equal(db3.ApiDocumentations.Count, db3.ApiDocumentations.Select(x => x.Id).Distinct().Count());
+
+            // Spot-check that updates persisted
+            Assert.All(db3.Solutions.OrderBy(s => s.Id).Take(5), s => Assert.StartsWith("Updated-", s.DisplayName));
+            Assert.All(db3.Projects.OrderBy(p => p.Id).Take(5), p => Assert.Equal(ProjectStatus.Archived, p.Status));
+            Assert.All(db3.Configurations.OrderBy(c => c.Id).Take(5), c => Assert.Equal("Updated", c.ConfigValue));
+            Assert.All(db3.ApiDocumentations.OrderBy(a => a.Id).Take(5), a => Assert.False(a.IsPublished));
+        }
+    }
+
     [Fact]
     public async Task CanAddAndRetrieveSingleSolutionWithEnum()
     {