[MMAP] Add in-memory (mmap) stream data loading support and related tests

- Introduced `memstream.h` and `memstream.cpp` for memory-mapped stream functionality.
- Updated `io.h` and `simple.h` to include memory stream support.
- Enhanced `simple.cpp` and `flat.cpp` tests to validate loading from memory streams.

Author: rfsaliev

include/svs/core/data/io.h

@@ -19,6 +19,7 @@
 // svs
 #include "svs/concepts/data.h"
 #include "svs/core/io.h"
+#include "svs/core/io/memstream.h"
 
 #include "svs/lib/array.h"
 #include "svs/lib/exception.h"
@@ -113,6 +114,34 @@ void populate(Data& data, WriteAccessor& accessor, const File& file) {
     populate_impl(data, accessor, file, default_populate_tag);
 }
 
+template <typename T> [[nodiscard]] size_t streaming_size(const T& obj) noexcept;
+
+template <HasDataType T> [[nodiscard]] size_t streaming_size(const T& obj) noexcept {
+    return element_size(datatype_v<T>);
+}
+
+template <typename T, typename Alloc = ::std::allocator<T>>
+[[nodiscard]] size_t streaming_size(const ::std::vector<T, Alloc>& vec) noexcept {
+    return vec.size() * streaming_size(T{});
+}
+
+template <typename T, size_t N>
+[[nodiscard]] size_t streaming_size(const ::std::array<T, N>& arr) noexcept {
+    return arr.size() * streaming_size(T{});
+}
+
+template <typename... Ts>
+[[nodiscard]] size_t streaming_size(const ::std::tuple<Ts...>& tup) noexcept {
+    size_t total = 0;
+    lib::foreach (tup, [&](const auto& x) { total += streaming_size(x); });
+    return total;
+}
+
+template <typename T, typename Dims, typename Alloc = lib::Allocator<T>>
+[[nodiscard]] size_t streaming_size(const svs::DenseArray<T, Dims, Alloc>& arr) noexcept {
+    return arr.size() * streaming_size(T{});
+}
+
 /////
 ///// Saving
 /////
@@ -202,6 +231,25 @@ load_dataset(std::istream& is, const F& lazy, size_t num_vectors, size_t dims) {
     return data;
 }
 
+template <typename T, lib::LazyInvocable<size_t, size_t, T*> F>
+lib::lazy_result_t<F, size_t, size_t, T*>
+load_dataset(std::istream& is, const F& lazy, size_t num_vectors, size_t dims) {
+    // If the allocator is vew, we should support in_memory_stream only since views are only
+    // compatible with memory-mapped data.
+    if (!is_memory_stream(is)) {
+        throw ANNEXCEPTION(
+            "Trying to load a dataset with a view allocator from a non-memory stream. This "
+            "is not supported since views are compatible only with memory-mapped streams."
+        );
+    }
+    // If the allocator is a view, we need to construct the view with the correct pointer.
+    auto data = lazy(num_vectors, dims, io::current_ptr<T>(is));
+    // Then we stream have to be seeked past the data we just loaded since the view will not
+    // take ownership of the stream's position.
+    is.seekg(streaming_size(data), std::ios_base::cur);
+    return data;
+}
+
 // Return whether or not a file is directly loadable via file-extension.
 inline bool special_by_file_extension(std::string_view path) {
     return (path.ends_with("svs") || path.ends_with("vecs") || path.ends_with("bin"));

include/svs/core/data/simple.h

@@ -21,6 +21,7 @@
 #include "svs/core/allocator.h"
 #include "svs/core/compact.h"
 #include "svs/core/data/io.h"
+#include "svs/core/io/memstream.h"
 
 #include "svs/lib/array.h"
 #include "svs/lib/boundscheck.h"
@@ -153,6 +154,25 @@ class GenericSerializer {
 
         return io::load_dataset(is, lazy, num_vectors, dims);
     }
+
+    template <typename T, lib::LazyInvocable<size_t, size_t, T*> F>
+    static lib::lazy_result_t<F, size_t, size_t, T*>
+    load(const lib::ContextFreeLoadTable& table, std::istream& is, const F& lazy) {
+        auto datatype = lib::load_at<DataType>(table, "eltype");
+        if (datatype != datatype_v<T>) {
+            throw ANNEXCEPTION(
+                "Trying to load an uncompressed dataset with element types {} to a dataset "
+                "with element types {}.",
+                name(datatype),
+                name<datatype_v<T>>()
+            );
+        }
+
+        size_t num_vectors = lib::load_at<size_t>(table, "num_vectors");
+        size_t dims = lib::load_at<size_t>(table, "dims");
+
+        return io::load_dataset<T>(is, lazy, num_vectors, dims);
+    }
 };
 
 struct Matcher {
@@ -486,6 +506,31 @@ class SimpleData {
         );
     }
 
+    static SimpleData load(const lib::LoadTable& SVS_UNUSED(table))
+        requires(is_view)
+    {
+        throw ANNEXCEPTION("Trying to load a SimpleData view without an istream. This is "
+                           "not supported since "
+                           "views are compatible only with memory-mapped streams.");
+    }
+
+    static SimpleData load(const lib::ContextFreeLoadTable& table, std::istream& is)
+        requires(is_view)
+    {
+        if (!io::is_memory_stream(is)) {
+            throw ANNEXCEPTION(
+                "Trying to load a SimpleData view from a non-mmstream istream. This is not "
+                "supported since views are compatible only with memory-mapped streams."
+            );
+        }
+
+        return GenericSerializer::load<T>(
+            table, is, lib::Lazy([](size_t n_elements, size_t n_dimensions, T* ptr) {
+                return SimpleData(n_elements, n_dimensions, View<T>{ptr});
+            })
+        );
+    }
+
     ///
     /// @brief Try to automatically load the dataset.
     ///
@@ -598,6 +643,11 @@ bool operator==(const SimpleData<T1, E1, A1>& x, const SimpleData<T2, E2, A2>& y
     return true;
 }
 
+template <typename T, size_t Extent = Dynamic, typename Alloc = lib::Allocator<T>>
+size_t streaming_size(const svs::data::SimpleData<T, Extent, Alloc>& data) noexcept {
+    return data.capacity() * data.element_size();
+}
+
 /////
 ///// Specialization for Blocked.
 /////