Skip to content

Commit 79c261f

Browse files
author
Dmitry Razdoburdin
committed
initial
1 parent bf7a3f8 commit 79c261f

17 files changed

Lines changed: 263 additions & 421 deletions

File tree

‎include/svs/core/data/simple.h‎

Lines changed: 4 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -136,12 +136,8 @@ class GenericSerializer {
136136
}
137137

138138
template <typename T, lib::LazyInvocable<size_t, size_t> F>
139-
static lib::lazy_result_t<F, size_t, size_t> load(
140-
const lib::ContextFreeLoadTable& table,
141-
const lib::detail::Deserializer& deserializer,
142-
std::istream& is,
143-
const F& lazy
144-
) {
139+
static lib::lazy_result_t<F, size_t, size_t>
140+
load(const lib::ContextFreeLoadTable& table, std::istream& is, const F& lazy) {
145141
auto datatype = lib::load_at<DataType>(table, "eltype");
146142
if (datatype != datatype_v<T>) {
147143
throw ANNEXCEPTION(
@@ -155,10 +151,6 @@ class GenericSerializer {
155151
size_t num_vectors = lib::load_at<size_t>(table, "num_vectors");
156152
size_t dims = lib::load_at<size_t>(table, "dims");
157153

158-
deserializer.read_name(is);
159-
deserializer.read_size(is);
160-
deserializer.read_binary<io::v1::Header>(is);
161-
162154
return io::load_dataset(is, lazy, num_vectors, dims);
163155
}
164156
};
@@ -482,14 +474,13 @@ class SimpleData {
482474

483475
static SimpleData load(
484476
const lib::ContextFreeLoadTable& table,
485-
const lib::detail::Deserializer& deserializer,
486477
std::istream& is,
487478
const allocator_type& allocator = {}
488479
)
489480
requires(!is_view)
490481
{
491482
return GenericSerializer::load<T>(
492-
table, deserializer, is, lib::Lazy([&](size_t n_elements, size_t n_dimensions) {
483+
table, is, lib::Lazy([&](size_t n_elements, size_t n_dimensions) {
493484
return SimpleData(n_elements, n_dimensions, allocator);
494485
})
495486
);
@@ -888,15 +879,11 @@ class SimpleData<T, Extent, Blocked<Alloc>> {
888879

889880
static SimpleData load(
890881
const lib::ContextFreeLoadTable& table,
891-
const lib::detail::Deserializer& deserializer,
892882
std::istream& is,
893883
const Blocked<Alloc>& allocator = {}
894884
) {
895885
return GenericSerializer::load<T>(
896-
table,
897-
deserializer,
898-
is,
899-
lib::Lazy([&allocator](size_t n_elements, size_t n_dimensions) {
886+
table, is, lib::Lazy([&allocator](size_t n_elements, size_t n_dimensions) {
900887
return SimpleData(n_elements, n_dimensions, allocator);
901888
})
902889
);

‎include/svs/core/graph/graph.h‎

Lines changed: 11 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -335,7 +335,6 @@ template <std::unsigned_integral Idx, data::MemoryDataset Data> class SimpleGrap
335335
static lib::lazy_result_t<F, data_type> load(
336336
const lib::ContextFreeLoadTable& table,
337337
const F& lazy,
338-
const lib::detail::Deserializer& deserializer,
339338
std::istream& is,
340339
AllocArgs&&... alloc_args
341340
) {
@@ -363,9 +362,9 @@ template <std::unsigned_integral Idx, data::MemoryDataset Data> class SimpleGrap
363362
{"dims", lib::save(max_degree + 1)},
364363
};
365364

366-
return lazy(data_type::load(
367-
lib::ContextFreeLoadTable(data_table), deserializer, is, alloc_args...
368-
));
365+
return lazy(
366+
data_type::load(lib::ContextFreeLoadTable(data_table), is, alloc_args...)
367+
);
369368
}
370369

371370
protected:
@@ -419,12 +418,11 @@ class SimpleGraph : public SimpleGraphBase<Idx, data::SimpleData<Idx, Dynamic, A
419418

420419
static constexpr SimpleGraph load(
421420
const lib::ContextFreeLoadTable& table,
422-
const lib::detail::Deserializer& deserializer,
423421
std::istream& is,
424422
const Alloc& allocator = {}
425423
) {
426424
auto lazy = lib::Lazy([](data_type data) { return SimpleGraph(std::move(data)); });
427-
return parent_type::load(table, lazy, deserializer, is, allocator);
425+
return parent_type::load(table, lazy, is, allocator);
428426
}
429427

430428
static constexpr SimpleGraph
@@ -436,12 +434,8 @@ class SimpleGraph : public SimpleGraphBase<Idx, data::SimpleData<Idx, Dynamic, A
436434
}
437435
}
438436

439-
static constexpr SimpleGraph load(
440-
const lib::detail::Deserializer& deserializer,
441-
std::istream& is,
442-
const Alloc& allocator = {}
443-
) {
444-
return lib::load_from_stream<SimpleGraph>(deserializer, is, allocator);
437+
static constexpr SimpleGraph load(std::istream& is, const Alloc& allocator = {}) {
438+
return lib::load_from_stream<SimpleGraph>(is, allocator);
445439
}
446440
};
447441

@@ -475,14 +469,11 @@ class SimpleBlockedGraph
475469
return parent_type::load(table, lazy);
476470
}
477471

478-
static constexpr SimpleBlockedGraph load(
479-
const lib::ContextFreeLoadTable& table,
480-
const lib::detail::Deserializer& deserializer,
481-
std::istream& is
482-
) {
472+
static constexpr SimpleBlockedGraph
473+
load(const lib::ContextFreeLoadTable& table, std::istream& is) {
483474
auto lazy =
484475
lib::Lazy([](data_type data) { return SimpleBlockedGraph(std::move(data)); });
485-
return parent_type::load(table, lazy, deserializer, is);
476+
return parent_type::load(table, lazy, is);
486477
}
487478

488479
static constexpr SimpleBlockedGraph load(const std::filesystem::path& path) {
@@ -493,9 +484,8 @@ class SimpleBlockedGraph
493484
}
494485
}
495486

496-
static constexpr SimpleBlockedGraph
497-
load(const lib::detail::Deserializer& deserializer, std::istream& is) {
498-
return lib::load_from_stream<SimpleBlockedGraph>(deserializer, is);
487+
static constexpr SimpleBlockedGraph load(std::istream& is) {
488+
return lib::load_from_stream<SimpleBlockedGraph>(is);
499489
}
500490
};
501491

‎include/svs/core/translation.h‎

Lines changed: 1 addition & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -369,6 +369,7 @@ class IDTranslator {
369369
}
370370

371371
static IDTranslator load(const lib::ContextFreeLoadTable& table, std::istream& is) {
372+
IDTranslator::validate(table);
372373
auto num_points = lib::load_at<size_t>(table, "num_points");
373374

374375
auto translator = IDTranslator{};
@@ -380,18 +381,6 @@ class IDTranslator {
380381
return translator;
381382
}
382383

383-
static IDTranslator load(
384-
const lib::ContextFreeLoadTable& table,
385-
const lib::detail::Deserializer& deserializer,
386-
std::istream& is
387-
) {
388-
IDTranslator::validate(table);
389-
deserializer.read_name(is);
390-
deserializer.read_size(is);
391-
392-
return IDTranslator::load(table, is);
393-
}
394-
395384
static IDTranslator load(const lib::LoadTable& table) {
396385
IDTranslator::validate(table);
397386

‎include/svs/index/flat/dynamic_flat.h‎

Lines changed: 10 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -790,7 +790,6 @@ auto auto_dynamic_assemble(
790790

791791
template <typename LazyDataLoader, typename Distance, typename ThreadPoolProto>
792792
auto auto_dynamic_assemble(
793-
const lib::detail::Deserializer& deserializer,
794793
std::istream& is,
795794
LazyDataLoader&& data_loader,
796795
Distance distance,
@@ -803,40 +802,16 @@ auto auto_dynamic_assemble(
803802
bool SVS_UNUSED(debug_load_from_static) = false,
804803
svs::logging::logger_ptr logger = svs::logging::get()
805804
) {
806-
using Data = decltype(data_loader());
807-
auto config_loader = [&] {
808-
auto table = lib::detail::read_metadata(deserializer, is);
809-
auto translation = table.template cast<toml::table>()
810-
.at("translation")
811-
.template cast<toml::table>();
812-
return IDTranslator::load(translation, deserializer, is);
813-
};
814-
815-
std::optional<IDTranslator> config;
816-
std::optional<Data> data;
817-
818-
if (deserializer.is_native()) {
819-
// Order is always config->data.
820-
config.emplace(config_loader());
821-
data.emplace(data_loader());
822-
} else {
823-
// Directory packing order is filesystem-dependent.
824-
// Read 2 data blocks: config and data in a corresponding order.
825-
for (int data_block_idx = 0; data_block_idx < 2; ++data_block_idx) {
826-
auto name = deserializer.read_name_in_advance(is);
827-
if (name.starts_with("config/")) {
828-
config.emplace(config_loader());
829-
} else if (name.starts_with("data/")) {
830-
data.emplace(data_loader());
831-
} else {
832-
throw ANNEXCEPTION("The stream is corrupted!");
833-
}
834-
}
835-
}
805+
auto table = lib::detail::read_metadata(is);
806+
auto translation =
807+
table.template cast<toml::table>().at("translation").template cast<toml::table>();
808+
IDTranslator translator = IDTranslator::load(translation, is);
809+
810+
auto data = data_loader();
836811

837812
// Validate the translator
838-
auto translator_size = config->size();
839-
auto datasize = data->size();
813+
auto translator_size = translator.size();
814+
auto datasize = data.size();
840815
if (translator_size != datasize) {
841816
throw ANNEXCEPTION(
842817
"Translator has {} IDs but should have {}", translator_size, datasize
@@ -845,8 +820,8 @@ auto auto_dynamic_assemble(
845820

846821
auto threadpool = threads::as_threadpool(std::move(threadpool_proto));
847822
return DynamicFlatIndex(
848-
std::move(*data),
849-
std::move(*config),
823+
std::move(data),
824+
std::move(translator),
850825
std::move(distance),
851826
std::move(threadpool),
852827
std::move(logger)

‎include/svs/index/vamana/dynamic_index.h‎

Lines changed: 19 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -1462,7 +1462,6 @@ template <
14621462
typename Distance,
14631463
typename ThreadPoolProto>
14641464
auto auto_dynamic_assemble(
1465-
const lib::detail::Deserializer& deserializer,
14661465
std::istream& is,
14671466
LazyGraphLoader graph_loader,
14681467
LazyDataLoader data_loader,
@@ -1471,62 +1470,31 @@ auto auto_dynamic_assemble(
14711470
bool SVS_UNUSED(debug_load_from_static) = false,
14721471
svs::logging::logger_ptr logger = svs::logging::get()
14731472
) {
1474-
using Data = decltype(data_loader());
1475-
using Graph = decltype(graph_loader());
1476-
1477-
// The config loader reads the combined TOML (parameters + translation)
1473+
// Read the combined TOML (parameters + translation)
14781474
// and the translator binary data.
1479-
auto config_loader = [&]() -> detail::VamanaStateLoader {
1480-
auto table = lib::detail::read_metadata(deserializer, is);
1475+
auto table = lib::detail::read_metadata(is);
14811476

1482-
auto parameters = lib::load<VamanaIndexParameters>(
1483-
table.template cast<toml::table>().at("parameters").template cast<toml::table>()
1484-
);
1477+
auto parameters = lib::load<VamanaIndexParameters>(
1478+
table.template cast<toml::table>().at("parameters").template cast<toml::table>()
1479+
);
14851480

1486-
auto translation = table.template cast<toml::table>()
1487-
.at("translation")
1488-
.template cast<toml::table>();
1489-
1490-
auto translator = IDTranslator::load(translation, deserializer, is);
1491-
1492-
return detail::VamanaStateLoader{std::move(parameters), std::move(translator)};
1493-
};
1494-
1495-
std::optional<detail::VamanaStateLoader> config;
1496-
std::optional<Data> data;
1497-
std::optional<Graph> graph;
1498-
1499-
if (deserializer.is_native()) {
1500-
// Order is always config->data->graph.
1501-
config.emplace(config_loader());
1502-
data.emplace(data_loader());
1503-
graph.emplace(graph_loader());
1504-
} else {
1505-
// Directory packing order is filesystem-dependent.
1506-
// Read 3 data blocks: config, data and graph in a corresponding order.
1507-
for (int data_block_idx = 0; data_block_idx < 3; ++data_block_idx) {
1508-
auto name = deserializer.read_name_in_advance(is);
1509-
if (name.starts_with("config/")) {
1510-
config.emplace(config_loader());
1511-
} else if (name.starts_with("data/")) {
1512-
data.emplace(data_loader());
1513-
} else if (name.starts_with("graph/")) {
1514-
graph.emplace(graph_loader());
1515-
} else {
1516-
throw ANNEXCEPTION("The stream is corrupted!");
1517-
}
1518-
}
1519-
}
1481+
auto translation =
1482+
table.template cast<toml::table>().at("translation").template cast<toml::table>();
1483+
1484+
auto translator = IDTranslator::load(translation, is);
1485+
1486+
auto data = data_loader();
1487+
auto graph = graph_loader();
15201488

1521-
auto datasize = data->size();
1522-
auto graphsize = graph->n_nodes();
1489+
auto datasize = data.size();
1490+
auto graphsize = graph.n_nodes();
15231491
if (datasize != graphsize) {
15241492
throw ANNEXCEPTION(
15251493
"Reloaded data has {} nodes while the graph has {} nodes!", datasize, graphsize
15261494
);
15271495
}
15281496

1529-
auto translator_size = config->translator_.size();
1497+
auto translator_size = translator.size();
15301498
if (translator_size != datasize) {
15311499
throw ANNEXCEPTION(
15321500
"Translator has {} IDs but should have {}", translator_size, datasize
@@ -1535,11 +1503,11 @@ auto auto_dynamic_assemble(
15351503

15361504
auto threadpool = threads::as_threadpool(std::move(threadpool_proto));
15371505
return MutableVamanaIndex{
1538-
config->parameters_,
1539-
std::move(*data),
1540-
std::move(*graph),
1506+
parameters,
1507+
std::move(data),
1508+
std::move(graph),
15411509
std::move(distance),
1542-
std::move(config->translator_),
1510+
std::move(translator),
15431511
std::move(threadpool),
15441512
std::move(logger)};
15451513
}

0 commit comments

Comments
 (0)