Skip to content

Commit

Permalink
Backport fixes to make newer lib tool code work correctly
Browse files Browse the repository at this point in the history
- Replaces a few function calls with their old names
- Removes some timeseries descriptor bindings exposed
  • Loading branch information
IvoDD committed Oct 29, 2024
1 parent 86181f2 commit cc6fb57
Show file tree
Hide file tree
Showing 6 changed files with 13 additions and 12 deletions.
10 changes: 3 additions & 7 deletions cpp/arcticdb/stream/python_bindings.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -101,16 +101,12 @@ void register_types(py::module &m) {
.def_property_readonly("fields", [](const TimeseriesDescriptor& desc){
return field_collection_to_ref_vector(desc.fields());
}).def_property_readonly("normalization", [](const TimeseriesDescriptor& self) {
return python_util::pb_to_python(self.normalization());
}).def_property_readonly("sorted", [](const TimeseriesDescriptor& self) {
return self.sorted();
}).def_property_readonly("index", [](const TimeseriesDescriptor& self) {
return self.index();
return python_util::pb_to_python(self.proto().normalization());
}).def_property_readonly("total_rows", [](const TimeseriesDescriptor& self) {
return self.total_rows();
return self.proto().total_rows();
}).def_property_readonly("next_key", [](const TimeseriesDescriptor& self) -> std::optional<AtomKey> {
if (self.proto().has_next_key()){
return key_from_proto(self.proto().next_key());
return decode_key(self.proto().next_key());
}
return std::nullopt;
});
Expand Down
7 changes: 4 additions & 3 deletions cpp/arcticdb/toolbox/library_tool.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
#include <arcticdb/codec/default_codecs.hpp>
#include <arcticdb/entity/atom_key.hpp>
#include <arcticdb/entity/protobufs.hpp>
#include <arcticdb/entity/protobuf_mappings.hpp>
#include <arcticdb/entity/types.hpp>
#include <arcticdb/pipeline/pipeline_utils.hpp>
#include <arcticdb/storage/library.hpp>
Expand Down Expand Up @@ -45,7 +46,7 @@ ReadResult LibraryTool::read(const VariantKey& key) {
// We construct a dummy atom key in case of a RefKey to be able to build the read_result
[](const RefKey& key){return AtomKeyBuilder().build<KeyType::VERSION_REF>(key.id());},
[](const auto&){});
return pipelines::read_result_from_single_frame(frame_and_descriptor, atom_key);
return pipelines::make_read_result_from_frame(frame_and_descriptor, atom_key);
}

Segment LibraryTool::read_to_segment(const VariantKey& key) {
Expand All @@ -70,7 +71,7 @@ TimeseriesDescriptor LibraryTool::read_timeseries_descriptor(const VariantKey& k

void LibraryTool::write(VariantKey key, Segment segment) {
storage::KeySegmentPair kv{std::move(key), std::move(segment)};
store()->write_compressed_sync(kv);
store()->write_compressed_sync(std::move(kv));
}

void LibraryTool::overwrite_segment_in_memory(VariantKey key, SegmentInMemory& segment_in_memory) {
Expand All @@ -92,7 +93,7 @@ SegmentInMemory LibraryTool::overwrite_append_data(
const auto& tsd = old_segment_in_memory.index_descriptor();
std::optional<AtomKey> next_key = std::nullopt;
if (tsd.proto().has_next_key()){
next_key = key_from_proto(tsd.proto().next_key());
next_key = decode_key(tsd.proto().next_key());
}

auto stream_id = util::variant_match(key, [](const auto& key){return key.id();});
Expand Down
2 changes: 2 additions & 0 deletions cpp/arcticdb/toolbox/library_tool.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@ class LibraryTool {

std::vector<VariantKey> find_keys(arcticdb::entity::KeyType);

bool key_exists(const VariantKey& key);

std::vector<bool> batch_key_exists(const std::vector<VariantKey>& keys);

std::string get_key_path(const VariantKey& key);
Expand Down
1 change: 1 addition & 0 deletions cpp/arcticdb/toolbox/python_bindings.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ void register_bindings(py::module &m) {
}))
.def("read_to_segment", &LibraryTool::read_to_segment)
.def("read_metadata", &LibraryTool::read_metadata)
.def("key_exists", &LibraryTool::key_exists)
.def("read_descriptor", &LibraryTool::read_descriptor, R"pbdoc(
Gives the <StreamDescriptor> for a Variant key. The Stream Descriptor contains the <FieldRef>s for all fields in
the value written for that key.
Expand Down
3 changes: 2 additions & 1 deletion cpp/arcticdb/version/local_versioned_engine.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -406,6 +406,8 @@ class LocalVersionedEngine : public VersionedEngine {
return store()->current_timestamp();
}

const arcticdb::proto::storage::VersionStoreConfig& cfg() const override { return cfg_; }

protected:
VersionedItem compact_incomplete_dynamic(
const StreamId& stream_id,
Expand All @@ -429,7 +431,6 @@ class LocalVersionedEngine : public VersionedEngine {
);

std::shared_ptr<Store>& store() override { return store_; }
const arcticdb::proto::storage::VersionStoreConfig& cfg() const override { return cfg_; }
std::shared_ptr<VersionMap>& version_map() override { return version_map_; }
SymbolList& symbol_list() override { return *symbol_list_; }
std::shared_ptr<SymbolList> symbol_list_ptr() { return symbol_list_; }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ def test_type_promotion_stored_in_index_key(lmdb_version_store_dynamic_schema):
def get_type_of_column():
index_key = lib_tool.find_keys_for_symbol(KeyType.TABLE_INDEX, sym)[-1]
tsd = lib_tool.read_timeseries_descriptor(index_key)
type_desc = [field.type() for field in tsd.fields() if field.name() == col][0]
type_desc = [field.type() for field in tsd.fields if field.name() == col][0]
return type_desc.data_type()

df_write = pd.DataFrame({col: [1, 2]}, dtype="int8", index=pd.date_range("2024-01-01", periods=2))
Expand Down

0 comments on commit cc6fb57

Please sign in to comment.