Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

rewrite some Tsfile C++ interface #304

Merged
merged 28 commits into from
Dec 18, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
5372068
rewrite some Tsfile C++ interface
zwhzzz0821 Nov 18, 2024
62b0a19
fix format
zwhzzz0821 Nov 18, 2024
36ae884
fix the problem in test/CMakeLists
zwhzzz0821 Nov 19, 2024
e30d562
add algorith and memory in tablet.h
zwhzzz0821 Nov 19, 2024
9d55a19
add some header file
zwhzzz0821 Nov 19, 2024
fe2a217
fix memory leak
zwhzzz0821 Nov 19, 2024
0d2ad42
fix for CWrapper and timeFilte
zwhzzz0821 Nov 26, 2024
243eb7a
Merge remote-tracking branch 'upstream/develop' into cppTsfileRewrite
zwhzzz0821 Nov 26, 2024
b10f548
fix for ut
zwhzzz0821 Nov 28, 2024
bd55cda
Add parser for path and fix for get device name
zwhzzz0821 Dec 10, 2024
2ee042e
fix complie error for ubuntu and windows
zwhzzz0821 Dec 10, 2024
49c368c
add path test add fix complie error
zwhzzz0821 Dec 10, 2024
0245a2d
Remove redundant tests
zwhzzz0821 Dec 10, 2024
57a4834
fix for windows
zwhzzz0821 Dec 11, 2024
ddad7f9
fix bug for complie when using mingw and change name parser/generated
zwhzzz0821 Dec 12, 2024
ed0700a
fix include name
zwhzzz0821 Dec 12, 2024
42c185b
add download uuid in linux autoly
zwhzzz0821 Dec 13, 2024
36f862a
fix warning
zwhzzz0821 Dec 13, 2024
216f170
fix bug in ubuntu
zwhzzz0821 Dec 13, 2024
b35551c
try to fix link bug in windows
zwhzzz0821 Dec 13, 2024
e283d68
try to fix bug in windows
zwhzzz0821 Dec 13, 2024
188a86f
output debug detail
zwhzzz0821 Dec 14, 2024
879f41e
Removed unnecessary include
zwhzzz0821 Dec 15, 2024
7171512
link static lib instead of shared lib
zwhzzz0821 Dec 15, 2024
0cd45c6
remove static crt
zwhzzz0821 Dec 15, 2024
1cdf7c2
debug detail
zwhzzz0821 Dec 16, 2024
f74258e
fix for bug in windows
zwhzzz0821 Dec 17, 2024
7e27a23
Add compilation details on windows in Readme
zwhzzz0821 Dec 17, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
3 changes: 2 additions & 1 deletion cpp/CMakeLists.txt
100644 → 100755
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ project(TsFile_CPP)

cmake_policy(SET CMP0079 NEW)
set(TsFile_CPP_VERSION 1.2.0.dev)

set(CMAKE_CXX_FLAGS "$ENV{CXXFLAGS} -Wall -Werror")
message("cmake using: USE_CPP11=${USE_CPP11}")

Expand Down Expand Up @@ -76,6 +75,8 @@ set(PROJECT_INCLUDE_DIR ${PROJECT_INCLUDE_DIR}
set(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/bin)
include_directories(${PROJECT_INCLUDE_DIR})

include_directories(${PROJECT_SOURCE_DIR}/third_party/antlr4-cpp-runtime-4/runtime/src)

add_subdirectory(third_party)

add_subdirectory(src)
Expand Down
7 changes: 7 additions & 0 deletions cpp/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,13 @@ To build tsfile, you can run: `bash build.sh`. If you have Maven tools, you can

Before you submit your code to GitHub, please ensure that the `mvn` compilation is correct.

If you compile using MinGW on windows and encounter an error, you can try replacing MinGW with the following version that we have tried without problems:

* GCC 14.2.0 (with **POSIX** threads) + LLVM/Clang/LLD/LLDB 18.1.8 + MinGW-w64 12.0.0 UCRT - release 1
* GCC 12.2.0 + LLVM/Clang/LLD/LLDB 16.0.0 + MinGW-w64 10.0.0 (UCRT) - release 5
* GCC 12.2.0 + LLVM/Clang/LLD/LLDB 16.0.0 + MinGW-w64 10.0.0 (MSVCRT) - release 5
* GCC 11.2.0 + MinGW-w64 10.0.0 (MSVCRT) - release 1

## Use TsFile

You can find examples on how to read and write data in `demo_read.cpp` and `demo_write.cpp` located under `./examples/cpp_examples`. There are also examples under `./examples/c_examples`on how to use a C-style API to read and write data in a C environment. You can run `bash build.sh` under `./examples` to generate an executable output under `./examples/build`.
6 changes: 3 additions & 3 deletions cpp/examples/cpp_examples/demo_read.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -70,16 +70,16 @@ int demo_read() {

std::cout << "begin to query expr" << std::endl;
ASSERT(ret == 0);
storage::QueryDataSet *qds = nullptr;
storage::ResultSet *qds = nullptr;
ret = reader.query(query_expr, qds);

storage::RowRecord *record;
std::cout << "begin to dump data from tsfile ---" << std::endl;
int row_cout = 0;
do {
record = qds->get_next();
if (record) {
if (qds->next()) {
std::cout << "dump QDS : " << record->get_timestamp() << ",";
record = qds->get_row_record();
if (record) {
int size = record->get_fields()->size();
for (int i = 0; i < size; ++i) {
Expand Down
8 changes: 3 additions & 5 deletions cpp/examples/cpp_examples/demo_write.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -66,16 +66,14 @@ int demo_write() {
std::vector<MeasurementSchema> schema_vec[50];
for (int i = 0; i < device_num; i++) {
std::string device_name = "test_device" + std::to_string(i);
schema_vec[i].reserve(measurement_num);
for (int j = 0; j < measurement_num; j++) {
std::string measure_name = "measurement" + std::to_string(j);
schema_vec[i].push_back(
schema_vec[i].emplace_back(
MeasurementSchema(measure_name, common::TSDataType::INT32,
common::TSEncoding::PLAIN,
common::CompressionType::UNCOMPRESSED));
tsfile_writer_->register_timeseries(
device_name, measure_name, common::TSDataType::INT32,
common::TSEncoding::PLAIN,
common::CompressionType::UNCOMPRESSED);
tsfile_writer_->register_timeseries(device_name, schema_vec[i][j]);
}
}

Expand Down
34 changes: 34 additions & 0 deletions cpp/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,40 @@
</plugins>
</build>
<profiles>
<profile>
<id>linux-install-uuid-dev</id>
<activation>
<os>
<family>unix</family>
<name>Linux</name>
</os>
</activation>
<build>
<plugins>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<version>1.1.1</version>
<executions>
<execution>
<id>install-uuid-dev</id>
<phase>validate</phase>
<goals>
<goal>exec</goal>
</goals>
</execution>
</executions>
<configuration>
<executable>bash</executable>
<arguments>
<argument>-c</argument>
<argument>sudo apt-get update &amp;&amp; sudo apt-get install -y uuid-dev</argument>
</arguments>
</configuration>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>with-code-coverage</id>
<properties>
Expand Down
25 changes: 14 additions & 11 deletions cpp/src/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,9 @@ message("Running in src diectory")
if (${COV_ENABLED})
add_compile_options(-fprofile-arcs -ftest-coverage)
endif ()
add_definitions(-DANTLR4CPP_STATIC)
set(ANTLR4_WITH_STATIC_CRT OFF)
add_subdirectory(parser)
add_subdirectory(common)
add_subdirectory(compress)
add_subdirectory(cwrapper)
Expand All @@ -29,13 +32,22 @@ add_subdirectory(reader)
add_subdirectory(utils)
add_subdirectory(writer)

set(SNAPPY_LIB_NAME "snappy")
set(LZ4_LIB_NAME "LZ4")
set(LZO_LIB_NAME "lzokay")
target_link_libraries(parser_obj antlr4_static)
target_link_libraries(compress_obj ${SNAPPY_LIB_NAME} ${LZ4_LIB_NAME} ${LZO_LIB_NAME} zlibstatic )
target_link_libraries(common_obj ${SNAPPY_LIB_NAME} ${LZ4_LIB_NAME} ${LZO_LIB_NAME} zlibstatic )
target_link_libraries(read_obj ${SNAPPY_LIB_NAME} ${LZ4_LIB_NAME} ${LZO_LIB_NAME} zlibstatic )
target_link_libraries(write_obj ${SNAPPY_LIB_NAME} ${LZ4_LIB_NAME} ${LZO_LIB_NAME} zlibstatic )

add_library(tsfile SHARED)
if (${COV_ENABLED})
message("Enable code cov...")
target_link_libraries(tsfile common_obj compress_obj cwrapper_obj file_obj read_obj write_obj -lgcov)
target_link_libraries(tsfile common_obj compress_obj cwrapper_obj file_obj read_obj write_obj parser_obj -lgcov)
else()
message("Disable code cov...")
target_link_libraries(tsfile common_obj compress_obj cwrapper_obj file_obj read_obj write_obj)
target_link_libraries(tsfile common_obj compress_obj cwrapper_obj file_obj read_obj write_obj parser_obj)
endif()

set(LIBTSFILE_PROJECT_VERSION ${TsFile_CPP_VERSION})
Expand All @@ -46,15 +58,6 @@ set_target_properties(tsfile PROPERTIES SOVERSION ${LIBTSFILE_SO_VERSION})
set(LIBTSFILE_SDK_DIR ${LIBRARY_OUTPUT_PATH})
install(TARGETS tsfile LIBRARY DESTINATION ${LIBTSFILE_SDK_DIR})

set(SNAPPY_LIB_NAME "snappy")
set(LZ4_LIB_NAME "LZ4")
set(LZO_LIB_NAME "lzokay")
set(ZLIB_LIB_NAME "z")

target_link_libraries(compress_obj ${SNAPPY_LIB_NAME} ${LZ4_LIB_NAME} ${LZO_LIB_NAME} ${ZLIB_LIB_NAME})
target_link_libraries(common_obj ${SNAPPY_LIB_NAME} ${LZ4_LIB_NAME} ${LZO_LIB_NAME} ${ZLIB_LIB_NAME})
target_link_libraries(read_obj ${SNAPPY_LIB_NAME} ${LZ4_LIB_NAME} ${LZO_LIB_NAME} ${ZLIB_LIB_NAME})
target_link_libraries(write_obj ${SNAPPY_LIB_NAME} ${LZ4_LIB_NAME} ${LZO_LIB_NAME} ${ZLIB_LIB_NAME})
# set(CMAKE_PREFIX_PATH ../../third-party/lz4-dev/lib)
# set(LZ4_LIB_DIR ../../third-party/lz4-dev/lib)
# find_library(my_lz4_lib NAMES lz4 PATHS ${LZ4_LIB_DIR} NO_DEFAULT_PATH REQUIRED)
Expand Down
1 change: 1 addition & 0 deletions cpp/src/common/allocator/my_string.h
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,7 @@ struct String {

return this->len_ < other.len_;
}
std::string to_std_string() { return std::string(buf_, len_); }

#ifndef NDEBUG
friend std::ostream &operator<<(std::ostream &os, const String &s) {
Expand Down
18 changes: 18 additions & 0 deletions cpp/src/common/constant/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
#[[
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at

https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
]]
44 changes: 44 additions & 0 deletions cpp/src/common/constant/tsfile_constant.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* License); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
#include <string>
#include <regex>

namespace storage
{
static const std::string TSFILE_SUFFIX = ".tsfile";
static const std::string TSFILE_HOME = "TSFILE_HOME";
static const std::string TSFILE_CONF = "TSFILE_CONF";
static const std::string PATH_ROOT = "root";
static const std::string TMP_SUFFIX = "tmp";
static const std::string PATH_SEPARATOR = ".";
static const char PATH_SEPARATOR_CHAR = '.';
static const std::string PATH_SEPARATER_NO_REGEX = "\\.";
static const char DOUBLE_QUOTE = '"';
static const char BACK_QUOTE = '`';
static const std::string BACK_QUOTE_STRING = "`";
static const std::string DOUBLE_BACK_QUOTE_STRING = "``";

static const unsigned char TIME_COLUMN_MASK = 0x80;
static const unsigned char VALUE_COLUMN_MASK = 0x40;

static const std::string TIME_COLUMN_ID = "";

static const std::regex IDENTIFIER_PATTERN("([a-zA-Z0-9_\\u2E80-\\u9FFF]+)");
static const std::regex NODE_NAME_PATTERN("(\\*{0,2}[a-zA-Z0-9_\\u2E80-\\u9FFF]+\\*{0,2})");
} // namespace storage
1 change: 1 addition & 0 deletions cpp/src/common/mutex/mutex.h
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ class Mutex {
void unlock() {
int ret = pthread_mutex_unlock(&mutex_);
ASSERT(ret == 0);
(void) ret;
}

bool try_lock() {
Expand Down
26 changes: 26 additions & 0 deletions cpp/src/common/path.h
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,10 @@
#ifndef COMMON_READ_COMMON_PATH_H
#define COMMON_READ_COMMON_PATH_H

#include "utils/errno_define.h"
#include "parser/generated/PathParser.h"
#include "parser/path_nodes_generator.h"

#include <string>

namespace storage {
Expand All @@ -35,6 +39,28 @@ struct Path {
full_path_ = device + "." + measurement;
}

Path(const std::string& path_sc, bool if_split = true) {
if (!path_sc.empty()) {
if (!if_split) {
full_path_ = path_sc;
device_ = path_sc;
} else {
std::vector<std::string> nodes = PathNodesGenerator::invokeParser(path_sc);
if (nodes.size() > 0) {
for (uint64_t i = 0; i + 1 < nodes.size(); i++) {
device_ += nodes[i] + (i + 2 < nodes.size() ? "." : "");
}
measurement_ = nodes[nodes.size() - 1];
full_path_ = device_ + "." + measurement_;
} else {
full_path_ = path_sc;
device_ = "";
measurement_ = path_sc;
}
}
}
}

bool operator==(const Path &path) {
if (measurement_.compare(path.measurement_) == 0 &&
device_.compare(path.device_) == 0) {
Expand Down
16 changes: 9 additions & 7 deletions cpp/src/common/record.h
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
#include <vector>

#include "common/db_common.h"

#include "utils/errno_define.h"
namespace storage {

// TODO: use std::move
Expand Down Expand Up @@ -120,22 +120,24 @@ struct DataPoint {

struct TsRecord {
int64_t timestamp_;
std::string device_name_;
std::string device_id_;
std::vector<DataPoint> points_;

TsRecord(const std::string &device_name) : device_name_(device_name) {}
TsRecord(const std::string &device_name) : device_id_(device_name) {}

TsRecord(int64_t timestamp, const std::string &device_name,
int32_t point_count_in_row = 0)
: timestamp_(timestamp), device_name_(device_name), points_() {
: timestamp_(timestamp), device_id_(device_name), points_() {
if (point_count_in_row > 0) {
points_.reserve(point_count_in_row);
}
}

void append_data_point(const DataPoint &point) {
// points_.emplace_back(point); C++11
points_.push_back(point);
template <typename T>
int add_point(const std::string &measurement_name, T val) {
int ret = common::E_OK;
points_.emplace_back(DataPoint(measurement_name, val));
return ret;
}
};

Expand Down
26 changes: 25 additions & 1 deletion cpp/src/common/row_record.h
Original file line number Diff line number Diff line change
Expand Up @@ -86,9 +86,31 @@ struct Field {
}
}

template <typename T>
FORCE_INLINE T get_value() {
switch (type_) {
case common::TSDataType::BOOLEAN:
return value_.bval_;
case common::TSDataType::INT32:
return value_.ival_;
case common::TSDataType::INT64:
return value_.lval_;
case common::TSDataType::FLOAT:
return value_.fval_;
case common::TSDataType::DOUBLE:
return value_.dval_;
// case common::TSDataType::TEXT :
// return value_.sval_;
HTHou marked this conversation as resolved.
Show resolved Hide resolved
default:
std::cout << "unknown data type" << std::endl;
break;
}
return -1; // when data type is unknown
}

public:
common::TSDataType type_;

std::string column_name;
union {
bool bval_;
int64_t lval_;
Expand Down Expand Up @@ -182,6 +204,8 @@ class RowRecord {

FORCE_INLINE std::vector<Field *> *get_fields() { return fields_; }

FORCE_INLINE uint32_t get_col_num() { return col_num_; }

private:
int64_t time_; // time value
uint32_t col_num_; // measurement num
Expand Down
9 changes: 9 additions & 0 deletions cpp/src/common/schema.h
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,15 @@ struct MeasurementSchema {
chunk_writer_(nullptr),
value_chunk_writer_(nullptr) {}

MeasurementSchema(const std::string &measurement_name,
common::TSDataType data_type)
: measurement_name_(measurement_name),
data_type_(data_type),
encoding_(get_default_encoding_for_type(data_type)),
compression_type_(common::LZ4),
chunk_writer_(nullptr),
value_chunk_writer_(nullptr) {}

MeasurementSchema(const std::string &measurement_name,
common::TSDataType data_type, common::TSEncoding encoding,
common::CompressionType compression_type)
Expand Down
Loading
Loading