impl truncation

Committed-by: xiaolei.zl from Dev container

flex/tests/hqps/trucate_test.cc

@@ -0,0 +1,12 @@
+#include "flex/utils/property/column.h"
+
+#include <string>
+#include <string_view>
+
+int main(int argc, char** argv) {
+  std::string str = "abcdefO(1/ε^2)";
+  std::cout << "str: " << str << ", size: " << str.size() << std::endl;
+  std::string_view sv = gs::truncate_utf8(str, 12);
+  std::cout << sv << ", size:" << sv.size() << std::endl;
+  return 0;
+}

flex/utils/property/column.cc

@@ -22,6 +22,35 @@
 
 namespace gs {
 
+std::string_view truncate_utf8(std::string_view str, size_t length) {
+  if (str.size() <= length) {
+    return str;
+  }
+  size_t byte_count = 0;
+
+  for (const char* p = str.data(); *p && byte_count < length;) {
+    unsigned char ch = *p;
+    size_t char_length = 0;
+    if ((ch & 0x80) == 0) {
+      char_length = 1;
+    } else if ((ch & 0xE0) == 0xC0) {
+      char_length = 2;
+    } else if ((ch & 0xF0) == 0xE0) {
+      char_length = 3;
+    } else if ((ch & 0xF8) == 0xF0) {
+      char_length = 4;
+    }
+    LOG(INFO) << "current char length: " << char_length
+              << ", byte_count: " << byte_count;
+    if (byte_count + char_length > length) {
+      break;
+    }
+    p += char_length;
+    byte_count += char_length;
+  }
+  return str.substr(0, byte_count);
+}
+
 template <typename T>
 class TypedEmptyColumn : public ColumnBase {
  public:

flex/utils/property/column.h

@@ -26,6 +26,8 @@
 
 namespace gs {
 
+std::string_view truncate_utf8(std::string_view str, size_t length);
+
 class ColumnBase {
  public:
   virtual ~ColumnBase() {}
@@ -503,7 +505,7 @@ class TypedColumn<std::string_view> : public ColumnBase {
     if (copied_val.size() >= width_) {
       VLOG(1) << "String length" << copied_val.size()
               << " exceeds the maximum length: " << width_ << ", cut off.";
-      copied_val = copied_val.substr(0, width_);
+      copied_val = truncate_utf8(copied_val, width_);
     }
     if (idx >= basic_size_ && idx < basic_size_ + extra_size_) {
       size_t offset = pos_.fetch_add(copied_val.size());