mysql 跨平台时 blob 读取失败

fasiondog · Apr 19, 2024 · a79da66 · a79da66
1 parent 69f9bae
commit a79da66
Show file tree

Hide file tree

Showing 9 changed files with 68 additions and 18 deletions.
diff --git a/hikyuu/data/common.py b/hikyuu/data/common.py
@@ -245,6 +245,7 @@ def historyfinancialreader(filepath):
         report_date = int(cw_info[313])  # 财务公告日期
         report_date = 19000000 + report_date if report_date > 800000 else 20000000 + report_date
         # results.append((modifiy_code(code), report_date, cw_info))
-        results.append((file_date, modifiy_code(code), report_date, cpp_bytes_to_vector_float_blob(info_data)))
+        # results.append((file_date, modifiy_code(code), report_date, cpp_bytes_to_vector_float_blob(info_data)))
+        results.append((file_date, modifiy_code(code), report_date, info_data))
     cw_file.close()
     return results
diff --git a/hikyuu_cpp/hikyuu/data_driver/base_info/mysql/MySQLBaseInfoDriver.cpp b/hikyuu_cpp/hikyuu/data_driver/base_info/mysql/MySQLBaseInfoDriver.cpp
@@ -412,7 +412,10 @@ vector<HistoryFinanceInfo> MySQLBaseInfoDriver::getHistoryFinance(const string &
             auto &finance = finances[i];
             auto &cur = result[i];
             cur.reportDate = Datetime(finance.report_date);
-            cur.values = std::move(finance.values);
+            // cur.values = std::move(finance.values);
+            size_t count = finance.values.size() / sizeof(float);
+            cur.values.resize(count);
+            memcpy(cur.values.data(), finance.values.data(), count * sizeof(float));
         }
 
     } catch (const std::exception &e) {

diff --git a/hikyuu_cpp/hikyuu/data_driver/base_info/sqlite/SQLiteBaseInfoDriver.cpp b/hikyuu_cpp/hikyuu/data_driver/base_info/sqlite/SQLiteBaseInfoDriver.cpp
@@ -402,7 +402,10 @@ vector<HistoryFinanceInfo> SQLiteBaseInfoDriver::getHistoryFinance(const string&
             auto& finance = finances[i];
             auto& cur = result[i];
             cur.reportDate = Datetime(finance.report_date);
-            cur.values = std::move(finance.values);
+            // cur.values = std::move(finance.values);
+            size_t count = finance.values.size() / sizeof(float);
+            cur.values.resize(count);
+            memcpy(cur.values.data(), finance.values.data(), count * sizeof(float));
         }
 
     } catch (const std::exception& e) {

diff --git a/hikyuu_cpp/hikyuu/data_driver/base_info/table/HistoryFinanceTable.h b/hikyuu_cpp/hikyuu/data_driver/base_info/table/HistoryFinanceTable.h
@@ -16,7 +16,8 @@ struct HistoryFinanceTable {
     uint64_t file_date;
     uint64_t report_date;
     std::string market_code;
-    std::vector<float> values;
+    // std::vector<float> values;
+    std::vector<char> values;
 };
 
 }  // namespace hku
diff --git a/hikyuu_cpp/hikyuu/utilities/db_connect/SQLStatementBase.h b/hikyuu_cpp/hikyuu/utilities/db_connect/SQLStatementBase.h
@@ -80,11 +80,8 @@ class HKU_API SQLStatementBase {
 
     /**
      * 将 item 的值绑定至 idx 指定的 SQL 参数中
-     * @param idx sql参数序号
-     * @param item 二进制数据起始指针
-     * @param len 二进制数据长度
      */
-    void bindBlob(int idx, const char *item, size_t len);
+    void bindBlob(int idx, const std::vector<char> &time);
 
     /** 将 item 的值绑定至 idx 指定的 SQL 参数中 */
     template <typename T>
@@ -94,6 +91,8 @@ class HKU_API SQLStatementBase {
     template <typename T>
     typename std::enable_if<!std::numeric_limits<T>::is_integer>::type bind(int idx, const T &item);
 
+    void bind(int idx, const std::vector<char> &item);
+
     /** 将 item 的值绑定至 idx 指定的 SQL 参数中 */
     template <typename T, typename... Args>
     void bind(int idx, const T &, const Args &...rest);
@@ -116,6 +115,8 @@ class HKU_API SQLStatementBase {
     /** 获取 idx 指定的数据至 item */
     void getColumn(int idx, Datetime &item);
 
+    void getColumn(int idx, std::vector<char> &);
+
     /** 获取 idx 指定的数据至 item */
     template <typename T>
     typename std::enable_if<std::numeric_limits<T>::is_integer>::type getColumn(int idx, T &);
@@ -142,14 +143,16 @@ class HKU_API SQLStatementBase {
     virtual void sub_bindText(int idx, const std::string &item) = 0;   ///< 子类接口 @see bind
     virtual void sub_bindText(int idx, const char *item, size_t len) = 0;  ///< 子类接口 @see bind
     virtual void sub_bindBlob(int idx, const std::string &item) = 0;  ///< 子类接口 @see bind
-    virtual void sub_bindBlob(int idx, const char *item, size_t len) = 0;  ///< 子类接口 @see bind
+    virtual void sub_bindBlob(int idx, const std::vector<char> &item) = 0;  ///< 子类接口 @see bind
 
     virtual int sub_getNumColumns() const = 0;                  ///< 子类接口 @see getNumColumns
     virtual void sub_getColumnAsInt64(int idx, int64_t &) = 0;  ///< 子类接口 @see getColumn
     virtual void sub_getColumnAsDouble(int idx, double &) = 0;  ///< 子类接口 @see getColumn
     virtual void sub_getColumnAsDatetime(int idx, Datetime &) = 0;  ///< 子类接口 @see getColumn
     virtual void sub_getColumnAsText(int idx, std::string &) = 0;   ///< 子类接口 @see getColumn
     virtual void sub_getColumnAsBlob(int idx, std::string &) = 0;   ///< 子类接口 @see getColumn
+    virtual void sub_getColumnAsBlob(int idx,
+                                     std::vector<char> &) = 0;  ///< 子类接口 @see getColumn
 
 private:
     SQLStatementBase() = delete;
@@ -210,8 +213,8 @@ inline void SQLStatementBase::bindBlob(int idx, const std::string &item) {
     sub_bindBlob(idx, item);
 }
 
-inline void SQLStatementBase::bindBlob(int idx, const char *item, size_t len) {
-    sub_bindBlob(idx, item, len);
+inline void SQLStatementBase::bindBlob(int idx, const std::vector<char> &item) {
+    sub_bindBlob(idx, item);
 }
 
 inline uint64_t SQLStatementBase::getLastRowid() {
@@ -240,6 +243,10 @@ inline void SQLStatementBase::getColumn(int idx, std::string &item) {
     sub_getColumnAsText(idx, item);
 }
 
+inline void SQLStatementBase::bind(int idx, const std::vector<char> &item) {
+    sub_bindBlob(idx, item);
+}
+
 template <typename T>
 typename std::enable_if<std::numeric_limits<T>::is_integer>::type SQLStatementBase::bind(
   int idx, const T &item) {
@@ -263,6 +270,10 @@ typename std::enable_if<std::numeric_limits<T>::is_integer>::type SQLStatementBa
     item = (T)temp;
 }
 
+inline void SQLStatementBase::getColumn(int idx, std::vector<char> &item) {
+    sub_getColumnAsBlob(idx, item);
+}
+
 template <typename T>
 typename std::enable_if<!std::numeric_limits<T>::is_integer>::type SQLStatementBase::getColumn(
   int idx, T &item) {

diff --git a/hikyuu_cpp/hikyuu/utilities/db_connect/mysql/MySQLStatement.cpp b/hikyuu_cpp/hikyuu/utilities/db_connect/mysql/MySQLStatement.cpp
@@ -266,12 +266,12 @@ void MySQLStatement::sub_bindBlob(int idx, const string& item) {
     m_param_bind[idx].is_null = 0;
 }
 
-void MySQLStatement::sub_bindBlob(int idx, const char* item, size_t len) {
+void MySQLStatement::sub_bindBlob(int idx, const std::vector<char>& item) {
     HKU_CHECK(idx < m_param_bind.size(), "idx out of range! idx: {}, total: {}", idx,
               m_param_bind.size());
-    m_param_buffer.push_back(std::string(item));
+    m_param_buffer.push_back(item);
     auto& buf = m_param_buffer.back();
-    std::string* p = boost::any_cast<std::string>(&buf);
+    std::vector<char>* p = boost::any_cast<std::vector<char>>(&buf);
     m_param_bind[idx].buffer_type = MYSQL_TYPE_BLOB;
     m_param_bind[idx].buffer = (void*)p->data();
     m_param_bind[idx].buffer_length = p->size();
@@ -403,6 +403,24 @@ void MySQLStatement::sub_getColumnAsBlob(int idx, string& item) {
     }
 }
 
+void MySQLStatement::sub_getColumnAsBlob(int idx, std::vector<char>& item) {
+    HKU_CHECK(idx < m_result_buffer.size(), "idx out of range! idx: {}, total: {}",
+              m_result_buffer.size());
+
+    HKU_CHECK(m_result_error[idx] == 0, "Error occurred in sub_getColumnAsBlob! idx: {}", idx);
+
+    if (m_result_is_null[idx]) {
+        item.clear();
+        return;
+    }
+
+    try {
+        item = boost::any_cast<vector<char>>(m_result_buffer[idx]);
+    } catch (...) {
+        HKU_THROW("Field type mismatch! idx: {}", idx);
+    }
+}
+
 }  // namespace hku
 
 #ifdef _MSC_VER

diff --git a/hikyuu_cpp/hikyuu/utilities/db_connect/mysql/MySQLStatement.h b/hikyuu_cpp/hikyuu/utilities/db_connect/mysql/MySQLStatement.h
@@ -49,14 +49,15 @@ class HKU_API MySQLStatement : public SQLStatementBase {
     virtual void sub_bindText(int idx, const std::string& item) override;
     virtual void sub_bindText(int idx, const char* item, size_t len) override;
     virtual void sub_bindBlob(int idx, const std::string& item) override;
-    virtual void sub_bindBlob(int idx, const char* item, size_t len) override;
+    virtual void sub_bindBlob(int idx, const std::vector<char>& item) override;
 
     virtual int sub_getNumColumns() const override;
     virtual void sub_getColumnAsInt64(int idx, int64_t& item) override;
     virtual void sub_getColumnAsDouble(int idx, double& item) override;
     virtual void sub_getColumnAsDatetime(int idx, Datetime& item) override;
     virtual void sub_getColumnAsText(int idx, std::string& item) override;
     virtual void sub_getColumnAsBlob(int idx, std::string& item) override;
+    virtual void sub_getColumnAsBlob(int idx, std::vector<char>& item) override;
 
 private:
     void _reset();

diff --git a/hikyuu_cpp/hikyuu/utilities/db_connect/sqlite/SQLiteStatement.cpp b/hikyuu_cpp/hikyuu/utilities/db_connect/sqlite/SQLiteStatement.cpp
@@ -129,9 +129,10 @@ void SQLiteStatement::sub_bindBlob(int idx, const string &item) {
     SQL_CHECK(status == SQLITE_OK, status, sqlite3_errmsg(m_db));
 }
 
-void SQLiteStatement::sub_bindBlob(int idx, const char *item, size_t len) {
+void SQLiteStatement::sub_bindBlob(int idx, const std::vector<char> &item) {
     _reset();
-    int status = sqlite3_bind_blob(m_stmt, idx + 1, item, (int)len, SQLITE_TRANSIENT);
+    int status =
+      sqlite3_bind_blob(m_stmt, idx + 1, item.data(), (int)item.size(), SQLITE_TRANSIENT);
     SQL_CHECK(status == SQLITE_OK, status, sqlite3_errmsg(m_db));
 }
 
@@ -163,6 +164,16 @@ void SQLiteStatement::sub_getColumnAsBlob(int idx, std::string &item) {
     item = std::string(data, size);
 }
 
+void SQLiteStatement::sub_getColumnAsBlob(int idx, std::vector<char> &item) {
+    const char *data = static_cast<const char *>(sqlite3_column_blob(m_stmt, idx));
+    if (data == NULL) {
+        throw null_blob_exception();
+    }
+    const int size = sqlite3_column_bytes(m_stmt, idx);
+    item.resize(size);
+    memcpy(item.data(), data, size);
+}
+
 uint64_t SQLiteStatement::sub_getLastRowid() {
     return sqlite3_last_insert_rowid(m_db);
 }

diff --git a/hikyuu_cpp/hikyuu/utilities/db_connect/sqlite/SQLiteStatement.h b/hikyuu_cpp/hikyuu/utilities/db_connect/sqlite/SQLiteStatement.h
@@ -44,14 +44,15 @@ class HKU_API SQLiteStatement : public SQLStatementBase {
     virtual void sub_bindText(int idx, const std::string &item) override;
     virtual void sub_bindText(int idx, const char *item, size_t len) override;
     virtual void sub_bindBlob(int idx, const std::string &item) override;
-    virtual void sub_bindBlob(int idx, const char *item, size_t len) override;
+    virtual void sub_bindBlob(int idx, const std::vector<char> &item) override;
 
     virtual int sub_getNumColumns() const override;
     virtual void sub_getColumnAsInt64(int idx, int64_t &item) override;
     virtual void sub_getColumnAsDouble(int idx, double &item) override;
     virtual void sub_getColumnAsDatetime(int idx, Datetime &item) override;
     virtual void sub_getColumnAsText(int idx, std::string &item) override;
     virtual void sub_getColumnAsBlob(int idx, std::string &item) override;
+    virtual void sub_getColumnAsBlob(int idx, std::vector<char> &item) override;
 
 private:
     void _reset();