From 00db50dab79416b991d0c7e7f2433ac62bd03881 Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Fri, 1 Nov 2024 13:24:25 +0100 Subject: [PATCH 01/43] User defined c files is compiled and callable --- Gemfile | 5 ----- cpp/bindings.cpp | 11 ++++++++++ example/Gemfile | 6 ++++-- example/Gemfile.lock | 30 ++++++++++++++++------------ example/c_sources/custom_tokenizer.c | 5 +++++ example/c_sources/custom_tokenizer.h | 6 ++++++ example/ios/Podfile.lock | 4 ++-- example/src/App.tsx | 5 ++++- op-sqlite.podspec | 24 ++++++++++++++++++---- src/index.ts | 5 +++++ 10 files changed, 74 insertions(+), 27 deletions(-) delete mode 100644 Gemfile create mode 100644 example/c_sources/custom_tokenizer.c create mode 100644 example/c_sources/custom_tokenizer.h diff --git a/Gemfile b/Gemfile deleted file mode 100644 index f963de13..00000000 --- a/Gemfile +++ /dev/null @@ -1,5 +0,0 @@ -source 'https://rubygems.org' -# You may use http://rbenv.org/ or https://rvm.io/ to install and use this version -ruby '>= 2.7.6' -gem 'cocoapods', '>= 1.13', '!= 1.15.0', '!= 1.15.1' -gem 'activesupport', '>= 6.1.7.5', '!= 7.1.0' \ No newline at end of file diff --git a/cpp/bindings.cpp b/cpp/bindings.cpp index 1aa502ce..74ae99d0 100644 --- a/cpp/bindings.cpp +++ b/cpp/bindings.cpp @@ -14,6 +14,12 @@ #include #include #include +extern "C" { + #include "../example/c_sources/custom_tokenizer.h" +} +//#include "example/c_sources/custom_tokenizer.h" +//#include "../example/c_sources/custom_tokenizer.h" +//#include "custom_tokenizer.h" namespace opsqlite { @@ -53,6 +59,10 @@ void install(jsi::Runtime &rt, const std::shared_ptr& invoke _base_path = std::string(base_path); _crsqlite_path = std::string(crsqlite_path); _sqlite_vec_path = std::string(sqlite_vec_path); + + auto answer = HOSTFN("answer") { + return custom_tokenizer(); + }); auto open = HOSTFN("open") { jsi::Object options = args[0].asObject(rt); @@ -160,6 +170,7 @@ void install(jsi::Runtime &rt, const std::shared_ptr& invoke module.setProperty(rt, "open", std::move(open)); module.setProperty(rt, "isSQLCipher", std::move(is_sqlcipher)); module.setProperty(rt, "isLibsql", std::move(is_libsql)); + module.setProperty(rt, "answer", std::move(answer)); #ifdef OP_SQLITE_USE_LIBSQL module.setProperty(rt, "openRemote", std::move(open_remote)); module.setProperty(rt, "openSync", std::move(open_sync)); diff --git a/example/Gemfile b/example/Gemfile index 5f7f7e4e..bc0f3637 100644 --- a/example/Gemfile +++ b/example/Gemfile @@ -2,5 +2,7 @@ source 'https://rubygems.org' ruby '>= 2.7.6' -gem 'cocoapods', '>= 1.13', '!= 1.15.0', '!= 1.15.1' -gem 'activesupport', '>= 6.1.7.5', '!= 7.1.0' \ No newline at end of file +gem 'cocoapods', '=1.15.2' +gem 'activesupport', '>= 6.1.7.5', '!= 7.1.0' +gem 'bigdecimal' +gem 'mutex_m' \ No newline at end of file diff --git a/example/Gemfile.lock b/example/Gemfile.lock index d7b8e7d5..3a7d5735 100644 --- a/example/Gemfile.lock +++ b/example/Gemfile.lock @@ -10,18 +10,19 @@ GEM i18n (>= 1.6, < 2) minitest (>= 5.1) tzinfo (~> 2.0) - addressable (2.8.6) - public_suffix (>= 2.0.2, < 6.0) + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) algoliasearch (1.27.5) httpclient (~> 2.8, >= 2.8.3) json (>= 1.5.1) atomos (0.1.3) base64 (0.2.0) + bigdecimal (3.1.5) claide (1.1.0) - cocoapods (1.14.3) + cocoapods (1.15.2) addressable (~> 2.8) claide (>= 1.0.2, < 2.0) - cocoapods-core (= 1.14.3) + cocoapods-core (= 1.15.2) cocoapods-deintegrate (>= 1.0.3, < 2.0) cocoapods-downloader (>= 2.1, < 3.0) cocoapods-plugins (>= 1.0.0, < 2.0) @@ -36,7 +37,7 @@ GEM nap (~> 1.0) ruby-macho (>= 2.3.0, < 3.0) xcodeproj (>= 1.23.0, < 2.0) - cocoapods-core (1.14.3) + cocoapods-core (1.15.2) activesupport (>= 5.0, < 8) addressable (~> 2.8) algoliasearch (~> 1.0) @@ -60,41 +61,44 @@ GEM escape (0.0.4) ethon (0.16.0) ffi (>= 1.15.0) - ffi (1.16.3) + ffi (1.17.0) fourflusher (2.3.1) fuzzy_match (2.0.4) gh_inspector (1.1.3) httpclient (2.8.3) i18n (1.14.4) concurrent-ruby (~> 1.0) - json (2.7.2) + json (2.7.5) minitest (5.22.3) molinillo (0.8.0) - nanaimo (0.3.0) + mutex_m (0.2.0) + nanaimo (0.4.0) nap (1.1.0) netrc (0.11.0) nkf (0.2.0) public_suffix (4.0.7) - rexml (3.2.6) + rexml (3.3.9) ruby-macho (2.5.1) typhoeus (1.4.1) ethon (>= 0.9.0) tzinfo (2.0.6) concurrent-ruby (~> 1.0) - xcodeproj (1.24.0) + xcodeproj (1.27.0) CFPropertyList (>= 2.3.3, < 4.0) atomos (~> 0.1.3) claide (>= 1.0.2, < 2.0) colored2 (~> 3.1) - nanaimo (~> 0.3.0) - rexml (~> 3.2.4) + nanaimo (~> 0.4.0) + rexml (>= 3.3.6, < 4.0) PLATFORMS ruby DEPENDENCIES activesupport (>= 6.1.7.5, != 7.1.0) - cocoapods (>= 1.13, != 1.15.1, != 1.15.0) + bigdecimal + cocoapods (= 1.15.2) + mutex_m RUBY VERSION ruby 3.3.1p55 diff --git a/example/c_sources/custom_tokenizer.c b/example/c_sources/custom_tokenizer.c new file mode 100644 index 00000000..b452957f --- /dev/null +++ b/example/c_sources/custom_tokenizer.c @@ -0,0 +1,5 @@ +#include "custom_tokenizer.h" + +int custom_tokenizer(void) { + return 42; +} diff --git a/example/c_sources/custom_tokenizer.h b/example/c_sources/custom_tokenizer.h new file mode 100644 index 00000000..1b0d88e1 --- /dev/null +++ b/example/c_sources/custom_tokenizer.h @@ -0,0 +1,6 @@ +#ifndef CUSTOM_TOKENIZER_H +#define CUSTOM_TOKENIZER_H + +int custom_tokenizer(void); + +#endif // CUSTOM_TOKENIZER_H \ No newline at end of file diff --git a/example/ios/Podfile.lock b/example/ios/Podfile.lock index 899770f3..83ba57ef 100644 --- a/example/ios/Podfile.lock +++ b/example/ios/Podfile.lock @@ -10,7 +10,7 @@ PODS: - hermes-engine (0.76.1): - hermes-engine/Pre-built (= 0.76.1) - hermes-engine/Pre-built (0.76.1) - - op-sqlite (9.3.0): + - op-sqlite (10.0.0): - DoubleConversion - glog - hermes-engine @@ -1785,7 +1785,7 @@ SPEC CHECKSUMS: GCDWebServer: 2c156a56c8226e2d5c0c3f208a3621ccffbe3ce4 glog: 08b301085f15bcbb6ff8632a8ebaf239aae04e6a hermes-engine: 46f1ffbf0297f4298862068dd4c274d4ac17a1fd - op-sqlite: 1d96ea1e6fcab14883b7315d4975374ff91eb899 + op-sqlite: 4b8f341746c641f8beaca99071aba72d3dee1ab8 RCT-Folly: bf5c0376ffe4dd2cf438dcf86db385df9fdce648 RCTDeprecation: fde92935b3caa6cb65cbff9fbb7d3a9867ffb259 RCTRequired: 75c6cee42d21c1530a6f204ba32ff57335d19007 diff --git a/example/src/App.tsx b/example/src/App.tsx index 77d84b4f..5d827f09 100644 --- a/example/src/App.tsx +++ b/example/src/App.tsx @@ -1,6 +1,7 @@ import clsx from 'clsx'; import {useEffect, useState} from 'react'; import { + Alert, Button, Clipboard, SafeAreaView, @@ -15,7 +16,7 @@ import {blobTests, dbSetupTests, queriesTests, runTests} from './tests/index'; import {preparedStatementsTests} from './tests/preparedStatements.spec'; import {reactiveTests} from './tests/reactive.spec'; import {setServerResults, startServer, stopServer} from './server'; -import {open} from '@op-engineering/op-sqlite'; +import {answer, open} from '@op-engineering/op-sqlite'; import Share from 'react-native-share'; import {createLargeDB, queryLargeDB} from './Database'; import RNRestart from 'react-native-restart'; @@ -45,6 +46,8 @@ export default function App() { startServer(); + Alert.alert('Custom C file answer: ' + answer()); + return () => { stopServer(); }; diff --git a/op-sqlite.podspec b/op-sqlite.podspec index d14e7d93..f51a0a7a 100644 --- a/op-sqlite.podspec +++ b/op-sqlite.podspec @@ -28,6 +28,7 @@ sqlite_flags = "" fts5 = false rtree = false use_sqlite_vec = false +extra_c_sources = [] if(op_sqlite_config != nil) use_sqlcipher = op_sqlite_config["sqlcipher"] == true @@ -39,6 +40,7 @@ if(op_sqlite_config != nil) fts5 = op_sqlite_config["fts5"] == true rtree = op_sqlite_config["rtree"] == true use_sqlite_vec = op_sqlite_config["sqliteVec"] == true + extra_c_sources = op_sqlite_config["cSources"] || [] end if phone_version then @@ -74,12 +76,26 @@ Pod::Spec.new do |s| s.platforms = { :ios => "13.0", :osx => "10.15", :visionos => "1.0" } s.source = { :git => "https://github.com/op-engineering/op-sqlite.git", :tag => "#{s.version}" } - s.source_files = "ios/**/*.{h,m,mm}", "cpp/**/*.{h,cpp,c}" + # Base source files + source_files = Dir.glob("ios/**/*.{h,m,mm}") + Dir.glob("cpp/**/*.{h,cpp,c}") + + # Set the path to the `c_sources` directory based on environment + if __dir__.include?("node_modules") + c_sources_dir = File.join("..", "..", "..", "c_sources") + else + c_sources_dir = File.join("example", "c_sources") + end + + # Add all .h and .c files from the `c_sources` directory + source_files += Dir.glob(File.join(c_sources_dir, "**/*.{h,c}")) + # s.public_header_files = 'example/c_sources/**/*.h' + + # Assign the collected source files to `s.source_files` + s.source_files = source_files xcconfig = { :GCC_PREPROCESSOR_DEFINITIONS => "HAVE_FULLFSYNC=1", :WARNING_CFLAGS => "-Wno-shorten-64-to-32 -Wno-comma -Wno-unreachable-code -Wno-conditional-uninitialized -Wno-deprecated-declarations", - :USE_HEADERMAP => "No", :CLANG_CXX_LANGUAGE_STANDARD => "c++17", } @@ -87,7 +103,7 @@ Pod::Spec.new do |s| if use_sqlcipher then log_message.call("[OP-SQLITE] using SQLCipher πŸ”’") - s.exclude_files = "cpp/sqlite3.c", "cpp/sqlite3.h", "cpp/libsql/bridge.c", "cpp/libsql/bridge.h" + s.exclude_files = "cpp/sqlite3.c", "cpp/sqlite3.h", "cpp/libsql/bridge.c", "cpp/libsql/bridge.h", "cpp/libsql/bridge.cpp", "cpp/libsql/libsql.h" xcconfig[:GCC_PREPROCESSOR_DEFINITIONS] += " OP_SQLITE_USE_SQLCIPHER=1 HAVE_FULLFSYNC=1 SQLITE_HAS_CODEC SQLITE_TEMP_STORE=2" s.dependency "OpenSSL-Universal" elsif use_libsql then @@ -95,7 +111,7 @@ Pod::Spec.new do |s| s.exclude_files = "cpp/sqlite3.c", "cpp/sqlite3.h", "cpp/sqlcipher/sqlite3.c", "cpp/sqlcipher/sqlite3.h", "cpp/bridge.h", "cpp/bridge.cpp" else log_message.call("[OP-SQLITE] using vanilla SQLite πŸ“¦") - s.exclude_files = "cpp/sqlcipher/sqlite3.c", "cpp/sqlcipher/sqlite3.h", "cpp/libsql/bridge.c", "cpp/libsql/bridge.h" + s.exclude_files = "cpp/sqlcipher/sqlite3.c", "cpp/sqlcipher/sqlite3.h", "cpp/libsql/bridge.c", "cpp/libsql/bridge.h", "cpp/libsql/bridge.cpp", "cpp/libsql/libsql.h" end s.dependency "React-callinvoker" diff --git a/src/index.ts b/src/index.ts index 01963067..25dac0ee 100644 --- a/src/index.ts +++ b/src/index.ts @@ -197,6 +197,7 @@ type OPSQLiteProxy = { }) => DB; isSQLCipher: () => boolean; isLibsql: () => boolean; + answer: () => number; }; const locks: Record< @@ -459,3 +460,7 @@ export const isSQLCipher = (): boolean => { export const isLibsql = (): boolean => { return OPSQLite.isLibsql(); }; + +export const answer = (): number => { + return OPSQLite.answer(); +}; From b8f70131a36adfbe48e2af14214a87d5864e2fea Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Fri, 1 Nov 2024 15:26:56 +0100 Subject: [PATCH 02/43] basic header generation --- cpp/bindings.cpp | 10 +++------- example/c_sources/custom_tokenizer.c | 5 ----- example/c_sources/custom_tokenizer.h | 6 ------ example/c_sources/tokenizers.cpp | 6 ++++++ example/c_sources/tokenizers.h | 10 ++++++++++ example/ios/Podfile.lock | 2 +- example/package.json | 5 ++++- generate_tokenizers_header_file.rb | 28 ++++++++++++++++++++++++++++ op-sqlite.podspec | 16 +++++++++++----- 9 files changed, 63 insertions(+), 25 deletions(-) delete mode 100644 example/c_sources/custom_tokenizer.c delete mode 100644 example/c_sources/custom_tokenizer.h create mode 100644 example/c_sources/tokenizers.cpp create mode 100644 example/c_sources/tokenizers.h create mode 100644 generate_tokenizers_header_file.rb diff --git a/cpp/bindings.cpp b/cpp/bindings.cpp index 74ae99d0..be04c139 100644 --- a/cpp/bindings.cpp +++ b/cpp/bindings.cpp @@ -14,12 +14,7 @@ #include #include #include -extern "C" { - #include "../example/c_sources/custom_tokenizer.h" -} -//#include "example/c_sources/custom_tokenizer.h" -//#include "../example/c_sources/custom_tokenizer.h" -//#include "custom_tokenizer.h" +#include "../example/c_sources/tokenizers.h" namespace opsqlite { @@ -61,7 +56,8 @@ void install(jsi::Runtime &rt, const std::shared_ptr& invoke _sqlite_vec_path = std::string(sqlite_vec_path); auto answer = HOSTFN("answer") { - return custom_tokenizer(); + ngram(); + return 52; }); auto open = HOSTFN("open") { diff --git a/example/c_sources/custom_tokenizer.c b/example/c_sources/custom_tokenizer.c deleted file mode 100644 index b452957f..00000000 --- a/example/c_sources/custom_tokenizer.c +++ /dev/null @@ -1,5 +0,0 @@ -#include "custom_tokenizer.h" - -int custom_tokenizer(void) { - return 42; -} diff --git a/example/c_sources/custom_tokenizer.h b/example/c_sources/custom_tokenizer.h deleted file mode 100644 index 1b0d88e1..00000000 --- a/example/c_sources/custom_tokenizer.h +++ /dev/null @@ -1,6 +0,0 @@ -#ifndef CUSTOM_TOKENIZER_H -#define CUSTOM_TOKENIZER_H - -int custom_tokenizer(void); - -#endif // CUSTOM_TOKENIZER_H \ No newline at end of file diff --git a/example/c_sources/tokenizers.cpp b/example/c_sources/tokenizers.cpp new file mode 100644 index 00000000..0184f00e --- /dev/null +++ b/example/c_sources/tokenizers.cpp @@ -0,0 +1,6 @@ +#include "tokenizers.h" +#include + +namespace opsqlite { +void ngram() { std::cout << "ngram" << std::endl; } +} // namespace opsqlite \ No newline at end of file diff --git a/example/c_sources/tokenizers.h b/example/c_sources/tokenizers.h new file mode 100644 index 00000000..9523a481 --- /dev/null +++ b/example/c_sources/tokenizers.h @@ -0,0 +1,10 @@ +#ifndef TOKENIZERS_H +#define TOKENIZERS_H + +namespace opsqlite { + +void ngram(); + +} // namespace opsqlite + +#endif // TOKENIZERS_H diff --git a/example/ios/Podfile.lock b/example/ios/Podfile.lock index 83ba57ef..352d96b6 100644 --- a/example/ios/Podfile.lock +++ b/example/ios/Podfile.lock @@ -1785,7 +1785,7 @@ SPEC CHECKSUMS: GCDWebServer: 2c156a56c8226e2d5c0c3f208a3621ccffbe3ce4 glog: 08b301085f15bcbb6ff8632a8ebaf239aae04e6a hermes-engine: 46f1ffbf0297f4298862068dd4c274d4ac17a1fd - op-sqlite: 4b8f341746c641f8beaca99071aba72d3dee1ab8 + op-sqlite: dbfab1f7d79984b6acbe1b80602d9a0236c68eb6 RCT-Folly: bf5c0376ffe4dd2cf438dcf86db385df9fdce648 RCTDeprecation: fde92935b3caa6cb65cbff9fbb7d3a9867ffb259 RCTRequired: 75c6cee42d21c1530a6f204ba32ff57335d19007 diff --git a/example/package.json b/example/package.json index 3ffc9adb..b52055d0 100644 --- a/example/package.json +++ b/example/package.json @@ -73,6 +73,9 @@ "fts5": true, "rtree": true, "libsql": false, - "sqliteVec": true + "sqliteVec": true, + "tokenizers": [ + "ngram" + ] } } diff --git a/generate_tokenizers_header_file.rb b/generate_tokenizers_header_file.rb new file mode 100644 index 00000000..8e1a3e3d --- /dev/null +++ b/generate_tokenizers_header_file.rb @@ -0,0 +1,28 @@ +require 'fileutils' + +def generate_tokenizers_header_file(names, file_path) + # Ensure the directory exists + dir_path = File.dirname(file_path) + FileUtils.mkdir_p(dir_path) unless Dir.exist?(dir_path) + + File.open(file_path, 'w') do |file| + file.puts "#ifndef TOKENIZERS_H" + file.puts "#define TOKENIZERS_H" + file.puts + file.puts "namespace opsqlite {" + file.puts + + names.each do |name| + file.puts "void #{name}();" + end + + file.puts + file.puts "} // namespace opsqlite" + file.puts + file.puts "#endif // TOKENIZERS_H" + end +end + +# # Example usage: +# names = ["functionOne", "functionTwo", "functionThree"] +# generate_header_file(names, "/path/to/generated_header.h") \ No newline at end of file diff --git a/op-sqlite.podspec b/op-sqlite.podspec index f51a0a7a..41efe62c 100644 --- a/op-sqlite.podspec +++ b/op-sqlite.podspec @@ -1,4 +1,5 @@ require "json" +require_relative "./generate_tokenizers_header_file" log_message = lambda do |message| puts "\e[34m#{message}\e[0m" @@ -28,7 +29,7 @@ sqlite_flags = "" fts5 = false rtree = false use_sqlite_vec = false -extra_c_sources = [] +tokenizers = [] if(op_sqlite_config != nil) use_sqlcipher = op_sqlite_config["sqlcipher"] == true @@ -40,7 +41,7 @@ if(op_sqlite_config != nil) fts5 = op_sqlite_config["fts5"] == true rtree = op_sqlite_config["rtree"] == true use_sqlite_vec = op_sqlite_config["sqliteVec"] == true - extra_c_sources = op_sqlite_config["cSources"] || [] + tokenizers = op_sqlite_config["tokenizers"] || [] end if phone_version then @@ -79,16 +80,21 @@ Pod::Spec.new do |s| # Base source files source_files = Dir.glob("ios/**/*.{h,m,mm}") + Dir.glob("cpp/**/*.{h,cpp,c}") - # Set the path to the `c_sources` directory based on environment + + + # # Set the path to the `c_sources` directory based on environment if __dir__.include?("node_modules") c_sources_dir = File.join("..", "..", "..", "c_sources") else c_sources_dir = File.join("example", "c_sources") end + if tokenizers.any? + generate_tokenizers_header_file(tokenizers, File.join(c_sources_dir, "tokenizers.h")) + end + # Add all .h and .c files from the `c_sources` directory - source_files += Dir.glob(File.join(c_sources_dir, "**/*.{h,c}")) - # s.public_header_files = 'example/c_sources/**/*.h' + source_files += Dir.glob(File.join(c_sources_dir, "**/*.{h,cpp}")) # Assign the collected source files to `s.source_files` s.source_files = source_files From 03f83191bf73117005306589e258acd5978fb922 Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Sat, 2 Nov 2024 18:56:30 +0100 Subject: [PATCH 03/43] Pass tokenizer list as compilation param --- cpp/bindings.cpp | 7 +++++++ example/c_sources/tokenizers.cpp | 1 + example/c_sources/tokenizers.h | 1 + example/ios/Podfile.lock | 2 +- example/package.json | 3 ++- op-sqlite.podspec | 4 ++++ 6 files changed, 16 insertions(+), 2 deletions(-) diff --git a/cpp/bindings.cpp b/cpp/bindings.cpp index be04c139..c28107c9 100644 --- a/cpp/bindings.cpp +++ b/cpp/bindings.cpp @@ -16,6 +16,10 @@ #include #include "../example/c_sources/tokenizers.h" +#ifndef STRING_LIST +#define STRING_LIST "" +#endif + namespace opsqlite { namespace jsi = facebook::jsi; @@ -56,7 +60,10 @@ void install(jsi::Runtime &rt, const std::shared_ptr& invoke _sqlite_vec_path = std::string(sqlite_vec_path); auto answer = HOSTFN("answer") { + std::cout << "List of tokenizers" << std::endl; + std::cout << STRING_LIST << std::endl; ngram(); + edgengram(); return 52; }); diff --git a/example/c_sources/tokenizers.cpp b/example/c_sources/tokenizers.cpp index 0184f00e..d2a6a709 100644 --- a/example/c_sources/tokenizers.cpp +++ b/example/c_sources/tokenizers.cpp @@ -3,4 +3,5 @@ namespace opsqlite { void ngram() { std::cout << "ngram" << std::endl; } +void edgengram() { std::cout << "edgengram" << std::endl; } } // namespace opsqlite \ No newline at end of file diff --git a/example/c_sources/tokenizers.h b/example/c_sources/tokenizers.h index 9523a481..2529e919 100644 --- a/example/c_sources/tokenizers.h +++ b/example/c_sources/tokenizers.h @@ -4,6 +4,7 @@ namespace opsqlite { void ngram(); +void edgengram(); } // namespace opsqlite diff --git a/example/ios/Podfile.lock b/example/ios/Podfile.lock index 352d96b6..58a01186 100644 --- a/example/ios/Podfile.lock +++ b/example/ios/Podfile.lock @@ -1785,7 +1785,7 @@ SPEC CHECKSUMS: GCDWebServer: 2c156a56c8226e2d5c0c3f208a3621ccffbe3ce4 glog: 08b301085f15bcbb6ff8632a8ebaf239aae04e6a hermes-engine: 46f1ffbf0297f4298862068dd4c274d4ac17a1fd - op-sqlite: dbfab1f7d79984b6acbe1b80602d9a0236c68eb6 + op-sqlite: 18d137327dfbb3a1322d9998ac8de140796470ce RCT-Folly: bf5c0376ffe4dd2cf438dcf86db385df9fdce648 RCTDeprecation: fde92935b3caa6cb65cbff9fbb7d3a9867ffb259 RCTRequired: 75c6cee42d21c1530a6f204ba32ff57335d19007 diff --git a/example/package.json b/example/package.json index b52055d0..b9a45802 100644 --- a/example/package.json +++ b/example/package.json @@ -75,7 +75,8 @@ "libsql": false, "sqliteVec": true, "tokenizers": [ - "ngram" + "ngram", + "edgengram" ] } } diff --git a/op-sqlite.podspec b/op-sqlite.podspec index 41efe62c..b7aa9aa3 100644 --- a/op-sqlite.podspec +++ b/op-sqlite.podspec @@ -185,6 +185,10 @@ Pod::Spec.new do |s| xcconfig[:OTHER_CFLAGS] += " #{sqlite_flags}" end + if tokenizers.any? then + xcconfig[:OTHER_CFLAGS] += " -DTOKENIZER_LIST=\\\"#{tokenizers.join(",")}\\\"" + end + s.pod_target_xcconfig = xcconfig s.vendored_frameworks = frameworks end From d5efc759aa84ed15294ea5ede6b2a71268ba304d Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Sat, 2 Nov 2024 19:31:28 +0100 Subject: [PATCH 04/43] Correct tokenizers list variable --- cpp/bindings.cpp | 6 +++--- example/ios/Podfile.lock | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cpp/bindings.cpp b/cpp/bindings.cpp index c28107c9..7b218499 100644 --- a/cpp/bindings.cpp +++ b/cpp/bindings.cpp @@ -16,8 +16,8 @@ #include #include "../example/c_sources/tokenizers.h" -#ifndef STRING_LIST -#define STRING_LIST "" +#ifndef TOKENIZER_LIST +#define TOKENIZER_LIST "" #endif namespace opsqlite { @@ -61,7 +61,7 @@ void install(jsi::Runtime &rt, const std::shared_ptr& invoke auto answer = HOSTFN("answer") { std::cout << "List of tokenizers" << std::endl; - std::cout << STRING_LIST << std::endl; + std::cout << TOKENIZER_LIST << std::endl; ngram(); edgengram(); return 52; diff --git a/example/ios/Podfile.lock b/example/ios/Podfile.lock index 58a01186..0393f1ed 100644 --- a/example/ios/Podfile.lock +++ b/example/ios/Podfile.lock @@ -1779,14 +1779,14 @@ EXTERNAL SOURCES: SPEC CHECKSUMS: boost: 1dca942403ed9342f98334bf4c3621f011aa7946 - DoubleConversion: f16ae600a246532c4020132d54af21d0ddb2a385 + DoubleConversion: 76ab83afb40bddeeee456813d9c04f67f78771b5 FBLazyVector: 7075bb12898bc3998fd60f4b7ca422496cc2cdf7 - fmt: 10c6e61f4be25dc963c36bd73fc7b1705fe975be + fmt: 4c2741a687cc09f0634a2e2c72a838b99f1ff120 GCDWebServer: 2c156a56c8226e2d5c0c3f208a3621ccffbe3ce4 - glog: 08b301085f15bcbb6ff8632a8ebaf239aae04e6a + glog: c5d68082e772fa1c511173d6b30a9de2c05a69a2 hermes-engine: 46f1ffbf0297f4298862068dd4c274d4ac17a1fd op-sqlite: 18d137327dfbb3a1322d9998ac8de140796470ce - RCT-Folly: bf5c0376ffe4dd2cf438dcf86db385df9fdce648 + RCT-Folly: 045d6ecaa59d826c5736dfba0b2f4083ff8d79df RCTDeprecation: fde92935b3caa6cb65cbff9fbb7d3a9867ffb259 RCTRequired: 75c6cee42d21c1530a6f204ba32ff57335d19007 RCTTypeSafety: 7e6fe47bfb693c50d4669db1a480ca5331795f5b From e648e74af94b43d59ed0f84c8e757ea7a3f8778f Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Sun, 3 Nov 2024 10:01:37 +0100 Subject: [PATCH 05/43] Tokenizer init works --- cpp/bindings.cpp | 21 +++----- cpp/bridge.cpp | 9 +++- cpp/utils.cpp | 10 ++++ cpp/utils.h | 10 ++++ example/c_sources/tokenizers.cpp | 86 ++++++++++++++++++++++++++++-- example/c_sources/tokenizers.h | 5 +- example/ios/Podfile.lock | 2 +- example/package.json | 3 +- generate_tokenizers_header_file.rb | 10 +++- op-sqlite.podspec | 14 ++--- 10 files changed, 138 insertions(+), 32 deletions(-) diff --git a/cpp/bindings.cpp b/cpp/bindings.cpp index 7b218499..730fe9ee 100644 --- a/cpp/bindings.cpp +++ b/cpp/bindings.cpp @@ -14,11 +14,7 @@ #include #include #include -#include "../example/c_sources/tokenizers.h" -#ifndef TOKENIZER_LIST -#define TOKENIZER_LIST "" -#endif namespace opsqlite { @@ -51,21 +47,18 @@ void clearState() { thread_pool->restartPool(); } -void install(jsi::Runtime &rt, const std::shared_ptr& invoker, +void install(jsi::Runtime &rt, + const std::shared_ptr &invoker, const char *base_path, const char *crsqlite_path, const char *sqlite_vec_path) { invalidated = false; _base_path = std::string(base_path); _crsqlite_path = std::string(crsqlite_path); _sqlite_vec_path = std::string(sqlite_vec_path); - - auto answer = HOSTFN("answer") { - std::cout << "List of tokenizers" << std::endl; - std::cout << TOKENIZER_LIST << std::endl; - ngram(); - edgengram(); - return 52; - }); + + auto answer = HOSTFN("answer") { + return 52; + }); auto open = HOSTFN("open") { jsi::Object options = args[0].asObject(rt); @@ -173,7 +166,7 @@ void install(jsi::Runtime &rt, const std::shared_ptr& invoke module.setProperty(rt, "open", std::move(open)); module.setProperty(rt, "isSQLCipher", std::move(is_sqlcipher)); module.setProperty(rt, "isLibsql", std::move(is_libsql)); - module.setProperty(rt, "answer", std::move(answer)); + module.setProperty(rt, "answer", std::move(answer)); #ifdef OP_SQLITE_USE_LIBSQL module.setProperty(rt, "openRemote", std::move(open_remote)); module.setProperty(rt, "openSync", std::move(open_sync)); diff --git a/cpp/bridge.cpp b/cpp/bridge.cpp index 65e07efc..6645eaa8 100644 --- a/cpp/bridge.cpp +++ b/cpp/bridge.cpp @@ -6,6 +6,12 @@ #include #include #include +#include +#include "../example/c_sources/tokenizers.h" + +#ifndef TOKENIZER_LIST +#define TOKENIZER_LIST "" +#endif namespace opsqlite { @@ -109,9 +115,10 @@ BridgeResult opsqlite_open(std::string const &name, if (errMsg != nullptr) { return {.type = SQLiteError, .message = errMsg}; } - #endif + TOKENIZER_LIST + return {.type = SQLiteOk, .affectedRows = 0}; } diff --git a/cpp/utils.cpp b/cpp/utils.cpp index c1ee3e71..79f0ee80 100644 --- a/cpp/utils.cpp +++ b/cpp/utils.cpp @@ -341,4 +341,14 @@ int mkdir(std::string const &path) { return 0; } +std::vector parse_string_list(const std::string& str) { + std::vector result; + std::istringstream stream(str); + std::string token; + while (std::getline(stream, token, ',')) { + result.push_back(token); + } + return result; +} + } // namespace opsqlite diff --git a/cpp/utils.h b/cpp/utils.h index 2950d9fb..cd2383b5 100644 --- a/cpp/utils.h +++ b/cpp/utils.h @@ -9,20 +9,28 @@ #include #include #include +#include namespace opsqlite { namespace jsi = facebook::jsi; jsi::Value toJSI(jsi::Runtime &rt, const JSVariant &value); + JSVariant toVariant(jsi::Runtime &rt, jsi::Value const &value); + std::vector to_string_vec(jsi::Runtime &rt, jsi::Value const &xs); + std::vector to_variant_vec(jsi::Runtime &rt, jsi::Value const &xs); + std::vector to_int_vec(jsi::Runtime &rt, jsi::Value const &xs); + jsi::Value createResult(jsi::Runtime &rt, BridgeResult status, std::vector *results, std::shared_ptr> metadata); + jsi::Value create_js_rows(jsi::Runtime &rt, const BridgeResult &status); + jsi::Value create_raw_result(jsi::Runtime &rt, BridgeResult status, const std::vector> *results); @@ -38,6 +46,8 @@ bool folder_exists(const std::string &foldername); bool file_exists(const std::string &path); +std::vector parse_string_list(const std::string& str); + } // namespace opsqlite #endif /* utils_h */ diff --git a/example/c_sources/tokenizers.cpp b/example/c_sources/tokenizers.cpp index d2a6a709..27713feb 100644 --- a/example/c_sources/tokenizers.cpp +++ b/example/c_sources/tokenizers.cpp @@ -1,7 +1,87 @@ #include "tokenizers.h" +#include +#include +#include #include +#include "../../cpp/sqlite3.h" + +//SQLITE_EXTENSION_INIT1 namespace opsqlite { -void ngram() { std::cout << "ngram" << std::endl; } -void edgengram() { std::cout << "edgengram" << std::endl; } -} // namespace opsqlite \ No newline at end of file + +fts5_api *fts5_api_from_db(sqlite3 *db){ + fts5_api *pRet = 0; + sqlite3_stmt *pStmt = 0; + + if( SQLITE_OK == sqlite3_prepare_v2(db, "SELECT fts5(?1)", -1, &pStmt, 0) ){ + sqlite3_bind_pointer(pStmt, 1, (void*)&pRet, "fts5_api_ptr", NULL); + sqlite3_step(pStmt); + } + sqlite3_finalize(pStmt); + return pRet; +} + +class WordTokenizer { +public: + WordTokenizer() = default; + ~WordTokenizer() = default; +}; + +// Define `xCreate`, which initializes the tokenizer +int wordTokenizerCreate(void *pUnused, const char **azArg, int nArg, + Fts5Tokenizer **ppOut) { + auto tokenizer = std::make_unique(); + *ppOut = reinterpret_cast( + tokenizer.release()); // Cast to Fts5Tokenizer* + return SQLITE_OK; +} + +// Define `xDelete`, which frees the tokenizer +void wordTokenizerDelete(Fts5Tokenizer *pTokenizer) { + delete reinterpret_cast(pTokenizer); +} + +// Define `xTokenize`, which performs the actual tokenization +int wordTokenizerTokenize(Fts5Tokenizer *pTokenizer, void *pCtx, int flags, + const char *pText, int nText, + int (*xToken)(void *, int, const char *, int, int, + int)) { + int start = 0; + int i = 0; + + while (i <= nText) { + if (i == nText || !std::isalnum(static_cast(pText[i]))) { + if (start < i) { // Found a token + int rc = xToken(pCtx, 0, pText + start, i - start, start, i); + if (rc != SQLITE_OK) + return rc; + } + start = i + 1; + } + i++; + } + return SQLITE_OK; +} + +// int sqlite_ngram_init(sqlite3 *db, char **error, +// const sqlite3_api_routines *api) {} +// int sqlite_edgengram_init(sqlite3 *db, char **error, +// const sqlite3_api_routines *api) {} +int sqlite_wordtokenizer_init(sqlite3 *db, char **error, + const sqlite3_api_routines *api) { +// SQLITE_EXTENSION_INIT2(api); + + // Create the FTS5 tokenizer structure + fts5_tokenizer wordtokenizer = {wordTokenizerCreate, wordTokenizerDelete, + wordTokenizerTokenize}; + + // Register the tokenizer with FTS5 + fts5_api *ftsApi = (fts5_api *)fts5_api_from_db(db); + if (ftsApi == NULL) + return SQLITE_ERROR; + + return ftsApi->xCreateTokenizer(ftsApi, "wordtokenizer", NULL, &wordtokenizer, + NULL); +} + +} // namespace opsqlite diff --git a/example/c_sources/tokenizers.h b/example/c_sources/tokenizers.h index 2529e919..8ce51179 100644 --- a/example/c_sources/tokenizers.h +++ b/example/c_sources/tokenizers.h @@ -1,10 +1,11 @@ #ifndef TOKENIZERS_H #define TOKENIZERS_H +#include "../../cpp/sqlite3.h" + namespace opsqlite { -void ngram(); -void edgengram(); +int sqlite_wordtokenizer_init(sqlite3 *db, char **error, const sqlite3_api_routines *api); } // namespace opsqlite diff --git a/example/ios/Podfile.lock b/example/ios/Podfile.lock index 0393f1ed..661be8df 100644 --- a/example/ios/Podfile.lock +++ b/example/ios/Podfile.lock @@ -1785,7 +1785,7 @@ SPEC CHECKSUMS: GCDWebServer: 2c156a56c8226e2d5c0c3f208a3621ccffbe3ce4 glog: c5d68082e772fa1c511173d6b30a9de2c05a69a2 hermes-engine: 46f1ffbf0297f4298862068dd4c274d4ac17a1fd - op-sqlite: 18d137327dfbb3a1322d9998ac8de140796470ce + op-sqlite: b59c730cda63c0017c14b1b2dd6499719f50433b RCT-Folly: 045d6ecaa59d826c5736dfba0b2f4083ff8d79df RCTDeprecation: fde92935b3caa6cb65cbff9fbb7d3a9867ffb259 RCTRequired: 75c6cee42d21c1530a6f204ba32ff57335d19007 diff --git a/example/package.json b/example/package.json index b9a45802..e55ef58a 100644 --- a/example/package.json +++ b/example/package.json @@ -75,8 +75,7 @@ "libsql": false, "sqliteVec": true, "tokenizers": [ - "ngram", - "edgengram" + "wordtokenizer" ] } } diff --git a/generate_tokenizers_header_file.rb b/generate_tokenizers_header_file.rb index 8e1a3e3d..d5f81611 100644 --- a/generate_tokenizers_header_file.rb +++ b/generate_tokenizers_header_file.rb @@ -1,6 +1,6 @@ require 'fileutils' -def generate_tokenizers_header_file(names, file_path) +def generate_tokenizers_header_file(names, file_path, is_user_app) # Ensure the directory exists dir_path = File.dirname(file_path) FileUtils.mkdir_p(dir_path) unless Dir.exist?(dir_path) @@ -9,11 +9,17 @@ def generate_tokenizers_header_file(names, file_path) file.puts "#ifndef TOKENIZERS_H" file.puts "#define TOKENIZERS_H" file.puts + if is_user_app + file.puts "#include \"node-modules/@op-engineering/op-sqlite/cpp/sqlite3ext.h\"" + else + file.puts "#include \"../../cpp/sqlite3.h\"" + end + file.puts file.puts "namespace opsqlite {" file.puts names.each do |name| - file.puts "void #{name}();" + file.puts "int sqlite_#{name}_init(sqlite3 *db, char **error, const sqlite3_api_routines *api);" end file.puts diff --git a/op-sqlite.podspec b/op-sqlite.podspec index b7aa9aa3..c96b129c 100644 --- a/op-sqlite.podspec +++ b/op-sqlite.podspec @@ -5,6 +5,7 @@ log_message = lambda do |message| puts "\e[34m#{message}\e[0m" end +is_user_app = __dir__.include?("node_modules") package = JSON.parse(File.read(File.join(__dir__, "package.json"))) folly_compiler_flags = '-DFOLLY_NO_CONFIG -DFOLLY_MOBILE=1 -DFOLLY_USE_LIBCPP=1 -Wno-comma -Wno-shorten-64-to-32' fabric_enabled = ENV['RCT_NEW_ARCH_ENABLED'] == '1' @@ -12,7 +13,7 @@ fabric_enabled = ENV['RCT_NEW_ARCH_ENABLED'] == '1' parent_folder_name = File.basename(__dir__) app_package = nil # When installed on user node_modules lives inside node_modules/@op-engineering/op-sqlite -if __dir__.include?("node_modules") +if is_user_app app_package = JSON.parse(File.read(File.join(__dir__, "..", "..", "..", "package.json"))) # When running on the example app else @@ -80,17 +81,15 @@ Pod::Spec.new do |s| # Base source files source_files = Dir.glob("ios/**/*.{h,m,mm}") + Dir.glob("cpp/**/*.{h,cpp,c}") - - - # # Set the path to the `c_sources` directory based on environment - if __dir__.include?("node_modules") + # Set the path to the `c_sources` directory based on environment + if is_user_app c_sources_dir = File.join("..", "..", "..", "c_sources") else c_sources_dir = File.join("example", "c_sources") end if tokenizers.any? - generate_tokenizers_header_file(tokenizers, File.join(c_sources_dir, "tokenizers.h")) + generate_tokenizers_header_file(tokenizers, File.join(c_sources_dir, "tokenizers.h"), is_user_app) end # Add all .h and .c files from the `c_sources` directory @@ -186,7 +185,8 @@ Pod::Spec.new do |s| end if tokenizers.any? then - xcconfig[:OTHER_CFLAGS] += " -DTOKENIZER_LIST=\\\"#{tokenizers.join(",")}\\\"" + tokenizer_inits = tokenizers.map { |tokenizer| "sqlite_#{tokenizer}_init(db,&errMsg,nullptr);" } + xcconfig[:OTHER_CFLAGS] += " -DTOKENIZER_LIST=\"#{tokenizer_inits.join(" ")}\"" end s.pod_target_xcconfig = xcconfig From 1ccfe32583dcaf6b5cda132c27eaf657af0ed87d Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Sun, 3 Nov 2024 13:23:51 +0100 Subject: [PATCH 06/43] Working tokenizer plus test --- cpp/bindings.cpp | 6 ---- example/c_sources/tokenizers.cpp | 25 ++++++++-------- example/c_sources/tokenizers.h | 3 +- example/ios/Podfile.lock | 2 +- example/package.json | 3 +- example/src/App.tsx | 15 +++++----- example/src/tests/tokenizer.spec.ts | 44 +++++++++++++++++++++++++++++ generate_tokenizers_header_file.rb | 8 ++---- op-sqlite.podspec | 2 +- src/index.ts | 5 ---- 10 files changed, 71 insertions(+), 42 deletions(-) create mode 100644 example/src/tests/tokenizer.spec.ts diff --git a/cpp/bindings.cpp b/cpp/bindings.cpp index 730fe9ee..168998e0 100644 --- a/cpp/bindings.cpp +++ b/cpp/bindings.cpp @@ -15,7 +15,6 @@ #include #include - namespace opsqlite { namespace jsi = facebook::jsi; @@ -56,10 +55,6 @@ void install(jsi::Runtime &rt, _crsqlite_path = std::string(crsqlite_path); _sqlite_vec_path = std::string(sqlite_vec_path); - auto answer = HOSTFN("answer") { - return 52; - }); - auto open = HOSTFN("open") { jsi::Object options = args[0].asObject(rt); std::string name = options.getProperty(rt, "name").asString(rt).utf8(rt); @@ -166,7 +161,6 @@ void install(jsi::Runtime &rt, module.setProperty(rt, "open", std::move(open)); module.setProperty(rt, "isSQLCipher", std::move(is_sqlcipher)); module.setProperty(rt, "isLibsql", std::move(is_libsql)); - module.setProperty(rt, "answer", std::move(answer)); #ifdef OP_SQLITE_USE_LIBSQL module.setProperty(rt, "openRemote", std::move(open_remote)); module.setProperty(rt, "openSync", std::move(open_sync)); diff --git a/example/c_sources/tokenizers.cpp b/example/c_sources/tokenizers.cpp index 27713feb..1a720642 100644 --- a/example/c_sources/tokenizers.cpp +++ b/example/c_sources/tokenizers.cpp @@ -2,10 +2,6 @@ #include #include #include -#include -#include "../../cpp/sqlite3.h" - -//SQLITE_EXTENSION_INIT1 namespace opsqlite { @@ -63,19 +59,11 @@ int wordTokenizerTokenize(Fts5Tokenizer *pTokenizer, void *pCtx, int flags, return SQLITE_OK; } -// int sqlite_ngram_init(sqlite3 *db, char **error, -// const sqlite3_api_routines *api) {} -// int sqlite_edgengram_init(sqlite3 *db, char **error, -// const sqlite3_api_routines *api) {} -int sqlite_wordtokenizer_init(sqlite3 *db, char **error, +int opsqlite_wordtokenizer_init(sqlite3 *db, char **error, const sqlite3_api_routines *api) { -// SQLITE_EXTENSION_INIT2(api); - - // Create the FTS5 tokenizer structure fts5_tokenizer wordtokenizer = {wordTokenizerCreate, wordTokenizerDelete, wordTokenizerTokenize}; - // Register the tokenizer with FTS5 fts5_api *ftsApi = (fts5_api *)fts5_api_from_db(db); if (ftsApi == NULL) return SQLITE_ERROR; @@ -84,4 +72,15 @@ int sqlite_wordtokenizer_init(sqlite3 *db, char **error, NULL); } + +int opsqlite_porter_init(sqlite3 *db, char **error, const sqlite3_api_routines *api) { + fts5_tokenizer porter_tokenizer = {wordTokenizerCreate, wordTokenizerDelete, wordTokenizerTokenize}; + + fts5_api *ftsApi = (fts5_api *)fts5_api_from_db(db); + if (ftsApi == nullptr) + return SQLITE_ERROR; + + return ftsApi->xCreateTokenizer(ftsApi, "portertokenizer", NULL, &porter_tokenizer, NULL); +} + } // namespace opsqlite diff --git a/example/c_sources/tokenizers.h b/example/c_sources/tokenizers.h index 8ce51179..2487edc3 100644 --- a/example/c_sources/tokenizers.h +++ b/example/c_sources/tokenizers.h @@ -5,7 +5,8 @@ namespace opsqlite { -int sqlite_wordtokenizer_init(sqlite3 *db, char **error, const sqlite3_api_routines *api); +int opsqlite_wordtokenizer_init(sqlite3 *db, char **error, const sqlite3_api_routines *api); +int opsqlite_porter_init(sqlite3 *db, char **error, const sqlite3_api_routines *api); } // namespace opsqlite diff --git a/example/ios/Podfile.lock b/example/ios/Podfile.lock index 661be8df..2faf0c0e 100644 --- a/example/ios/Podfile.lock +++ b/example/ios/Podfile.lock @@ -1785,7 +1785,7 @@ SPEC CHECKSUMS: GCDWebServer: 2c156a56c8226e2d5c0c3f208a3621ccffbe3ce4 glog: c5d68082e772fa1c511173d6b30a9de2c05a69a2 hermes-engine: 46f1ffbf0297f4298862068dd4c274d4ac17a1fd - op-sqlite: b59c730cda63c0017c14b1b2dd6499719f50433b + op-sqlite: 47d86aff0fcabfe76845066dd4aba17041ae14a0 RCT-Folly: 045d6ecaa59d826c5736dfba0b2f4083ff8d79df RCTDeprecation: fde92935b3caa6cb65cbff9fbb7d3a9867ffb259 RCTRequired: 75c6cee42d21c1530a6f204ba32ff57335d19007 diff --git a/example/package.json b/example/package.json index e55ef58a..9017a3e7 100644 --- a/example/package.json +++ b/example/package.json @@ -75,7 +75,8 @@ "libsql": false, "sqliteVec": true, "tokenizers": [ - "wordtokenizer" + "wordtokenizer", + "porter" ] } } diff --git a/example/src/App.tsx b/example/src/App.tsx index 5d827f09..80d1022b 100644 --- a/example/src/App.tsx +++ b/example/src/App.tsx @@ -1,7 +1,7 @@ +import {open} from '@op-engineering/op-sqlite'; import clsx from 'clsx'; import {useEffect, useState} from 'react'; import { - Alert, Button, Clipboard, SafeAreaView, @@ -9,17 +9,17 @@ import { Text, View, } from 'react-native'; +import RNRestart from 'react-native-restart'; +import Share from 'react-native-share'; import 'reflect-metadata'; +import {createLargeDB, queryLargeDB} from './Database'; +import {setServerResults, startServer, stopServer} from './server'; import {constantsTests} from './tests/constants.spec'; import {registerHooksTests} from './tests/hooks.spec'; import {blobTests, dbSetupTests, queriesTests, runTests} from './tests/index'; import {preparedStatementsTests} from './tests/preparedStatements.spec'; import {reactiveTests} from './tests/reactive.spec'; -import {setServerResults, startServer, stopServer} from './server'; -import {answer, open} from '@op-engineering/op-sqlite'; -import Share from 'react-native-share'; -import {createLargeDB, queryLargeDB} from './Database'; -import RNRestart from 'react-native-restart'; +import {tokenizerTests} from './tests/tokenizer.spec'; export default function App() { const [times, setTimes] = useState([]); @@ -39,6 +39,7 @@ export default function App() { preparedStatementsTests, constantsTests, reactiveTests, + tokenizerTests, ).then(results => { setServerResults(results as any); setResults(results); @@ -46,8 +47,6 @@ export default function App() { startServer(); - Alert.alert('Custom C file answer: ' + answer()); - return () => { stopServer(); }; diff --git a/example/src/tests/tokenizer.spec.ts b/example/src/tests/tokenizer.spec.ts new file mode 100644 index 00000000..8dc8f40e --- /dev/null +++ b/example/src/tests/tokenizer.spec.ts @@ -0,0 +1,44 @@ +import {open, type DB} from '@op-engineering/op-sqlite'; +import chai from 'chai'; +import {afterEach, beforeEach, describe, it, itOnly} from './MochaRNAdapter'; + +const expect = chai.expect; + +export function tokenizerTests() { + let db: DB; + + describe('Tokenizer tests', () => { + beforeEach(async () => { + db = open({ + name: 'tokenizers.sqlite', + encryptionKey: 'test', + }); + + await db.execute( + `CREATE VIRTUAL TABLE tokenizer_table USING fts5(content, tokenize = 'wordtokenizer');`, + ); + }); + + afterEach(() => { + if (db) { + db.close(); + db.delete(); + // @ts-ignore + db = null; + } + }); + + itOnly('Should match the word split by the tokenizer', async () => { + await db.execute('INSERT INTO tokenizer_table(content) VALUES (?)', [ + 'This is a test document', + ]); + const res = await db.execute( + 'SELECT content FROM tokenizer_table WHERE content MATCH ?', + ['test'], + ); + console.warn(res); + expect(res.rows.length).to.be.equal(1); + expect(res.rows[0]!.content).to.be.equal('This is a test document'); + }); + }); +} diff --git a/generate_tokenizers_header_file.rb b/generate_tokenizers_header_file.rb index d5f81611..5d01e6fb 100644 --- a/generate_tokenizers_header_file.rb +++ b/generate_tokenizers_header_file.rb @@ -19,7 +19,7 @@ def generate_tokenizers_header_file(names, file_path, is_user_app) file.puts names.each do |name| - file.puts "int sqlite_#{name}_init(sqlite3 *db, char **error, const sqlite3_api_routines *api);" + file.puts "int opsqlite_#{name}_init(sqlite3 *db, char **error, const sqlite3_api_routines *api);" end file.puts @@ -27,8 +27,4 @@ def generate_tokenizers_header_file(names, file_path, is_user_app) file.puts file.puts "#endif // TOKENIZERS_H" end -end - -# # Example usage: -# names = ["functionOne", "functionTwo", "functionThree"] -# generate_header_file(names, "/path/to/generated_header.h") \ No newline at end of file +end \ No newline at end of file diff --git a/op-sqlite.podspec b/op-sqlite.podspec index c96b129c..52185ebe 100644 --- a/op-sqlite.podspec +++ b/op-sqlite.podspec @@ -185,7 +185,7 @@ Pod::Spec.new do |s| end if tokenizers.any? then - tokenizer_inits = tokenizers.map { |tokenizer| "sqlite_#{tokenizer}_init(db,&errMsg,nullptr);" } + tokenizer_inits = tokenizers.map { |tokenizer| "opsqlite_#{tokenizer}_init(db,&errMsg,nullptr);" } xcconfig[:OTHER_CFLAGS] += " -DTOKENIZER_LIST=\"#{tokenizer_inits.join(" ")}\"" end diff --git a/src/index.ts b/src/index.ts index 25dac0ee..01963067 100644 --- a/src/index.ts +++ b/src/index.ts @@ -197,7 +197,6 @@ type OPSQLiteProxy = { }) => DB; isSQLCipher: () => boolean; isLibsql: () => boolean; - answer: () => number; }; const locks: Record< @@ -460,7 +459,3 @@ export const isSQLCipher = (): boolean => { export const isLibsql = (): boolean => { return OPSQLite.isLibsql(); }; - -export const answer = (): number => { - return OPSQLite.answer(); -}; From 59c03ae6f5afb709e6c2c3b01c5ab2b58b1ca932 Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Sun, 3 Nov 2024 13:34:35 +0100 Subject: [PATCH 07/43] Working include of dynamically generated header --- cpp/bridge.cpp | 3 ++- example/ios/Podfile.lock | 2 +- op-sqlite.podspec | 5 +++++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/cpp/bridge.cpp b/cpp/bridge.cpp index 6645eaa8..8b402985 100644 --- a/cpp/bridge.cpp +++ b/cpp/bridge.cpp @@ -7,10 +7,11 @@ #include #include #include -#include "../example/c_sources/tokenizers.h" #ifndef TOKENIZER_LIST #define TOKENIZER_LIST "" +#else +#include TOKENIZERS_HEADER_PATH #endif namespace opsqlite { diff --git a/example/ios/Podfile.lock b/example/ios/Podfile.lock index 2faf0c0e..a53e6801 100644 --- a/example/ios/Podfile.lock +++ b/example/ios/Podfile.lock @@ -1785,7 +1785,7 @@ SPEC CHECKSUMS: GCDWebServer: 2c156a56c8226e2d5c0c3f208a3621ccffbe3ce4 glog: c5d68082e772fa1c511173d6b30a9de2c05a69a2 hermes-engine: 46f1ffbf0297f4298862068dd4c274d4ac17a1fd - op-sqlite: 47d86aff0fcabfe76845066dd4aba17041ae14a0 + op-sqlite: 2456fe4d6381cc08540795d5b08047e250a0303b RCT-Folly: 045d6ecaa59d826c5736dfba0b2f4083ff8d79df RCTDeprecation: fde92935b3caa6cb65cbff9fbb7d3a9867ffb259 RCTRequired: 75c6cee42d21c1530a6f204ba32ff57335d19007 diff --git a/op-sqlite.podspec b/op-sqlite.podspec index 52185ebe..7b6aba21 100644 --- a/op-sqlite.podspec +++ b/op-sqlite.podspec @@ -187,6 +187,11 @@ Pod::Spec.new do |s| if tokenizers.any? then tokenizer_inits = tokenizers.map { |tokenizer| "opsqlite_#{tokenizer}_init(db,&errMsg,nullptr);" } xcconfig[:OTHER_CFLAGS] += " -DTOKENIZER_LIST=\"#{tokenizer_inits.join(" ")}\"" + if is_user_app then + xcconfig[:OTHER_CFLAGS] += " -DTOKENIZERS_HEADER_PATH=\\\"../../../c_sources/tokenizers.h\\\"" + else + xcconfig[:OTHER_CFLAGS] += " -DTOKENIZERS_HEADER_PATH=\\\"../example/c_sources/tokenizers.h\\\"" + end end s.pod_target_xcconfig = xcconfig From 66c350a25d037fbb5040b5c8e391f2945f000f6c Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Mon, 4 Nov 2024 12:21:08 +0100 Subject: [PATCH 08/43] Partial Android implementation --- android/CMakeLists.txt | 13 ++++++++++ android/build.gradle | 38 ++++++++++++++++++++++++++---- cpp/bridge.cpp | 8 +++---- example/c_sources/tokenizers.h | 2 ++ example/ios/Podfile.lock | 2 +- generate_tokenizers_header_file.rb | 3 +++ op-sqlite.podspec | 4 ++-- 7 files changed, 59 insertions(+), 11 deletions(-) diff --git a/android/CMakeLists.txt b/android/CMakeLists.txt index 5ff0655d..3bae5590 100644 --- a/android/CMakeLists.txt +++ b/android/CMakeLists.txt @@ -72,6 +72,19 @@ find_package(ReactAndroid REQUIRED CONFIG) find_package(fbjni REQUIRED CONFIG) find_library(LOG_LIB log) +# Add user defined files +if (USER_DEFINED_SOURCE_FILES) + string(REPLACE ";" " " user_source_files "${USER_DEFINED_SOURCE_FILES}") + message(WARNING "User source files: ${user_source_files}") + message(WARNING "INITLIST ${USER_DEFINED_TOKENIZER_INIT_STRINGS}") + message(WARNING "TOKENIZERS_HEADER_PATH ${USER_DEFINED_TOKENIZERS_HEADER_PATH}") + + target_sources(${PACKAGE_NAME} PRIVATE ${source_files}) +# add_definitions("-DTOKENIZER_LIST=${USER_DEFINED_TOKENIZER_INIT_STRINGS}") + + add_definitions("-DTOKENIZERS_HEADER_PATH=\"${USER_DEFINED_TOKENIZERS_HEADER_PATH}\"") +endif() + if (USE_SQLCIPHER) if (ReactAndroid_VERSION_MINOR GREATER_EQUAL 76) target_link_libraries( diff --git a/android/build.gradle b/android/build.gradle index 4d5b8338..f79708c9 100644 --- a/android/build.gradle +++ b/android/build.gradle @@ -35,9 +35,18 @@ def sqliteFlags = "" def enableFTS5 = false def useSqliteVec = false def enableRtree = false +def tokenizers = [] -def packageJsonFile = new File("$rootDir/../package.json") -def packageJson = new JsonSlurper().parseText(packageJsonFile.text) +def isInsideNodeModules = rootDir.absolutePath.contains("node_modules") +def packageJson + +if ( isInsideNodeModules ) { + def packageJsonFile = new File("$rootDir/../../../package.json") + packageJson = new JsonSlurper().parseText(packageJsonFile.text) +} else { + def packageJsonFile = new File("$rootDir/../package.json") + packageJson = new JsonSlurper().parseText(packageJsonFile.text) +} def opsqliteConfig = packageJson["op-sqlite"] if(opsqliteConfig) { @@ -49,6 +58,7 @@ if(opsqliteConfig) { enableFTS5 = opsqliteConfig["fts5"] useLibsql = opsqliteConfig["libsql"] enableRtree = opsqliteConfig["rtree"] + tokenizers = opsqliteConfig["tokenizers"] ? opsqliteConfig["tokenizers"] : [] } if(useSQLCipher) { @@ -153,14 +163,34 @@ android { cppFlags += "-DOP_SQLITE_USE_SQLITE_VEC=1" } - cppFlags "-O2", "-fexceptions", "-frtti", "-std=c++1y", "-DONANDROID" + // This are zeroes because they will be passed as C flags, so they become falsy + def sourceFiles = 0 + // def tokenizerInitStrings = 0 + def tokenizersHeaderPath = 0 + if (!tokenizers.isEmpty()) { + // Scan user directories for .h and .cpp files + def sourceDir = isInsideNodeModules ? file("$rootDir/../../../c_sources") : file("$rootDir/../c_sources") + sourceFiles = fileTree(dir: sourceDir, include: ["**/*.cpp", "**/*.h"]) + .files + .join(";") // Join paths with `;` for CMake + // tokenizerInitStrings = tokenizers.collect { tokenizer -> + // "opsqlite_${tokenizer}_init(db,&errMsg,nullptr);" + // }.join("") + tokenizersHeaderPath = isInsideNodeModules ? "../../../c_sources/tokenizers.h" : "../example/c_sources/tokenizers.h" + println("tokenizers header path $tokenizersHeaderPath") + } + + cppFlags "-O2", "-fexceptions", "-DONANDROID" abiFilters 'x86', 'x86_64', 'armeabi-v7a', 'arm64-v8a' arguments "-DANDROID_STL=c++_shared", "-DSQLITE_FLAGS='$sqliteFlags'", "-DUSE_SQLCIPHER=${useSQLCipher ? 1 : 0}", "-DUSE_CRSQLITE=${useCRSQLite ? 1 : 0}", "-DUSE_LIBSQL=${useLibsql ? 1 : 0}", - "-DUSE_SQLITE_VEC=${useSqliteVec ? 1 : 0}" + "-DUSE_SQLITE_VEC=${useSqliteVec ? 1 : 0}", + "-DUSER_DEFINED_SOURCE_FILES=${sourceFiles}", + // "-DUSER_DEFINED_TOKENIZER_INIT_STRINGS='${tokenizerInitStrings}'", + "-DUSER_DEFINED_TOKENIZERS_HEADER_PATH='${tokenizersHeaderPath}'" } } diff --git a/cpp/bridge.cpp b/cpp/bridge.cpp index 8b402985..48933996 100644 --- a/cpp/bridge.cpp +++ b/cpp/bridge.cpp @@ -8,10 +8,10 @@ #include #include -#ifndef TOKENIZER_LIST -#define TOKENIZER_LIST "" -#else +#ifdef TOKENIZERS_HEADER_PATH #include TOKENIZERS_HEADER_PATH +#else +#define TOKENIZERS_LIST #endif namespace opsqlite { @@ -118,7 +118,7 @@ BridgeResult opsqlite_open(std::string const &name, } #endif - TOKENIZER_LIST + TOKENIZER_LIST return {.type = SQLiteOk, .affectedRows = 0}; } diff --git a/example/c_sources/tokenizers.h b/example/c_sources/tokenizers.h index 2487edc3..3c5137ab 100644 --- a/example/c_sources/tokenizers.h +++ b/example/c_sources/tokenizers.h @@ -1,6 +1,8 @@ #ifndef TOKENIZERS_H #define TOKENIZERS_H +#define TOKENIZER_LIST opsqlite_wordtokenizer_init(db,&errMsg,nullptr);opsqlite_porter_init(db,&errMsg,nullptr); + #include "../../cpp/sqlite3.h" namespace opsqlite { diff --git a/example/ios/Podfile.lock b/example/ios/Podfile.lock index a53e6801..d47e5411 100644 --- a/example/ios/Podfile.lock +++ b/example/ios/Podfile.lock @@ -1785,7 +1785,7 @@ SPEC CHECKSUMS: GCDWebServer: 2c156a56c8226e2d5c0c3f208a3621ccffbe3ce4 glog: c5d68082e772fa1c511173d6b30a9de2c05a69a2 hermes-engine: 46f1ffbf0297f4298862068dd4c274d4ac17a1fd - op-sqlite: 2456fe4d6381cc08540795d5b08047e250a0303b + op-sqlite: 9a4e3e88a0f5d1aecce6ffe9e1f4b0125b6ab003 RCT-Folly: 045d6ecaa59d826c5736dfba0b2f4083ff8d79df RCTDeprecation: fde92935b3caa6cb65cbff9fbb7d3a9867ffb259 RCTRequired: 75c6cee42d21c1530a6f204ba32ff57335d19007 diff --git a/generate_tokenizers_header_file.rb b/generate_tokenizers_header_file.rb index 5d01e6fb..c7e2dda1 100644 --- a/generate_tokenizers_header_file.rb +++ b/generate_tokenizers_header_file.rb @@ -4,11 +4,14 @@ def generate_tokenizers_header_file(names, file_path, is_user_app) # Ensure the directory exists dir_path = File.dirname(file_path) FileUtils.mkdir_p(dir_path) unless Dir.exist?(dir_path) + tokenizer_list = names.map { |name| "opsqlite_#{name}_init(db,&errMsg,nullptr);" }.join File.open(file_path, 'w') do |file| file.puts "#ifndef TOKENIZERS_H" file.puts "#define TOKENIZERS_H" file.puts + file.puts "#define TOKENIZER_LIST #{tokenizer_list}" + file.puts if is_user_app file.puts "#include \"node-modules/@op-engineering/op-sqlite/cpp/sqlite3ext.h\"" else diff --git a/op-sqlite.podspec b/op-sqlite.podspec index 7b6aba21..72b482ba 100644 --- a/op-sqlite.podspec +++ b/op-sqlite.podspec @@ -185,8 +185,8 @@ Pod::Spec.new do |s| end if tokenizers.any? then - tokenizer_inits = tokenizers.map { |tokenizer| "opsqlite_#{tokenizer}_init(db,&errMsg,nullptr);" } - xcconfig[:OTHER_CFLAGS] += " -DTOKENIZER_LIST=\"#{tokenizer_inits.join(" ")}\"" + # tokenizer_inits = tokenizers.map { |tokenizer| "opsqlite_#{tokenizer}_init(db,&errMsg,nullptr);" } + # xcconfig[:OTHER_CFLAGS] += " -DTOKENIZER_LIST=\"#{tokenizer_inits.join(" ")}\"" if is_user_app then xcconfig[:OTHER_CFLAGS] += " -DTOKENIZERS_HEADER_PATH=\\\"../../../c_sources/tokenizers.h\\\"" else From 8366e553e34def3ca8845734660a5562924b20a8 Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Sun, 10 Nov 2024 20:22:38 +0100 Subject: [PATCH 09/43] Working Android adding of sources --- android/CMakeLists.txt | 6 +++--- android/build.gradle | 4 +--- example/c_sources/tokenizers.cpp | 28 +++++++++++++++------------- example/c_sources/tokenizers.h | 10 +++++++--- 4 files changed, 26 insertions(+), 22 deletions(-) diff --git a/android/CMakeLists.txt b/android/CMakeLists.txt index 3bae5590..b83f4863 100644 --- a/android/CMakeLists.txt +++ b/android/CMakeLists.txt @@ -10,6 +10,7 @@ set (BUILD_DIR ${CMAKE_SOURCE_DIR}/build) ../cpp ../cpp/sqlcipher ../cpp/libsql +# ../example/c_sources ) add_definitions( @@ -74,12 +75,11 @@ find_library(LOG_LIB log) # Add user defined files if (USER_DEFINED_SOURCE_FILES) - string(REPLACE ";" " " user_source_files "${USER_DEFINED_SOURCE_FILES}") +# string(REPLACE ";" "\n" user_source_files "${USER_DEFINED_SOURCE_FILES}") message(WARNING "User source files: ${user_source_files}") - message(WARNING "INITLIST ${USER_DEFINED_TOKENIZER_INIT_STRINGS}") message(WARNING "TOKENIZERS_HEADER_PATH ${USER_DEFINED_TOKENIZERS_HEADER_PATH}") - target_sources(${PACKAGE_NAME} PRIVATE ${source_files}) + target_sources(${PACKAGE_NAME} PRIVATE ${USER_DEFINED_SOURCE_FILES}) # add_definitions("-DTOKENIZER_LIST=${USER_DEFINED_TOKENIZER_INIT_STRINGS}") add_definitions("-DTOKENIZERS_HEADER_PATH=\"${USER_DEFINED_TOKENIZERS_HEADER_PATH}\"") diff --git a/android/build.gradle b/android/build.gradle index f79708c9..d15d7704 100644 --- a/android/build.gradle +++ b/android/build.gradle @@ -171,11 +171,9 @@ android { // Scan user directories for .h and .cpp files def sourceDir = isInsideNodeModules ? file("$rootDir/../../../c_sources") : file("$rootDir/../c_sources") sourceFiles = fileTree(dir: sourceDir, include: ["**/*.cpp", "**/*.h"]) +// sourceFiles = fileTree(dir: sourceDir, include: ["**/*.cpp"]) .files .join(";") // Join paths with `;` for CMake - // tokenizerInitStrings = tokenizers.collect { tokenizer -> - // "opsqlite_${tokenizer}_init(db,&errMsg,nullptr);" - // }.join("") tokenizersHeaderPath = isInsideNodeModules ? "../../../c_sources/tokenizers.h" : "../example/c_sources/tokenizers.h" println("tokenizers header path $tokenizersHeaderPath") } diff --git a/example/c_sources/tokenizers.cpp b/example/c_sources/tokenizers.cpp index 1a720642..5b0a977d 100644 --- a/example/c_sources/tokenizers.cpp +++ b/example/c_sources/tokenizers.cpp @@ -5,12 +5,12 @@ namespace opsqlite { -fts5_api *fts5_api_from_db(sqlite3 *db){ +fts5_api *fts5_api_from_db(sqlite3 *db) { fts5_api *pRet = 0; sqlite3_stmt *pStmt = 0; - if( SQLITE_OK == sqlite3_prepare_v2(db, "SELECT fts5(?1)", -1, &pStmt, 0) ){ - sqlite3_bind_pointer(pStmt, 1, (void*)&pRet, "fts5_api_ptr", NULL); + if (SQLITE_OK == sqlite3_prepare_v2(db, "SELECT fts5(?1)", -1, &pStmt, 0)) { + sqlite3_bind_pointer(pStmt, 1, (void *)&pRet, "fts5_api_ptr", NULL); sqlite3_step(pStmt); } sqlite3_finalize(pStmt); @@ -60,27 +60,29 @@ int wordTokenizerTokenize(Fts5Tokenizer *pTokenizer, void *pCtx, int flags, } int opsqlite_wordtokenizer_init(sqlite3 *db, char **error, - const sqlite3_api_routines *api) { + sqlite3_api_routines const *api) { fts5_tokenizer wordtokenizer = {wordTokenizerCreate, wordTokenizerDelete, wordTokenizerTokenize}; fts5_api *ftsApi = (fts5_api *)fts5_api_from_db(db); if (ftsApi == NULL) return SQLITE_ERROR; - + return ftsApi->xCreateTokenizer(ftsApi, "wordtokenizer", NULL, &wordtokenizer, NULL); } +int opsqlite_porter_init(sqlite3 *db, char **error, + sqlite3_api_routines const *api) { + fts5_tokenizer porter_tokenizer = {wordTokenizerCreate, wordTokenizerDelete, + wordTokenizerTokenize}; + + fts5_api *ftsApi = (fts5_api *)fts5_api_from_db(db); + if (ftsApi == nullptr) + return SQLITE_ERROR; -int opsqlite_porter_init(sqlite3 *db, char **error, const sqlite3_api_routines *api) { - fts5_tokenizer porter_tokenizer = {wordTokenizerCreate, wordTokenizerDelete, wordTokenizerTokenize}; - - fts5_api *ftsApi = (fts5_api *)fts5_api_from_db(db); - if (ftsApi == nullptr) - return SQLITE_ERROR; - - return ftsApi->xCreateTokenizer(ftsApi, "portertokenizer", NULL, &porter_tokenizer, NULL); + return ftsApi->xCreateTokenizer(ftsApi, "portertokenizer", NULL, + &porter_tokenizer, NULL); } } // namespace opsqlite diff --git a/example/c_sources/tokenizers.h b/example/c_sources/tokenizers.h index 3c5137ab..b7a73e44 100644 --- a/example/c_sources/tokenizers.h +++ b/example/c_sources/tokenizers.h @@ -1,14 +1,18 @@ #ifndef TOKENIZERS_H #define TOKENIZERS_H -#define TOKENIZER_LIST opsqlite_wordtokenizer_init(db,&errMsg,nullptr);opsqlite_porter_init(db,&errMsg,nullptr); +#define TOKENIZER_LIST \ + opsqlite_wordtokenizer_init(db, &errMsg, nullptr); \ + opsqlite_porter_init(db, &errMsg, nullptr); #include "../../cpp/sqlite3.h" namespace opsqlite { -int opsqlite_wordtokenizer_init(sqlite3 *db, char **error, const sqlite3_api_routines *api); -int opsqlite_porter_init(sqlite3 *db, char **error, const sqlite3_api_routines *api); +int opsqlite_wordtokenizer_init(sqlite3 *db, char **error, + sqlite3_api_routines const *api); +int opsqlite_porter_init(sqlite3 *db, char **error, + sqlite3_api_routines const *api); } // namespace opsqlite From 182ae6cada5f8027c8b623815f1fe6c36bc74e83 Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Mon, 11 Nov 2024 01:23:09 +0300 Subject: [PATCH 10/43] Fix codegen script --- example/c_sources/tokenizers.h | 4 ++-- example/ios/Podfile.lock | 8 ++++---- generate_tokenizers_header_file.rb | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/example/c_sources/tokenizers.h b/example/c_sources/tokenizers.h index b7a73e44..fbbd48ca 100644 --- a/example/c_sources/tokenizers.h +++ b/example/c_sources/tokenizers.h @@ -10,9 +10,9 @@ namespace opsqlite { int opsqlite_wordtokenizer_init(sqlite3 *db, char **error, - sqlite3_api_routines const *api); + const sqlite3_api_routines *api); int opsqlite_porter_init(sqlite3 *db, char **error, - sqlite3_api_routines const *api); + const sqlite3_api_routines *api); } // namespace opsqlite diff --git a/example/ios/Podfile.lock b/example/ios/Podfile.lock index d47e5411..77c585d2 100644 --- a/example/ios/Podfile.lock +++ b/example/ios/Podfile.lock @@ -1779,14 +1779,14 @@ EXTERNAL SOURCES: SPEC CHECKSUMS: boost: 1dca942403ed9342f98334bf4c3621f011aa7946 - DoubleConversion: 76ab83afb40bddeeee456813d9c04f67f78771b5 + DoubleConversion: f16ae600a246532c4020132d54af21d0ddb2a385 FBLazyVector: 7075bb12898bc3998fd60f4b7ca422496cc2cdf7 - fmt: 4c2741a687cc09f0634a2e2c72a838b99f1ff120 + fmt: 10c6e61f4be25dc963c36bd73fc7b1705fe975be GCDWebServer: 2c156a56c8226e2d5c0c3f208a3621ccffbe3ce4 - glog: c5d68082e772fa1c511173d6b30a9de2c05a69a2 + glog: 08b301085f15bcbb6ff8632a8ebaf239aae04e6a hermes-engine: 46f1ffbf0297f4298862068dd4c274d4ac17a1fd op-sqlite: 9a4e3e88a0f5d1aecce6ffe9e1f4b0125b6ab003 - RCT-Folly: 045d6ecaa59d826c5736dfba0b2f4083ff8d79df + RCT-Folly: bf5c0376ffe4dd2cf438dcf86db385df9fdce648 RCTDeprecation: fde92935b3caa6cb65cbff9fbb7d3a9867ffb259 RCTRequired: 75c6cee42d21c1530a6f204ba32ff57335d19007 RCTTypeSafety: 7e6fe47bfb693c50d4669db1a480ca5331795f5b diff --git a/generate_tokenizers_header_file.rb b/generate_tokenizers_header_file.rb index c7e2dda1..34e83818 100644 --- a/generate_tokenizers_header_file.rb +++ b/generate_tokenizers_header_file.rb @@ -13,7 +13,7 @@ def generate_tokenizers_header_file(names, file_path, is_user_app) file.puts "#define TOKENIZER_LIST #{tokenizer_list}" file.puts if is_user_app - file.puts "#include \"node-modules/@op-engineering/op-sqlite/cpp/sqlite3ext.h\"" + file.puts "#include \"node-modules/@op-engineering/op-sqlite/cpp/sqlite3.h\"" else file.puts "#include \"../../cpp/sqlite3.h\"" end From 840367f171e7399966804e6c7cdf69638afac460 Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Mon, 11 Nov 2024 01:23:41 +0300 Subject: [PATCH 11/43] Bump version to beta --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 72151c37..6ed4eb51 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@op-engineering/op-sqlite", - "version": "10.0.0", + "version": "10.0.0-tokenizers-beta1", "description": "Next generation SQLite for React Native", "main": "lib/commonjs/index", "module": "lib/module/index", From 9b73943181164cc413371610766671d8d0d58870 Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Mon, 11 Nov 2024 01:39:03 +0300 Subject: [PATCH 12/43] Add missing file to publish --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 6ed4eb51..328f70ea 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@op-engineering/op-sqlite", - "version": "10.0.0-tokenizers-beta1", + "version": "10.0.0-tokenizers-beta2", "description": "Next generation SQLite for React Native", "main": "lib/commonjs/index", "module": "lib/module/index", @@ -14,6 +14,7 @@ "ios", "cpp", "op-sqlite.podspec", + "generate_tokenizers_header_file.rb", "ios/**.xcframework", "!lib/typescript/example", "!android/build", From 356dee869d51900ae791fa3913d877ff543450d5 Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Mon, 11 Nov 2024 02:04:21 +0300 Subject: [PATCH 13/43] Corrections --- generate_tokenizers_header_file.rb | 4 ++-- op-sqlite.podspec | 14 +++++++------- package.json | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/generate_tokenizers_header_file.rb b/generate_tokenizers_header_file.rb index 34e83818..90e8ebfa 100644 --- a/generate_tokenizers_header_file.rb +++ b/generate_tokenizers_header_file.rb @@ -13,7 +13,7 @@ def generate_tokenizers_header_file(names, file_path, is_user_app) file.puts "#define TOKENIZER_LIST #{tokenizer_list}" file.puts if is_user_app - file.puts "#include \"node-modules/@op-engineering/op-sqlite/cpp/sqlite3.h\"" + file.puts "#include \"sqlite3.h\"" else file.puts "#include \"../../cpp/sqlite3.h\"" end @@ -22,7 +22,7 @@ def generate_tokenizers_header_file(names, file_path, is_user_app) file.puts names.each do |name| - file.puts "int opsqlite_#{name}_init(sqlite3 *db, char **error, const sqlite3_api_routines *api);" + file.puts "int opsqlite_#{name}_init(sqlite3 *db, char **error, sqlite3_api_routines const *api);" end file.puts diff --git a/op-sqlite.podspec b/op-sqlite.podspec index 72b482ba..6f6f82dd 100644 --- a/op-sqlite.podspec +++ b/op-sqlite.podspec @@ -150,12 +150,12 @@ Pod::Spec.new do |s| if performance_mode == '1' then log_message.call("[OP-SQLITE] Thread unsafe (1) performance mode enabled. Use only transactions! πŸš€πŸš€") - xcconfig[:OTHER_CFLAGS] = optimizedCflags + ' -DSQLITE_THREADSAFE=0 ' + other_cflags = optimizedCflags + ' -DSQLITE_THREADSAFE=0 ' end if performance_mode == '2' then log_message.call("[OP-SQLITE] Thread safe (2) performance mode enabled πŸš€") - xcconfig[:OTHER_CFLAGS] = optimizedCflags + ' -DSQLITE_THREADSAFE=1 ' + other_cflags = optimizedCflags + ' -DSQLITE_THREADSAFE=1 ' end if use_crsqlite then @@ -181,19 +181,19 @@ Pod::Spec.new do |s| if sqlite_flags != "" then log_message.call("[OP-SQLITE] Custom SQLite flags: #{sqlite_flags}") - xcconfig[:OTHER_CFLAGS] += " #{sqlite_flags}" + other_cflags += " #{sqlite_flags}" end if tokenizers.any? then - # tokenizer_inits = tokenizers.map { |tokenizer| "opsqlite_#{tokenizer}_init(db,&errMsg,nullptr);" } - # xcconfig[:OTHER_CFLAGS] += " -DTOKENIZER_LIST=\"#{tokenizer_inits.join(" ")}\"" + log_message.call("[OP_SQLITE] Tokenizers enabled: #{tokenizers}") if is_user_app then - xcconfig[:OTHER_CFLAGS] += " -DTOKENIZERS_HEADER_PATH=\\\"../../../c_sources/tokenizers.h\\\"" + other_cflags += " -DTOKENIZERS_HEADER_PATH=\\\"../../../c_sources/tokenizers.h\\\"" else - xcconfig[:OTHER_CFLAGS] += " -DTOKENIZERS_HEADER_PATH=\\\"../example/c_sources/tokenizers.h\\\"" + other_cflags += " -DTOKENIZERS_HEADER_PATH=\\\"../example/c_sources/tokenizers.h\\\"" end end + xcconfig[:OTHER_CFLAGS] = other_cflags s.pod_target_xcconfig = xcconfig s.vendored_frameworks = frameworks end diff --git a/package.json b/package.json index 328f70ea..4aa96744 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@op-engineering/op-sqlite", - "version": "10.0.0-tokenizers-beta2", + "version": "10.0.0-tokenizers-beta3", "description": "Next generation SQLite for React Native", "main": "lib/commonjs/index", "module": "lib/module/index", From 53264f2cf3bc1e3d6089469da7fc1ab15fea714b Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Mon, 11 Nov 2024 22:29:48 +0700 Subject: [PATCH 14/43] Fix header path --- op-sqlite.podspec | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/op-sqlite.podspec b/op-sqlite.podspec index 6f6f82dd..5000a7d9 100644 --- a/op-sqlite.podspec +++ b/op-sqlite.podspec @@ -187,7 +187,7 @@ Pod::Spec.new do |s| if tokenizers.any? then log_message.call("[OP_SQLITE] Tokenizers enabled: #{tokenizers}") if is_user_app then - other_cflags += " -DTOKENIZERS_HEADER_PATH=\\\"../../../c_sources/tokenizers.h\\\"" + other_cflags += " -DTOKENIZERS_HEADER_PATH=\\\"../../../../c_sources/tokenizers.h\\\"" else other_cflags += " -DTOKENIZERS_HEADER_PATH=\\\"../example/c_sources/tokenizers.h\\\"" end diff --git a/package.json b/package.json index 4aa96744..cb66b131 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@op-engineering/op-sqlite", - "version": "10.0.0-tokenizers-beta3", + "version": "10.0.0-tokenizers-beta4", "description": "Next generation SQLite for React Native", "main": "lib/commonjs/index", "module": "lib/module/index", From 6c0ff387509615ada004361996dd02a0e7fa637f Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Mon, 11 Nov 2024 22:36:06 +0700 Subject: [PATCH 15/43] Fix sources path --- op-sqlite.podspec | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/op-sqlite.podspec b/op-sqlite.podspec index 5000a7d9..0ba4f1c2 100644 --- a/op-sqlite.podspec +++ b/op-sqlite.podspec @@ -83,7 +83,7 @@ Pod::Spec.new do |s| # Set the path to the `c_sources` directory based on environment if is_user_app - c_sources_dir = File.join("..", "..", "..", "c_sources") + c_sources_dir = File.join("..", "..", "..", "..", "c_sources") else c_sources_dir = File.join("example", "c_sources") end diff --git a/package.json b/package.json index cb66b131..c0d5fdf2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@op-engineering/op-sqlite", - "version": "10.0.0-tokenizers-beta4", + "version": "10.0.0-tokenizers-beta5", "description": "Next generation SQLite for React Native", "main": "lib/commonjs/index", "module": "lib/module/index", From 7b64a8718a1645d4ab48c9e750b563d502535e33 Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Mon, 11 Nov 2024 22:41:58 +0700 Subject: [PATCH 16/43] Fix sources path --- op-sqlite.podspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/op-sqlite.podspec b/op-sqlite.podspec index 0ba4f1c2..5000a7d9 100644 --- a/op-sqlite.podspec +++ b/op-sqlite.podspec @@ -83,7 +83,7 @@ Pod::Spec.new do |s| # Set the path to the `c_sources` directory based on environment if is_user_app - c_sources_dir = File.join("..", "..", "..", "..", "c_sources") + c_sources_dir = File.join("..", "..", "..", "c_sources") else c_sources_dir = File.join("example", "c_sources") end From 36dc9d29bdd6140446a925492a786ce620c3a0ab Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Tue, 12 Nov 2024 16:14:00 +0700 Subject: [PATCH 17/43] Copy file sources to podspec dir --- op-sqlite.podspec | 18 +++++++++++++----- package.json | 2 +- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/op-sqlite.podspec b/op-sqlite.podspec index 5000a7d9..858f29d5 100644 --- a/op-sqlite.podspec +++ b/op-sqlite.podspec @@ -87,14 +87,22 @@ Pod::Spec.new do |s| else c_sources_dir = File.join("example", "c_sources") end + if tokenizers.any? - generate_tokenizers_header_file(tokenizers, File.join(c_sources_dir, "tokenizers.h"), is_user_app) + FileUtils.cp_r(c_sources_dir, __dir__) + # generate_tokenizers_header_file(tokenizers, File.join(c_sources_dir, "tokenizers.h"), is_user_app) + # puts "Current directory: #{__dir__}" + # c_sources_dir_output = Dir.glob(File.join(c_sources_dir, "**/*.{h,cpp}")) + + # puts "c_sources_dir: #{c_sources_dir_output}" + + # # Add all .h and .c files from the `c_sources` directory + source_files += Dir.glob(File.join("c_sources", "**/*.{h,cpp}")) + # source_files += ["../../c_sources/tokenizers.h", "../../c_sources/tokenizers.cpp"] + # puts "Source files: #{source_files}" end - # Add all .h and .c files from the `c_sources` directory - source_files += Dir.glob(File.join(c_sources_dir, "**/*.{h,cpp}")) - # Assign the collected source files to `s.source_files` s.source_files = source_files @@ -187,7 +195,7 @@ Pod::Spec.new do |s| if tokenizers.any? then log_message.call("[OP_SQLITE] Tokenizers enabled: #{tokenizers}") if is_user_app then - other_cflags += " -DTOKENIZERS_HEADER_PATH=\\\"../../../../c_sources/tokenizers.h\\\"" + other_cflags += " -DTOKENIZERS_HEADER_PATH=\\\"../c_sources/tokenizers.h\\\"" else other_cflags += " -DTOKENIZERS_HEADER_PATH=\\\"../example/c_sources/tokenizers.h\\\"" end diff --git a/package.json b/package.json index c0d5fdf2..e85873c2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@op-engineering/op-sqlite", - "version": "10.0.0-tokenizers-beta5", + "version": "10.0.0-tokenizers-beta6", "description": "Next generation SQLite for React Native", "main": "lib/commonjs/index", "module": "lib/module/index", From 6ed63eedb958470fc3322d5c561f5c2e0c7ed623 Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Tue, 12 Nov 2024 18:02:26 +0700 Subject: [PATCH 18/43] Copy files on Android --- android/build.gradle | 17 ++++++++++------- package.json | 2 +- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/android/build.gradle b/android/build.gradle index d15d7704..046e97d5 100644 --- a/android/build.gradle +++ b/android/build.gradle @@ -168,14 +168,17 @@ android { // def tokenizerInitStrings = 0 def tokenizersHeaderPath = 0 if (!tokenizers.isEmpty()) { - // Scan user directories for .h and .cpp files def sourceDir = isInsideNodeModules ? file("$rootDir/../../../c_sources") : file("$rootDir/../c_sources") - sourceFiles = fileTree(dir: sourceDir, include: ["**/*.cpp", "**/*.h"]) -// sourceFiles = fileTree(dir: sourceDir, include: ["**/*.cpp"]) - .files - .join(";") // Join paths with `;` for CMake - tokenizersHeaderPath = isInsideNodeModules ? "../../../c_sources/tokenizers.h" : "../example/c_sources/tokenizers.h" - println("tokenizers header path $tokenizersHeaderPath") + def destDir = file("$buildscript.sourceFile.parentFile/c_sources") + copy { + from sourceDir + into destDir + include "**/*.cpp", "**/*.h" + } + sourceFiles = fileTree(dir: destDir, include: ["**/*.cpp", "**/*.h"]).files.join(";") + tokenizersHeaderPath = "../c_sources/tokenizers.h" + println("tokenizers header path $tokenizersHeaderPath") + } cppFlags "-O2", "-fexceptions", "-DONANDROID" diff --git a/package.json b/package.json index e85873c2..6934f7b8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@op-engineering/op-sqlite", - "version": "10.0.0-tokenizers-beta6", + "version": "10.0.0-tokenizers-beta7", "description": "Next generation SQLite for React Native", "main": "lib/commonjs/index", "module": "lib/module/index", From 573a3107cdf82e4cad3967b45fcbdbf90877124b Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Fri, 15 Nov 2024 19:15:28 +0700 Subject: [PATCH 19/43] Fix typo in empty macro --- cpp/bridge.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/bridge.cpp b/cpp/bridge.cpp index 48933996..c4da7657 100644 --- a/cpp/bridge.cpp +++ b/cpp/bridge.cpp @@ -4,14 +4,14 @@ #include "logs.h" #include "utils.h" #include +#include #include #include -#include #ifdef TOKENIZERS_HEADER_PATH #include TOKENIZERS_HEADER_PATH #else -#define TOKENIZERS_LIST +#define TOKENIZER_LIST #endif namespace opsqlite { From 4a323a633b4d056bdcc52c41eafdf115553a99af Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Fri, 15 Nov 2024 19:15:49 +0700 Subject: [PATCH 20/43] Bump --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 6934f7b8..891f7ba2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@op-engineering/op-sqlite", - "version": "10.0.0-tokenizers-beta7", + "version": "10.0.0-tokenizers-beta8", "description": "Next generation SQLite for React Native", "main": "lib/commonjs/index", "module": "lib/module/index", From f42bf52f6b428a92683873532e1652e44ba7bed4 Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Fri, 15 Nov 2024 19:35:02 +0700 Subject: [PATCH 21/43] Fix tokenizers header generation --- c_sources/tokenizers.cpp | 88 ++++++++++++++++++++++++++++++ c_sources/tokenizers.h | 15 +++++ example/c_sources/tokenizers.h | 12 ++-- example/ios/Podfile.lock | 8 +-- example/package.json | 1 + generate_tokenizers_header_file.rb | 8 +-- op-sqlite.podspec | 2 +- 7 files changed, 115 insertions(+), 19 deletions(-) create mode 100644 c_sources/tokenizers.cpp create mode 100644 c_sources/tokenizers.h diff --git a/c_sources/tokenizers.cpp b/c_sources/tokenizers.cpp new file mode 100644 index 00000000..5b0a977d --- /dev/null +++ b/c_sources/tokenizers.cpp @@ -0,0 +1,88 @@ +#include "tokenizers.h" +#include +#include +#include + +namespace opsqlite { + +fts5_api *fts5_api_from_db(sqlite3 *db) { + fts5_api *pRet = 0; + sqlite3_stmt *pStmt = 0; + + if (SQLITE_OK == sqlite3_prepare_v2(db, "SELECT fts5(?1)", -1, &pStmt, 0)) { + sqlite3_bind_pointer(pStmt, 1, (void *)&pRet, "fts5_api_ptr", NULL); + sqlite3_step(pStmt); + } + sqlite3_finalize(pStmt); + return pRet; +} + +class WordTokenizer { +public: + WordTokenizer() = default; + ~WordTokenizer() = default; +}; + +// Define `xCreate`, which initializes the tokenizer +int wordTokenizerCreate(void *pUnused, const char **azArg, int nArg, + Fts5Tokenizer **ppOut) { + auto tokenizer = std::make_unique(); + *ppOut = reinterpret_cast( + tokenizer.release()); // Cast to Fts5Tokenizer* + return SQLITE_OK; +} + +// Define `xDelete`, which frees the tokenizer +void wordTokenizerDelete(Fts5Tokenizer *pTokenizer) { + delete reinterpret_cast(pTokenizer); +} + +// Define `xTokenize`, which performs the actual tokenization +int wordTokenizerTokenize(Fts5Tokenizer *pTokenizer, void *pCtx, int flags, + const char *pText, int nText, + int (*xToken)(void *, int, const char *, int, int, + int)) { + int start = 0; + int i = 0; + + while (i <= nText) { + if (i == nText || !std::isalnum(static_cast(pText[i]))) { + if (start < i) { // Found a token + int rc = xToken(pCtx, 0, pText + start, i - start, start, i); + if (rc != SQLITE_OK) + return rc; + } + start = i + 1; + } + i++; + } + return SQLITE_OK; +} + +int opsqlite_wordtokenizer_init(sqlite3 *db, char **error, + sqlite3_api_routines const *api) { + fts5_tokenizer wordtokenizer = {wordTokenizerCreate, wordTokenizerDelete, + wordTokenizerTokenize}; + + fts5_api *ftsApi = (fts5_api *)fts5_api_from_db(db); + if (ftsApi == NULL) + return SQLITE_ERROR; + + return ftsApi->xCreateTokenizer(ftsApi, "wordtokenizer", NULL, &wordtokenizer, + NULL); +} + +int opsqlite_porter_init(sqlite3 *db, char **error, + sqlite3_api_routines const *api) { + fts5_tokenizer porter_tokenizer = {wordTokenizerCreate, wordTokenizerDelete, + wordTokenizerTokenize}; + + fts5_api *ftsApi = (fts5_api *)fts5_api_from_db(db); + if (ftsApi == nullptr) + return SQLITE_ERROR; + + return ftsApi->xCreateTokenizer(ftsApi, "portertokenizer", NULL, + &porter_tokenizer, NULL); +} + +} // namespace opsqlite diff --git a/c_sources/tokenizers.h b/c_sources/tokenizers.h new file mode 100644 index 00000000..ccbf95be --- /dev/null +++ b/c_sources/tokenizers.h @@ -0,0 +1,15 @@ +#ifndef TOKENIZERS_H +#define TOKENIZERS_H + +#define TOKENIZER_LIST opsqlite_wordtokenizer_init(db,&errMsg,nullptr);opsqlite_porter_init(db,&errMsg,nullptr); + +#include "sqlite3.h" + +namespace opsqlite { + +int opsqlite_wordtokenizer_init(sqlite3 *db, char **error, sqlite3_api_routines const *api); +int opsqlite_porter_init(sqlite3 *db, char **error, sqlite3_api_routines const *api); + +} // namespace opsqlite + +#endif // TOKENIZERS_H diff --git a/example/c_sources/tokenizers.h b/example/c_sources/tokenizers.h index fbbd48ca..ccbf95be 100644 --- a/example/c_sources/tokenizers.h +++ b/example/c_sources/tokenizers.h @@ -1,18 +1,14 @@ #ifndef TOKENIZERS_H #define TOKENIZERS_H -#define TOKENIZER_LIST \ - opsqlite_wordtokenizer_init(db, &errMsg, nullptr); \ - opsqlite_porter_init(db, &errMsg, nullptr); +#define TOKENIZER_LIST opsqlite_wordtokenizer_init(db,&errMsg,nullptr);opsqlite_porter_init(db,&errMsg,nullptr); -#include "../../cpp/sqlite3.h" +#include "sqlite3.h" namespace opsqlite { -int opsqlite_wordtokenizer_init(sqlite3 *db, char **error, - const sqlite3_api_routines *api); -int opsqlite_porter_init(sqlite3 *db, char **error, - const sqlite3_api_routines *api); +int opsqlite_wordtokenizer_init(sqlite3 *db, char **error, sqlite3_api_routines const *api); +int opsqlite_porter_init(sqlite3 *db, char **error, sqlite3_api_routines const *api); } // namespace opsqlite diff --git a/example/ios/Podfile.lock b/example/ios/Podfile.lock index 77c585d2..035df6fc 100644 --- a/example/ios/Podfile.lock +++ b/example/ios/Podfile.lock @@ -10,7 +10,7 @@ PODS: - hermes-engine (0.76.1): - hermes-engine/Pre-built (= 0.76.1) - hermes-engine/Pre-built (0.76.1) - - op-sqlite (10.0.0): + - op-sqlite (10.0.0-tokenizers-beta8): - DoubleConversion - glog - hermes-engine @@ -1540,7 +1540,7 @@ PODS: - React-logger (= 0.76.1) - React-perflogger (= 0.76.1) - React-utils (= 0.76.1) - - RNShare (11.0.3): + - RNShare (11.0.4): - DoubleConversion - glog - hermes-engine @@ -1785,7 +1785,7 @@ SPEC CHECKSUMS: GCDWebServer: 2c156a56c8226e2d5c0c3f208a3621ccffbe3ce4 glog: 08b301085f15bcbb6ff8632a8ebaf239aae04e6a hermes-engine: 46f1ffbf0297f4298862068dd4c274d4ac17a1fd - op-sqlite: 9a4e3e88a0f5d1aecce6ffe9e1f4b0125b6ab003 + op-sqlite: d694c00d61984c2f7245b915e17d05187533ec6e RCT-Folly: bf5c0376ffe4dd2cf438dcf86db385df9fdce648 RCTDeprecation: fde92935b3caa6cb65cbff9fbb7d3a9867ffb259 RCTRequired: 75c6cee42d21c1530a6f204ba32ff57335d19007 @@ -1844,7 +1844,7 @@ SPEC CHECKSUMS: React-utils: 5362bd16a9563f9916e7a56c011ddc533507650f ReactCodegen: 865bafc5c17ec2181620ced1a32c39c38ab2951d ReactCommon: 422e364463f33e336fc4db196aeb50fd801d90d6 - RNShare: e1721a8818a3bf111ed686ed5d8c1dc76b91c8ad + RNShare: 4305edead1b8f614ab994046c68193e8d50aaadc SocketRocket: d4aabe649be1e368d1318fdf28a022d714d65748 Yoga: db69236006b8b1c6d55ab453390c882306cbf219 diff --git a/example/package.json b/example/package.json index 9017a3e7..42cf6b87 100644 --- a/example/package.json +++ b/example/package.json @@ -69,6 +69,7 @@ "sqlcipher": false, "crsqlite": false, "performanceMode": "2", + "sqliteFlags": "-DSQLITE_TEMP_STORE=2", "iosSqlite": false, "fts5": true, "rtree": true, diff --git a/generate_tokenizers_header_file.rb b/generate_tokenizers_header_file.rb index 90e8ebfa..9e34afd4 100644 --- a/generate_tokenizers_header_file.rb +++ b/generate_tokenizers_header_file.rb @@ -1,6 +1,6 @@ require 'fileutils' -def generate_tokenizers_header_file(names, file_path, is_user_app) +def generate_tokenizers_header_file(names, file_path) # Ensure the directory exists dir_path = File.dirname(file_path) FileUtils.mkdir_p(dir_path) unless Dir.exist?(dir_path) @@ -12,11 +12,7 @@ def generate_tokenizers_header_file(names, file_path, is_user_app) file.puts file.puts "#define TOKENIZER_LIST #{tokenizer_list}" file.puts - if is_user_app - file.puts "#include \"sqlite3.h\"" - else - file.puts "#include \"../../cpp/sqlite3.h\"" - end + file.puts "#include \"sqlite3.h\"" file.puts file.puts "namespace opsqlite {" file.puts diff --git a/op-sqlite.podspec b/op-sqlite.podspec index 858f29d5..6b45dfff 100644 --- a/op-sqlite.podspec +++ b/op-sqlite.podspec @@ -90,8 +90,8 @@ Pod::Spec.new do |s| if tokenizers.any? + generate_tokenizers_header_file(tokenizers, File.join(c_sources_dir, "tokenizers.h")) FileUtils.cp_r(c_sources_dir, __dir__) - # generate_tokenizers_header_file(tokenizers, File.join(c_sources_dir, "tokenizers.h"), is_user_app) # puts "Current directory: #{__dir__}" # c_sources_dir_output = Dir.glob(File.join(c_sources_dir, "**/*.{h,cpp}")) From cffc893764532ccb5fde70c854544d6709a8e26f Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Fri, 15 Nov 2024 19:35:50 +0700 Subject: [PATCH 22/43] Bump --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 891f7ba2..9313580c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@op-engineering/op-sqlite", - "version": "10.0.0-tokenizers-beta8", + "version": "10.0.0-tokenizers-beta9", "description": "Next generation SQLite for React Native", "main": "lib/commonjs/index", "module": "lib/module/index", From e854895fce52b401cb0b60e935bfb721bc75b129 Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Mon, 18 Nov 2024 15:18:55 +0700 Subject: [PATCH 23/43] Clean up source files --- .gitignore | 4 +++- android/CMakeLists.txt | 5 ----- android/build.gradle | 3 --- example/ios/Podfile.lock | 4 ++-- 4 files changed, 5 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index ed9f7eda..1db2b1aa 100644 --- a/.gitignore +++ b/.gitignore @@ -81,4 +81,6 @@ android/gradle/ !.yarn/plugins !.yarn/releases !.yarn/sdks -!.yarn/versions \ No newline at end of file +!.yarn/versions + +android/c_sources \ No newline at end of file diff --git a/android/CMakeLists.txt b/android/CMakeLists.txt index b83f4863..8df9623a 100644 --- a/android/CMakeLists.txt +++ b/android/CMakeLists.txt @@ -75,12 +75,7 @@ find_library(LOG_LIB log) # Add user defined files if (USER_DEFINED_SOURCE_FILES) -# string(REPLACE ";" "\n" user_source_files "${USER_DEFINED_SOURCE_FILES}") - message(WARNING "User source files: ${user_source_files}") - message(WARNING "TOKENIZERS_HEADER_PATH ${USER_DEFINED_TOKENIZERS_HEADER_PATH}") - target_sources(${PACKAGE_NAME} PRIVATE ${USER_DEFINED_SOURCE_FILES}) -# add_definitions("-DTOKENIZER_LIST=${USER_DEFINED_TOKENIZER_INIT_STRINGS}") add_definitions("-DTOKENIZERS_HEADER_PATH=\"${USER_DEFINED_TOKENIZERS_HEADER_PATH}\"") endif() diff --git a/android/build.gradle b/android/build.gradle index 046e97d5..fd98bcf9 100644 --- a/android/build.gradle +++ b/android/build.gradle @@ -177,8 +177,6 @@ android { } sourceFiles = fileTree(dir: destDir, include: ["**/*.cpp", "**/*.h"]).files.join(";") tokenizersHeaderPath = "../c_sources/tokenizers.h" - println("tokenizers header path $tokenizersHeaderPath") - } cppFlags "-O2", "-fexceptions", "-DONANDROID" @@ -190,7 +188,6 @@ android { "-DUSE_LIBSQL=${useLibsql ? 1 : 0}", "-DUSE_SQLITE_VEC=${useSqliteVec ? 1 : 0}", "-DUSER_DEFINED_SOURCE_FILES=${sourceFiles}", - // "-DUSER_DEFINED_TOKENIZER_INIT_STRINGS='${tokenizerInitStrings}'", "-DUSER_DEFINED_TOKENIZERS_HEADER_PATH='${tokenizersHeaderPath}'" } } diff --git a/example/ios/Podfile.lock b/example/ios/Podfile.lock index 035df6fc..7bbf4747 100644 --- a/example/ios/Podfile.lock +++ b/example/ios/Podfile.lock @@ -10,7 +10,7 @@ PODS: - hermes-engine (0.76.1): - hermes-engine/Pre-built (= 0.76.1) - hermes-engine/Pre-built (0.76.1) - - op-sqlite (10.0.0-tokenizers-beta8): + - op-sqlite (10.0.0-tokenizers-beta9): - DoubleConversion - glog - hermes-engine @@ -1785,7 +1785,7 @@ SPEC CHECKSUMS: GCDWebServer: 2c156a56c8226e2d5c0c3f208a3621ccffbe3ce4 glog: 08b301085f15bcbb6ff8632a8ebaf239aae04e6a hermes-engine: 46f1ffbf0297f4298862068dd4c274d4ac17a1fd - op-sqlite: d694c00d61984c2f7245b915e17d05187533ec6e + op-sqlite: c57328a22832c1f6093c01f13281b66cf5ee49d5 RCT-Folly: bf5c0376ffe4dd2cf438dcf86db385df9fdce648 RCTDeprecation: fde92935b3caa6cb65cbff9fbb7d3a9867ffb259 RCTRequired: 75c6cee42d21c1530a6f204ba32ff57335d19007 From 60673ce68fb46ee65368b2c1d6f814148dcc3782 Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Mon, 18 Nov 2024 15:21:25 +0700 Subject: [PATCH 24/43] Restore tests --- example/src/tests/tokenizer.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/example/src/tests/tokenizer.spec.ts b/example/src/tests/tokenizer.spec.ts index 8dc8f40e..0a9cca56 100644 --- a/example/src/tests/tokenizer.spec.ts +++ b/example/src/tests/tokenizer.spec.ts @@ -1,6 +1,6 @@ import {open, type DB} from '@op-engineering/op-sqlite'; import chai from 'chai'; -import {afterEach, beforeEach, describe, it, itOnly} from './MochaRNAdapter'; +import {afterEach, beforeEach, describe, it} from './MochaRNAdapter'; const expect = chai.expect; @@ -28,7 +28,7 @@ export function tokenizerTests() { } }); - itOnly('Should match the word split by the tokenizer', async () => { + it('Should match the word split by the tokenizer', async () => { await db.execute('INSERT INTO tokenizer_table(content) VALUES (?)', [ 'This is a test document', ]); From 2a25122d2369eaf31a0ac7221309ca1529088a01 Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Mon, 18 Nov 2024 15:21:40 +0700 Subject: [PATCH 25/43] Get rid of warn --- example/src/tests/tokenizer.spec.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/example/src/tests/tokenizer.spec.ts b/example/src/tests/tokenizer.spec.ts index 0a9cca56..66d8d859 100644 --- a/example/src/tests/tokenizer.spec.ts +++ b/example/src/tests/tokenizer.spec.ts @@ -36,7 +36,6 @@ export function tokenizerTests() { 'SELECT content FROM tokenizer_table WHERE content MATCH ?', ['test'], ); - console.warn(res); expect(res.rows.length).to.be.equal(1); expect(res.rows[0]!.content).to.be.equal('This is a test document'); }); From 15807dc4fd59f19bfff6a16817395366433b7a38 Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Mon, 18 Nov 2024 15:30:54 +0700 Subject: [PATCH 26/43] Add docs to libsql functions --- src/index.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/index.ts b/src/index.ts index 01963067..44edfad6 100644 --- a/src/index.ts +++ b/src/index.ts @@ -177,6 +177,13 @@ export type DB = { }[]; callback: (response: any) => void; }) => () => void; + /** This function is only available for libsql. + * Allows to trigger a sync the database with it's remote replica + * In order for this function to work you need to use openSync or openRemote functions + * with libsql: true in the package.json + * + * The database is hosted in turso + **/ sync: () => void; flushPendingReactiveQueries: () => Promise; }; @@ -405,6 +412,9 @@ function enhanceDB(db: DB, options: any): DB { return enhancedDb; } +/** Open a replicating connection via libsql to a turso db + * libsql needs to be enabled on your package.json + */ export const openSync = (options: { url: string; authToken: string; @@ -422,6 +432,9 @@ export const openSync = (options: { return enhancedDb; }; +/** Open a remote connection via libsql to a turso db + * libsql needs to be enabled on your package.json + */ export const openRemote = (options: { url: string; authToken: string }): DB => { if (!isLibsql()) { throw new Error('This function is only available for libsql'); From 4e477ccccd1a7d20af462bc15efb835bb42d0355 Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Mon, 18 Nov 2024 15:54:59 +0700 Subject: [PATCH 27/43] Modify turbo files --- example/ios/Podfile.lock | 2 +- example/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/example/ios/Podfile.lock b/example/ios/Podfile.lock index 7bbf4747..3a80ed7e 100644 --- a/example/ios/Podfile.lock +++ b/example/ios/Podfile.lock @@ -1785,7 +1785,7 @@ SPEC CHECKSUMS: GCDWebServer: 2c156a56c8226e2d5c0c3f208a3621ccffbe3ce4 glog: 08b301085f15bcbb6ff8632a8ebaf239aae04e6a hermes-engine: 46f1ffbf0297f4298862068dd4c274d4ac17a1fd - op-sqlite: c57328a22832c1f6093c01f13281b66cf5ee49d5 + op-sqlite: 63400939931ca67186f99f8b536a02b048133710 RCT-Folly: bf5c0376ffe4dd2cf438dcf86db385df9fdce648 RCTDeprecation: fde92935b3caa6cb65cbff9fbb7d3a9867ffb259 RCTRequired: 75c6cee42d21c1530a6f204ba32ff57335d19007 diff --git a/example/package.json b/example/package.json index 42cf6b87..68de311a 100644 --- a/example/package.json +++ b/example/package.json @@ -73,7 +73,7 @@ "iosSqlite": false, "fts5": true, "rtree": true, - "libsql": false, + "libsql": true, "sqliteVec": true, "tokenizers": [ "wordtokenizer", From e64213af44b58efda434ac111d2155f03a7357c7 Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Mon, 18 Nov 2024 16:04:42 +0700 Subject: [PATCH 28/43] Modify Android test script --- scripts/test-android.sh | 2 +- turbo.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/test-android.sh b/scripts/test-android.sh index 55607b4c..a47bf0e9 100755 --- a/scripts/test-android.sh +++ b/scripts/test-android.sh @@ -1,4 +1,4 @@ -JAVA_OPTS=-XX:MaxHeapSize=6g yarn turbo run run:android:release --cache-dir=.turbo/android +JAVA_OPTS=-XX:MaxHeapSize=6g yarn android adb forward tcp:9000 tcp:9000 echo "Polling in-app server..." node ./scripts/poll-in-app-server.js \ No newline at end of file diff --git a/turbo.json b/turbo.json index cde2f601..5ec77b7c 100644 --- a/turbo.json +++ b/turbo.json @@ -7,7 +7,7 @@ "android", "!android/build", "src/*.ts", - "src/*.tsx", + "src/tests/*.tsx", "example/package.json", "example/android", "cpp", @@ -24,7 +24,7 @@ "ios", "cpp", "src/*.ts", - "src/*.tsx", + "src/tests/*.tsx", "example/package.json", "example/ios", "!example/ios/build", From 8d6e624c88c76c9d52cc17b6e364d2aaebd7219a Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Mon, 18 Nov 2024 16:07:20 +0700 Subject: [PATCH 29/43] Modify android script --- scripts/test-android.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/test-android.sh b/scripts/test-android.sh index a47bf0e9..7d541353 100755 --- a/scripts/test-android.sh +++ b/scripts/test-android.sh @@ -1,3 +1,4 @@ +cd example JAVA_OPTS=-XX:MaxHeapSize=6g yarn android adb forward tcp:9000 tcp:9000 echo "Polling in-app server..." From 148f17ed0e1d4d8564a4d75d84e34383f6992f2b Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Mon, 18 Nov 2024 16:10:31 +0700 Subject: [PATCH 30/43] Restore turbo --- scripts/test-android.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/test-android.sh b/scripts/test-android.sh index 7d541353..55607b4c 100755 --- a/scripts/test-android.sh +++ b/scripts/test-android.sh @@ -1,5 +1,4 @@ -cd example -JAVA_OPTS=-XX:MaxHeapSize=6g yarn android +JAVA_OPTS=-XX:MaxHeapSize=6g yarn turbo run run:android:release --cache-dir=.turbo/android adb forward tcp:9000 tcp:9000 echo "Polling in-app server..." node ./scripts/poll-in-app-server.js \ No newline at end of file From 1b66ac7ba528679788ebd16713ada333af0c943e Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Mon, 18 Nov 2024 17:04:52 +0700 Subject: [PATCH 31/43] Change turbo --- turbo.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/turbo.json b/turbo.json index 5ec77b7c..e5c3058c 100644 --- a/turbo.json +++ b/turbo.json @@ -7,7 +7,7 @@ "android", "!android/build", "src/*.ts", - "src/tests/*.tsx", + "src/tests/*.ts", "example/package.json", "example/android", "cpp", @@ -24,7 +24,7 @@ "ios", "cpp", "src/*.ts", - "src/tests/*.tsx", + "src/tests/*.ts", "example/package.json", "example/ios", "!example/ios/build", From ebbddd291e14d079be37d5b8d6c9536b2df75c79 Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Mon, 18 Nov 2024 17:15:36 +0700 Subject: [PATCH 32/43] Get rid of turbo on android --- scripts/test-android.sh | 3 ++- scripts/turnOnLibsql.js | 4 ---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/scripts/test-android.sh b/scripts/test-android.sh index 55607b4c..c01741b2 100755 --- a/scripts/test-android.sh +++ b/scripts/test-android.sh @@ -1,4 +1,5 @@ -JAVA_OPTS=-XX:MaxHeapSize=6g yarn turbo run run:android:release --cache-dir=.turbo/android +cd example +JAVA_OPTS=-XX:MaxHeapSize=6g yarn run:android:release --cache-dir=.turbo/android adb forward tcp:9000 tcp:9000 echo "Polling in-app server..." node ./scripts/poll-in-app-server.js \ No newline at end of file diff --git a/scripts/turnOnLibsql.js b/scripts/turnOnLibsql.js index 6e13c013..402a52bd 100644 --- a/scripts/turnOnLibsql.js +++ b/scripts/turnOnLibsql.js @@ -1,7 +1,5 @@ const fs = require('fs'); -console.log('Current working directory:', process.cwd()); - // Read the package.json file const packageJson = JSON.parse(fs.readFileSync('./example/package.json')); @@ -14,5 +12,3 @@ fs.writeFileSync( './example/package.json', JSON.stringify(packageJson, null, 2) ); - -console.log('package.json updated successfully!'); From d121fb43598f02d18509713e01623754fbe92286 Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Mon, 18 Nov 2024 17:23:08 +0700 Subject: [PATCH 33/43] CI --- .github/workflows/ci.yml | 7 ++----- scripts/turnOnLibsql.js | 2 ++ 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ba0f28aa..82200a56 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -420,12 +420,9 @@ jobs: env: TURBO_CACHE_DIR: .turbo/android steps: - - name: Checkout - uses: actions/checkout@v4 + - uses: actions/checkout@v4 - - name: Turn on libsql - run: | - node ./scripts/turnOnLibsql.js + - run: node ./scripts/turnOnLibsql.js - name: Setup uses: ./.github/actions/setup diff --git a/scripts/turnOnLibsql.js b/scripts/turnOnLibsql.js index 402a52bd..9b125f88 100644 --- a/scripts/turnOnLibsql.js +++ b/scripts/turnOnLibsql.js @@ -12,3 +12,5 @@ fs.writeFileSync( './example/package.json', JSON.stringify(packageJson, null, 2) ); + +console.log('Turned on libsql in package.json', packageJson); From 301a73125a2d947c86edefb5e34cdf3facfa5103 Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Mon, 18 Nov 2024 17:52:27 +0700 Subject: [PATCH 34/43] CI --- scripts/test-android.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test-android.sh b/scripts/test-android.sh index c01741b2..870ce638 100755 --- a/scripts/test-android.sh +++ b/scripts/test-android.sh @@ -2,4 +2,4 @@ cd example JAVA_OPTS=-XX:MaxHeapSize=6g yarn run:android:release --cache-dir=.turbo/android adb forward tcp:9000 tcp:9000 echo "Polling in-app server..." -node ./scripts/poll-in-app-server.js \ No newline at end of file +node ../scripts/poll-in-app-server.js \ No newline at end of file From b271b15e4bca02d161d4f7c0318a72edc9bcc72e Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Mon, 18 Nov 2024 18:03:20 +0700 Subject: [PATCH 35/43] CI --- scripts/test-android.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test-android.sh b/scripts/test-android.sh index 870ce638..84f9627f 100755 --- a/scripts/test-android.sh +++ b/scripts/test-android.sh @@ -1,5 +1,5 @@ cd example -JAVA_OPTS=-XX:MaxHeapSize=6g yarn run:android:release --cache-dir=.turbo/android +JAVA_OPTS=-XX:MaxHeapSize=6g yarn run:android:release adb forward tcp:9000 tcp:9000 echo "Polling in-app server..." node ../scripts/poll-in-app-server.js \ No newline at end of file From 1b30cddcc39145d85b7bd3d777521fd5bae5568a Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Mon, 18 Nov 2024 18:51:11 +0700 Subject: [PATCH 36/43] Add message to build.gradle when tokenizers are enabled --- android/build.gradle | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/android/build.gradle b/android/build.gradle index fd98bcf9..632d76f1 100644 --- a/android/build.gradle +++ b/android/build.gradle @@ -93,6 +93,11 @@ if(useSqliteVec) { println "[OP-SQLITE] Sqlite Vec enabled! ↗️" } + +if (!tokenizers.isEmpty()) { + println "[OP-SQLITE] Tokenizers enabled! 🧾 Tokenizers: " + tokenizers +} + if (isNewArchitectureEnabled()) { apply plugin: "com.facebook.react" } From 4844bdf8ae650b37cbea7a88832e0f8089d6e025 Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Mon, 18 Nov 2024 18:59:46 +0700 Subject: [PATCH 37/43] CI --- scripts/poll-in-app-server.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/scripts/poll-in-app-server.js b/scripts/poll-in-app-server.js index b0c508f9..52d4e68a 100644 --- a/scripts/poll-in-app-server.js +++ b/scripts/poll-in-app-server.js @@ -10,16 +10,19 @@ async function pollInAppServer() { const response = await makeHttpRequest('http://127.0.0.1:9000/results'); if (response !== null) { - let parsed_response = JSON.parse(response); - const allTestsPassed = parsed_response.results.reduce((acc, r) => { + let parsedResponse = JSON.parse(response); + const allTestsPassed = parsedResponse.results.reduce((acc, r) => { + console.log(r.description); return acc && r.type !== 'incorrect'; }, true); if (allTestsPassed) { - console.log('🟒🟒🟒🟒🟒 All tests passed!'); + console.log( + `🟒🟒🟒🟒🟒 ${parsedResponse.results.length} tests passed!` + ); process.exit(0); } else { - parsed_response.results.forEach((r) => { + parsedResponse.results.forEach((r) => { if (r.type === 'incorrect') { console.log(`πŸŸ₯Failed: ${JSON.stringify(r, null, 2)}`); } From ccd79a46e793e9100176ae3a1053cbad682bb4e7 Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Mon, 18 Nov 2024 19:06:54 +0700 Subject: [PATCH 38/43] CI --- scripts/poll-in-app-server.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/poll-in-app-server.js b/scripts/poll-in-app-server.js index 52d4e68a..33129f85 100644 --- a/scripts/poll-in-app-server.js +++ b/scripts/poll-in-app-server.js @@ -11,8 +11,14 @@ async function pollInAppServer() { if (response !== null) { let parsedResponse = JSON.parse(response); + console.log('Received response:', parsedResponse); + + if (parsedResponse.results.length === 0) { + console.log('0 tests ran WTF'); + } + const allTestsPassed = parsedResponse.results.reduce((acc, r) => { - console.log(r.description); + console.log(`- ${r.description} : ${r.type}`); return acc && r.type !== 'incorrect'; }, true); From c2190fdb92a1dbced0637036d27ff499e4b6b52a Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Mon, 18 Nov 2024 19:17:10 +0700 Subject: [PATCH 39/43] CI --- scripts/poll-in-app-server.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/poll-in-app-server.js b/scripts/poll-in-app-server.js index 33129f85..1c266f73 100644 --- a/scripts/poll-in-app-server.js +++ b/scripts/poll-in-app-server.js @@ -14,7 +14,7 @@ async function pollInAppServer() { console.log('Received response:', parsedResponse); if (parsedResponse.results.length === 0) { - console.log('0 tests ran WTF'); + continue; } const allTestsPassed = parsedResponse.results.reduce((acc, r) => { From 33404b511c92c0b377d7147363f0ff5345a76dbf Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Mon, 18 Nov 2024 19:24:19 +0700 Subject: [PATCH 40/43] Set server error --- example/src/App.tsx | 19 ++++++++++++++----- example/src/server.ts | 7 ++++++- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/example/src/App.tsx b/example/src/App.tsx index 80d1022b..0f510ef3 100644 --- a/example/src/App.tsx +++ b/example/src/App.tsx @@ -13,7 +13,12 @@ import RNRestart from 'react-native-restart'; import Share from 'react-native-share'; import 'reflect-metadata'; import {createLargeDB, queryLargeDB} from './Database'; -import {setServerResults, startServer, stopServer} from './server'; +import { + setServerError, + setServerResults, + startServer, + stopServer, +} from './server'; import {constantsTests} from './tests/constants.spec'; import {registerHooksTests} from './tests/hooks.spec'; import {blobTests, dbSetupTests, queriesTests, runTests} from './tests/index'; @@ -40,10 +45,14 @@ export default function App() { constantsTests, reactiveTests, tokenizerTests, - ).then(results => { - setServerResults(results as any); - setResults(results); - }); + ) + .then(results => { + setServerResults(results as any); + setResults(results); + }) + .catch(e => { + setServerError(e); + }); startServer(); diff --git a/example/src/server.ts b/example/src/server.ts index 5adf82b4..c5451896 100644 --- a/example/src/server.ts +++ b/example/src/server.ts @@ -4,7 +4,6 @@ let results: any[] = []; const server = new BridgeServer('http_service', true); server.get('/ping', async (_req, _res) => { - // console.log("🟦 🟦🟦🟦🟦🟦🟦 🟦 Received request for '/ping'"); return {message: 'pong'}; }); @@ -23,5 +22,11 @@ export function stopServer() { } export function setServerResults(r: any[]) { + console.log('Setting server results'); results = r; } + +export function setServerError(e: any) { + console.log('Setting server error'); + results = e; +} From 8675ec379ac1aff3470f92ffcab0763aad6ee997 Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Mon, 18 Nov 2024 19:25:13 +0700 Subject: [PATCH 41/43] Fix tests --- scripts/poll-in-app-server.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/poll-in-app-server.js b/scripts/poll-in-app-server.js index 1c266f73..9fecb135 100644 --- a/scripts/poll-in-app-server.js +++ b/scripts/poll-in-app-server.js @@ -11,8 +11,8 @@ async function pollInAppServer() { if (response !== null) { let parsedResponse = JSON.parse(response); - console.log('Received response:', parsedResponse); + // Wait until some results are returned if (parsedResponse.results.length === 0) { continue; } From 109541f2f1d4894e7ba866bb417141a58d360732 Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Mon, 18 Nov 2024 19:26:15 +0700 Subject: [PATCH 42/43] Disable tokenizer tests for libsql --- example/src/tests/tokenizer.spec.ts | 34 ++++++++++++++++------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/example/src/tests/tokenizer.spec.ts b/example/src/tests/tokenizer.spec.ts index 66d8d859..21534a4e 100644 --- a/example/src/tests/tokenizer.spec.ts +++ b/example/src/tests/tokenizer.spec.ts @@ -1,4 +1,4 @@ -import {open, type DB} from '@op-engineering/op-sqlite'; +import {isLibsql, open, type DB} from '@op-engineering/op-sqlite'; import chai from 'chai'; import {afterEach, beforeEach, describe, it} from './MochaRNAdapter'; @@ -14,9 +14,11 @@ export function tokenizerTests() { encryptionKey: 'test', }); - await db.execute( - `CREATE VIRTUAL TABLE tokenizer_table USING fts5(content, tokenize = 'wordtokenizer');`, - ); + if (!isLibsql()) { + await db.execute( + `CREATE VIRTUAL TABLE tokenizer_table USING fts5(content, tokenize = 'wordtokenizer');`, + ); + } }); afterEach(() => { @@ -28,16 +30,18 @@ export function tokenizerTests() { } }); - it('Should match the word split by the tokenizer', async () => { - await db.execute('INSERT INTO tokenizer_table(content) VALUES (?)', [ - 'This is a test document', - ]); - const res = await db.execute( - 'SELECT content FROM tokenizer_table WHERE content MATCH ?', - ['test'], - ); - expect(res.rows.length).to.be.equal(1); - expect(res.rows[0]!.content).to.be.equal('This is a test document'); - }); + if (!isLibsql()) { + it('Should match the word split by the tokenizer', async () => { + await db.execute('INSERT INTO tokenizer_table(content) VALUES (?)', [ + 'This is a test document', + ]); + const res = await db.execute( + 'SELECT content FROM tokenizer_table WHERE content MATCH ?', + ['test'], + ); + expect(res.rows.length).to.be.equal(1); + expect(res.rows[0]!.content).to.be.equal('This is a test document'); + }); + } }); } From 52cdf259d4e3e4754729b302db509bd55a3d9c8b Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Mon, 18 Nov 2024 21:34:38 +0700 Subject: [PATCH 43/43] Turn off libsql --- example/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/package.json b/example/package.json index 68de311a..42cf6b87 100644 --- a/example/package.json +++ b/example/package.json @@ -73,7 +73,7 @@ "iosSqlite": false, "fts5": true, "rtree": true, - "libsql": true, + "libsql": false, "sqliteVec": true, "tokenizers": [ "wordtokenizer",