From 2c2c75a4596d961836858ef69f289ac8431c3707 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 7 Nov 2024 01:17:10 +0400 Subject: [PATCH 01/34] Try to strip code --- .../Native/lingua-ffi/Cargo.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/LanguageIdentification.Lingua.Native/Native/lingua-ffi/Cargo.toml b/src/LanguageIdentification.Lingua.Native/Native/lingua-ffi/Cargo.toml index 5b6f7984..3f99c6f8 100644 --- a/src/LanguageIdentification.Lingua.Native/Native/lingua-ffi/Cargo.toml +++ b/src/LanguageIdentification.Lingua.Native/Native/lingua-ffi/Cargo.toml @@ -9,4 +9,7 @@ libc = "0.2.157" [lib] name = "lingua" -crate-type = ["staticlib", "cdylib"] \ No newline at end of file +crate-type = ["staticlib", "cdylib"] + +[profile.release] +strip = true From a889d13492d347cfa342392b6557ea50ccf92c95 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 7 Nov 2024 01:25:25 +0400 Subject: [PATCH 02/34] Revert "Try to strip code" This reverts commit 2c2c75a4596d961836858ef69f289ac8431c3707. --- .../Native/lingua-ffi/Cargo.toml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/LanguageIdentification.Lingua.Native/Native/lingua-ffi/Cargo.toml b/src/LanguageIdentification.Lingua.Native/Native/lingua-ffi/Cargo.toml index 3f99c6f8..5b6f7984 100644 --- a/src/LanguageIdentification.Lingua.Native/Native/lingua-ffi/Cargo.toml +++ b/src/LanguageIdentification.Lingua.Native/Native/lingua-ffi/Cargo.toml @@ -9,7 +9,4 @@ libc = "0.2.157" [lib] name = "lingua" -crate-type = ["staticlib", "cdylib"] - -[profile.release] -strip = true +crate-type = ["staticlib", "cdylib"] \ No newline at end of file From c8e58c47a5b9ec456bc74bba3f7ab5ade057f18a Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 7 Nov 2024 01:58:24 +0400 Subject: [PATCH 03/34] Update run-build.sh --- .../Scripts/run-build.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/LanguageIdentification.Lingua.Native/Scripts/run-build.sh b/src/LanguageIdentification.Lingua.Native/Scripts/run-build.sh index 0ff8b100..43f84d83 100644 --- a/src/LanguageIdentification.Lingua.Native/Scripts/run-build.sh +++ b/src/LanguageIdentification.Lingua.Native/Scripts/run-build.sh @@ -18,9 +18,8 @@ ls -R . cd .. -find "$workspace/target/release" -name "liblingua.so" -exec cp {} liblingua.so \; -rm -rf "$workspace" -ldd liblingua.so +cp $workspace/target/release/liblingua.so libfasttext.so +ldd libfasttext.so echo "Goodbye world"; From 79954af2b7a287469e1e62229ae62f03be7fe38e Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 7 Nov 2024 02:00:44 +0400 Subject: [PATCH 04/34] Update Cargo.toml --- .../Native/lingua-ffi/Cargo.toml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/LanguageIdentification.Lingua.Native/Native/lingua-ffi/Cargo.toml b/src/LanguageIdentification.Lingua.Native/Native/lingua-ffi/Cargo.toml index 5b6f7984..5a599682 100644 --- a/src/LanguageIdentification.Lingua.Native/Native/lingua-ffi/Cargo.toml +++ b/src/LanguageIdentification.Lingua.Native/Native/lingua-ffi/Cargo.toml @@ -9,4 +9,12 @@ libc = "0.2.157" [lib] name = "lingua" -crate-type = ["staticlib", "cdylib"] \ No newline at end of file + +[target.'cfg(target_os = "linux")'.lib] +crate-type = ["cdylib"] + +[target.'cfg(target_os = "windows")'.lib] +crate-type = ["staticlib"] + +[target.'cfg(target_os = "macos")'.lib] +crate-type = ["staticlib", "cdylib"] From 530809142583d5a16792e07f2b2b6c19d6e70682 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Fri, 8 Nov 2024 00:28:27 +0400 Subject: [PATCH 05/34] Update run-build.sh --- .../Scripts/run-build.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/LanguageIdentification.Lingua.Native/Scripts/run-build.sh b/src/LanguageIdentification.Lingua.Native/Scripts/run-build.sh index 43f84d83..57e7a2a0 100644 --- a/src/LanguageIdentification.Lingua.Native/Scripts/run-build.sh +++ b/src/LanguageIdentification.Lingua.Native/Scripts/run-build.sh @@ -18,8 +18,9 @@ ls -R . cd .. -cp $workspace/target/release/liblingua.so libfasttext.so -ldd libfasttext.so +cd $workspace/target/release +ldd liblingua.so +cp liblingua.so ../../liblingua.so echo "Goodbye world"; From 7809d2cf42f1382bcb69acee8aebd2f8491f678d Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Fri, 8 Nov 2024 00:29:02 +0400 Subject: [PATCH 06/34] Lingua: add Windows and MacOS to IsSupported() --- src/LanguageIdentification.Lingua/LinguaDetector.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/LanguageIdentification.Lingua/LinguaDetector.cs b/src/LanguageIdentification.Lingua/LinguaDetector.cs index 3a1808e4..4a5c27ad 100644 --- a/src/LanguageIdentification.Lingua/LinguaDetector.cs +++ b/src/LanguageIdentification.Lingua/LinguaDetector.cs @@ -35,6 +35,8 @@ public static bool IsSupported() return RuntimeInformation.OSArchitecture switch { Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.Linux) => true, + Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.Windows) => true, + Architecture.Arm64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, _ => false, }; } From 8e0b857fc31a44645e16364d0d2c6e9f9486bc8e Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Fri, 8 Nov 2024 01:07:08 +0400 Subject: [PATCH 07/34] Update Cargo.toml --- .../Native/lingua-ffi/Cargo.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/LanguageIdentification.Lingua.Native/Native/lingua-ffi/Cargo.toml b/src/LanguageIdentification.Lingua.Native/Native/lingua-ffi/Cargo.toml index 5a599682..826b91fa 100644 --- a/src/LanguageIdentification.Lingua.Native/Native/lingua-ffi/Cargo.toml +++ b/src/LanguageIdentification.Lingua.Native/Native/lingua-ffi/Cargo.toml @@ -9,6 +9,7 @@ libc = "0.2.157" [lib] name = "lingua" +crate-type = ["cdylib"] [target.'cfg(target_os = "linux")'.lib] crate-type = ["cdylib"] @@ -17,4 +18,4 @@ crate-type = ["cdylib"] crate-type = ["staticlib"] [target.'cfg(target_os = "macos")'.lib] -crate-type = ["staticlib", "cdylib"] +crate-type = ["staticlib", "cdylib"] \ No newline at end of file From be7e3254a361aa4f9b174dd008c7580560c7fe5e Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Fri, 8 Nov 2024 01:07:12 +0400 Subject: [PATCH 08/34] Update run-build.sh --- src/LanguageIdentification.Lingua.Native/Scripts/run-build.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/LanguageIdentification.Lingua.Native/Scripts/run-build.sh b/src/LanguageIdentification.Lingua.Native/Scripts/run-build.sh index 57e7a2a0..4fc63529 100644 --- a/src/LanguageIdentification.Lingua.Native/Scripts/run-build.sh +++ b/src/LanguageIdentification.Lingua.Native/Scripts/run-build.sh @@ -16,9 +16,7 @@ cargo build --release ls -R . -cd .. - -cd $workspace/target/release +cd ./target/release ldd liblingua.so cp liblingua.so ../../liblingua.so From 698c0323056a8f2c581f85a2a4d3d13aea5d9a1e Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Fri, 8 Nov 2024 01:17:27 +0400 Subject: [PATCH 09/34] Fixes --- src/LanguageIdentification.Lingua.Native/Scripts/run-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LanguageIdentification.Lingua.Native/Scripts/run-build.sh b/src/LanguageIdentification.Lingua.Native/Scripts/run-build.sh index 4fc63529..7e55d50b 100644 --- a/src/LanguageIdentification.Lingua.Native/Scripts/run-build.sh +++ b/src/LanguageIdentification.Lingua.Native/Scripts/run-build.sh @@ -18,7 +18,7 @@ ls -R . cd ./target/release ldd liblingua.so -cp liblingua.so ../../liblingua.so +cp liblingua.so ../../../liblingua.so echo "Goodbye world"; From 6ae2a8c3c9bfea13c7e4021a23f0097562f3d619 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Fri, 8 Nov 2024 01:29:04 +0400 Subject: [PATCH 10/34] Lingua: try to build for MacOS --- .github/workflows/github-ci.yml | 1 + .../Scripts/run-build.macos.sh | 45 +++++++++++++++++++ .../Scripts/run-build.sh | 2 + 3 files changed, 48 insertions(+) create mode 100644 src/LanguageIdentification.Lingua.Native/Scripts/run-build.macos.sh diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index c8624c30..33a133ef 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -22,6 +22,7 @@ jobs: - { title: CLD2, projectName: LanguageIdentification.CLD2.Native, os: macos-14, arch: arm64, script: ./Scripts/run-build.macos.sh, artifact: libcld2.arm64.dylib } - { title: CLD3, projectName: LanguageIdentification.CLD3.Native, os: macos-13, arch: x86_64, script: ./Scripts/run-build.macos.sh, artifact: libcld3.x86_64.dylib } - { title: CLD3, projectName: LanguageIdentification.CLD3.Native, os: macos-14, arch: arm64, script: ./Scripts/run-build.macos.sh, artifact: libcld3.arm64.dylib } + - { title: Lingua, projectName: LanguageIdentification.Lingua.Native, os: macos-14, arch: arm64, script: ./Scripts/run-build.macos.sh, artifact: liblingua.arm64.dylib } permissions: contents: read packages: write diff --git a/src/LanguageIdentification.Lingua.Native/Scripts/run-build.macos.sh b/src/LanguageIdentification.Lingua.Native/Scripts/run-build.macos.sh new file mode 100644 index 00000000..890ec4d2 --- /dev/null +++ b/src/LanguageIdentification.Lingua.Native/Scripts/run-build.macos.sh @@ -0,0 +1,45 @@ +#!/bin/bash +set -e + +if [ -z "$1" ]; then + echo "Error: No architecture specified." + echo "Usage: $0 " + exit 1 +fi + +ARCH=$1 + +if [[ "$ARCH" != "x86_64" && "$ARCH" != "arm64" ]]; then + echo "Error: Invalid architecture specified. Use 'x86_64' or 'arm64'." + exit 1 +fi + +echo "Hello world $ARCH"; + +brew install llvm + +workspace="build_temp" + +mkdir "$workspace" -p +cp -a Native/lingua-ffi/. $workspace/. + +ls -R . + +cd "$workspace" + +cargo build --release + +echo "Build for MacOS on $ARCH"; +rm -rf * +cmake -DCMAKE_OSX_ARCHITECTURES=$ARCH .. +make -j $(sysctl -n hw.logicalcpu) + +ls -R + +cd ./target/release +otool -L liblingua.dylib +cp liblingua.dylib ../../liblingua.$ARCH.dylib + +# Clean up +rm -rf "$workspace" +echo "Goodbye world"; diff --git a/src/LanguageIdentification.Lingua.Native/Scripts/run-build.sh b/src/LanguageIdentification.Lingua.Native/Scripts/run-build.sh index 7e55d50b..43fd1d97 100644 --- a/src/LanguageIdentification.Lingua.Native/Scripts/run-build.sh +++ b/src/LanguageIdentification.Lingua.Native/Scripts/run-build.sh @@ -20,5 +20,7 @@ cd ./target/release ldd liblingua.so cp liblingua.so ../../../liblingua.so +# Clean up +rm -rf "$workspace" echo "Goodbye world"; From 020595ac914bce86c2fbc7f5d85402737b6b8955 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Fri, 8 Nov 2024 01:35:53 +0400 Subject: [PATCH 11/34] Update run-build.macos.sh --- .../Scripts/run-build.macos.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/LanguageIdentification.Lingua.Native/Scripts/run-build.macos.sh b/src/LanguageIdentification.Lingua.Native/Scripts/run-build.macos.sh index 890ec4d2..d3ccfd06 100644 --- a/src/LanguageIdentification.Lingua.Native/Scripts/run-build.macos.sh +++ b/src/LanguageIdentification.Lingua.Native/Scripts/run-build.macos.sh @@ -27,12 +27,8 @@ ls -R . cd "$workspace" -cargo build --release - echo "Build for MacOS on $ARCH"; -rm -rf * -cmake -DCMAKE_OSX_ARCHITECTURES=$ARCH .. -make -j $(sysctl -n hw.logicalcpu) +cargo build --release ls -R From 82e4b9523e09826bf9db09b5284696d80c6ab47d Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Fri, 8 Nov 2024 01:44:36 +0400 Subject: [PATCH 12/34] Lingua: build for windows --- .github/workflows/github-ci.yml | 1 + .../Scripts/run-build.macos.sh | 2 +- .../Scripts/run-build.ps1 | 31 +++++++++++++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 src/LanguageIdentification.Lingua.Native/Scripts/run-build.ps1 diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 33a133ef..e4e999e5 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -22,6 +22,7 @@ jobs: - { title: CLD2, projectName: LanguageIdentification.CLD2.Native, os: macos-14, arch: arm64, script: ./Scripts/run-build.macos.sh, artifact: libcld2.arm64.dylib } - { title: CLD3, projectName: LanguageIdentification.CLD3.Native, os: macos-13, arch: x86_64, script: ./Scripts/run-build.macos.sh, artifact: libcld3.x86_64.dylib } - { title: CLD3, projectName: LanguageIdentification.CLD3.Native, os: macos-14, arch: arm64, script: ./Scripts/run-build.macos.sh, artifact: libcld3.arm64.dylib } + - { title: Lingua, projectName: LanguageIdentification.Lingua.Native, os: windows-latest, arch: x86_64, script: ./Scripts/run-build.ps1, artifact: lingua.dll } - { title: Lingua, projectName: LanguageIdentification.Lingua.Native, os: macos-14, arch: arm64, script: ./Scripts/run-build.macos.sh, artifact: liblingua.arm64.dylib } permissions: contents: read diff --git a/src/LanguageIdentification.Lingua.Native/Scripts/run-build.macos.sh b/src/LanguageIdentification.Lingua.Native/Scripts/run-build.macos.sh index d3ccfd06..d7e80e24 100644 --- a/src/LanguageIdentification.Lingua.Native/Scripts/run-build.macos.sh +++ b/src/LanguageIdentification.Lingua.Native/Scripts/run-build.macos.sh @@ -34,7 +34,7 @@ ls -R cd ./target/release otool -L liblingua.dylib -cp liblingua.dylib ../../liblingua.$ARCH.dylib +cp liblingua.dylib ../../../liblingua.$ARCH.dylib # Clean up rm -rf "$workspace" diff --git a/src/LanguageIdentification.Lingua.Native/Scripts/run-build.ps1 b/src/LanguageIdentification.Lingua.Native/Scripts/run-build.ps1 new file mode 100644 index 00000000..b8d584a1 --- /dev/null +++ b/src/LanguageIdentification.Lingua.Native/Scripts/run-build.ps1 @@ -0,0 +1,31 @@ +# PowerShell equivalent script +# Ensure script stops if any command fails +$ErrorActionPreference = "Stop" + +Write-Output "Hello world" + +$workspace = "build_temp" + +# Copy directories +Copy-Item -Path "Native\lingua-ffi\." -Destination $workspace -Recurse -Force + +# List directory contents recursively +Get-ChildItem -Recurse -Path . + +Set-Location $workspace + +# Build for Windows +cargo build --release + +# List directory contents recursively +Get-ChildItem -Recurse -Path . + +# Display shared library dependencies +Copy-Item -Path ".\target\release\liblingua.dll" -Destination "../../../liblingua.dll" + +# List directory contents recursively +Get-ChildItem -Recurse -Path . + +# Clean up +cd ../.. +Write-Output "Goodbye world" \ No newline at end of file From 67937f8a8d5da5fb117a06544c5fbe65c2911b51 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Fri, 8 Nov 2024 01:48:43 +0400 Subject: [PATCH 13/34] Update run-build.ps1 --- src/LanguageIdentification.Lingua.Native/Scripts/run-build.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LanguageIdentification.Lingua.Native/Scripts/run-build.ps1 b/src/LanguageIdentification.Lingua.Native/Scripts/run-build.ps1 index b8d584a1..5d4a61f9 100644 --- a/src/LanguageIdentification.Lingua.Native/Scripts/run-build.ps1 +++ b/src/LanguageIdentification.Lingua.Native/Scripts/run-build.ps1 @@ -21,7 +21,7 @@ cargo build --release Get-ChildItem -Recurse -Path . # Display shared library dependencies -Copy-Item -Path ".\target\release\liblingua.dll" -Destination "../../../liblingua.dll" +Copy-Item -Path ".\target\release\lingua.dll" -Destination "../../../lingua.dll" # List directory contents recursively Get-ChildItem -Recurse -Path . From ed42831b93e4f314b826644502a499ff8d191e03 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Fri, 8 Nov 2024 01:56:06 +0400 Subject: [PATCH 14/34] Lingua: fixes --- .../Scripts/run-build.ps1 | 2 +- ...anguageIdentification.Lingua.Native.csproj | 20 +++++++++++++++++++ .../Scripts/run-build.ps1 | 2 +- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/LanguageIdentification.FastText.Native/Scripts/run-build.ps1 b/src/LanguageIdentification.FastText.Native/Scripts/run-build.ps1 index c34c510c..cc906c59 100644 --- a/src/LanguageIdentification.FastText.Native/Scripts/run-build.ps1 +++ b/src/LanguageIdentification.FastText.Native/Scripts/run-build.ps1 @@ -35,7 +35,7 @@ cmake --build . Get-ChildItem -Recurse -Path . # Display shared library dependencies -Copy-Item -Path ".\Debug\fasttext.dll" -Destination "../../fasttext.dll" +Copy-Item -Path ".\Debug\fasttext.dll" -Destination "..\..\fasttext.dll" # List directory contents recursively Get-ChildItem -Recurse -Path . diff --git a/src/LanguageIdentification.Lingua.Native/LanguageIdentification.Lingua.Native.csproj b/src/LanguageIdentification.Lingua.Native/LanguageIdentification.Lingua.Native.csproj index a72f98f8..1d6288ea 100644 --- a/src/LanguageIdentification.Lingua.Native/LanguageIdentification.Lingua.Native.csproj +++ b/src/LanguageIdentification.Lingua.Native/LanguageIdentification.Lingua.Native.csproj @@ -61,6 +61,26 @@ + + + PreserveNewest + true + runtimes/win-x64/native + true + false + + + + + + PreserveNewest + true + runtimes/osx-arm64/native/liblingua.dylib + true + false + + + diff --git a/src/LanguageIdentification.Lingua.Native/Scripts/run-build.ps1 b/src/LanguageIdentification.Lingua.Native/Scripts/run-build.ps1 index 5d4a61f9..a75d22c9 100644 --- a/src/LanguageIdentification.Lingua.Native/Scripts/run-build.ps1 +++ b/src/LanguageIdentification.Lingua.Native/Scripts/run-build.ps1 @@ -21,7 +21,7 @@ cargo build --release Get-ChildItem -Recurse -Path . # Display shared library dependencies -Copy-Item -Path ".\target\release\lingua.dll" -Destination "../../../lingua.dll" +Copy-Item -Path ".\target\release\lingua.dll" -Destination "..\..\..\lingua.dll" # List directory contents recursively Get-ChildItem -Recurse -Path . From cc2fd52e6f6ffc759165db8cf9ca3a68d4e9f4f3 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Fri, 8 Nov 2024 02:01:43 +0400 Subject: [PATCH 15/34] Update run-build.ps1 --- src/LanguageIdentification.Lingua.Native/Scripts/run-build.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LanguageIdentification.Lingua.Native/Scripts/run-build.ps1 b/src/LanguageIdentification.Lingua.Native/Scripts/run-build.ps1 index a75d22c9..6042b5fc 100644 --- a/src/LanguageIdentification.Lingua.Native/Scripts/run-build.ps1 +++ b/src/LanguageIdentification.Lingua.Native/Scripts/run-build.ps1 @@ -24,8 +24,8 @@ Get-ChildItem -Recurse -Path . Copy-Item -Path ".\target\release\lingua.dll" -Destination "..\..\..\lingua.dll" # List directory contents recursively +cd ..\..\.. Get-ChildItem -Recurse -Path . # Clean up -cd ../.. Write-Output "Goodbye world" \ No newline at end of file From c940e62e4222cd664b08ba942813ff03dc99e90a Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Fri, 8 Nov 2024 02:24:41 +0400 Subject: [PATCH 16/34] Update run-build.ps1 --- .../Scripts/run-build.ps1 | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/LanguageIdentification.Lingua.Native/Scripts/run-build.ps1 b/src/LanguageIdentification.Lingua.Native/Scripts/run-build.ps1 index 6042b5fc..5b562166 100644 --- a/src/LanguageIdentification.Lingua.Native/Scripts/run-build.ps1 +++ b/src/LanguageIdentification.Lingua.Native/Scripts/run-build.ps1 @@ -21,11 +21,7 @@ cargo build --release Get-ChildItem -Recurse -Path . # Display shared library dependencies -Copy-Item -Path ".\target\release\lingua.dll" -Destination "..\..\..\lingua.dll" - -# List directory contents recursively -cd ..\..\.. -Get-ChildItem -Recurse -Path . +Copy-Item -Path ".\target\release\lingua.dll" -Destination "..\lingua.dll" # Clean up Write-Output "Goodbye world" \ No newline at end of file From a84e6142537caac7b3e676a0522b76930a6cea32 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Fri, 8 Nov 2024 02:40:28 +0400 Subject: [PATCH 17/34] Lingua: split LinguaPredictionResult.cs --- .../Internal/LinguaPredictionListResult.cs | 12 +++++++ .../LinguaPredictionRangeListResult.cs | 12 +++++++ .../Internal/LinguaPredictionRangeResult.cs | 25 +++++++++++++ .../Internal/LinguaPredictionResult.cs | 35 ------------------- 4 files changed, 49 insertions(+), 35 deletions(-) create mode 100644 src/LanguageIdentification.Lingua/Internal/LinguaPredictionListResult.cs create mode 100644 src/LanguageIdentification.Lingua/Internal/LinguaPredictionRangeListResult.cs create mode 100644 src/LanguageIdentification.Lingua/Internal/LinguaPredictionRangeResult.cs diff --git a/src/LanguageIdentification.Lingua/Internal/LinguaPredictionListResult.cs b/src/LanguageIdentification.Lingua/Internal/LinguaPredictionListResult.cs new file mode 100644 index 00000000..969392fd --- /dev/null +++ b/src/LanguageIdentification.Lingua/Internal/LinguaPredictionListResult.cs @@ -0,0 +1,12 @@ +using System; +using System.Runtime.InteropServices; + +namespace Panlingo.LanguageIdentification.Lingua.Internal +{ + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] + internal readonly struct LinguaPredictionListResult + { + public readonly IntPtr Predictions; + public readonly uint PredictionsCount; + } +} diff --git a/src/LanguageIdentification.Lingua/Internal/LinguaPredictionRangeListResult.cs b/src/LanguageIdentification.Lingua/Internal/LinguaPredictionRangeListResult.cs new file mode 100644 index 00000000..2181d3df --- /dev/null +++ b/src/LanguageIdentification.Lingua/Internal/LinguaPredictionRangeListResult.cs @@ -0,0 +1,12 @@ +using System; +using System.Runtime.InteropServices; + +namespace Panlingo.LanguageIdentification.Lingua.Internal +{ + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] + internal readonly struct LinguaPredictionRangeListResult + { + public readonly IntPtr Predictions; + public readonly uint PredictionsCount; + } +} diff --git a/src/LanguageIdentification.Lingua/Internal/LinguaPredictionRangeResult.cs b/src/LanguageIdentification.Lingua/Internal/LinguaPredictionRangeResult.cs new file mode 100644 index 00000000..9bc5ed62 --- /dev/null +++ b/src/LanguageIdentification.Lingua/Internal/LinguaPredictionRangeResult.cs @@ -0,0 +1,25 @@ +using System.Runtime.InteropServices; + +namespace Panlingo.LanguageIdentification.Lingua.Internal +{ + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] + internal readonly struct LinguaPredictionRangeResult + { + [MarshalAs(UnmanagedType.U1)] + public readonly LinguaLanguage Language; + + [MarshalAs(UnmanagedType.R8)] + public readonly double Confidence; + + public readonly uint StartIndex; + + public readonly uint EndIndex; + + public readonly uint WordCount; + + public static explicit operator LinguaPredictionResult(LinguaPredictionRangeResult item) + { + return new LinguaPredictionResult(item.Language, item.Confidence); + } + } +} diff --git a/src/LanguageIdentification.Lingua/Internal/LinguaPredictionResult.cs b/src/LanguageIdentification.Lingua/Internal/LinguaPredictionResult.cs index f0183ca0..1e024ca4 100644 --- a/src/LanguageIdentification.Lingua/Internal/LinguaPredictionResult.cs +++ b/src/LanguageIdentification.Lingua/Internal/LinguaPredictionResult.cs @@ -18,39 +18,4 @@ public LinguaPredictionResult(LinguaLanguage language, double confidence) Confidence = confidence; } } - - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal readonly struct LinguaPredictionRangeResult - { - [MarshalAs(UnmanagedType.U1)] - public readonly LinguaLanguage Language; - - [MarshalAs(UnmanagedType.R8)] - public readonly double Confidence; - - public readonly uint StartIndex; - - public readonly uint EndIndex; - - public readonly uint WordCount; - - public static explicit operator LinguaPredictionResult(LinguaPredictionRangeResult item) - { - return new LinguaPredictionResult(item.Language, item.Confidence); - } - } - - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal readonly struct LinguaPredictionListResult - { - public readonly IntPtr Predictions; - public readonly uint PredictionsCount; - } - - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal readonly struct LinguaPredictionRangeListResult - { - public readonly IntPtr Predictions; - public readonly uint PredictionsCount; - } } From c99a9d6b1b7fdcd6eaf327be08e42a0736ae966f Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Fri, 8 Nov 2024 03:16:47 +0400 Subject: [PATCH 18/34] Lingua: windows build fixes (0xc0000374) --- .../Program.cs | 4 ++-- .../Properties/launchSettings.json | 8 ++++++++ src/LanguageIdentification.Lingua.Native/.gitignore | 2 ++ .../Internal/LinguaDetectorWrapper.cs | 11 +++++++---- src/LanguageIdentification.Lingua/LinguaDetector.cs | 10 ++++++++++ 5 files changed, 29 insertions(+), 6 deletions(-) diff --git a/src/LanguageIdentification.Lingua.ConsoleTest/Program.cs b/src/LanguageIdentification.Lingua.ConsoleTest/Program.cs index 296a85e1..441156a4 100644 --- a/src/LanguageIdentification.Lingua.ConsoleTest/Program.cs +++ b/src/LanguageIdentification.Lingua.ConsoleTest/Program.cs @@ -45,8 +45,8 @@ static void Main(string[] args) var code1 = lingua.GetLanguageCode(LinguaLanguage.Ukrainian, LinguaLanguageCode.Alpha2); var code2 = lingua.GetLanguageCode(LinguaLanguage.Ukrainian, LinguaLanguageCode.Alpha3); - var code3 = lingua.GetLanguageCode(LinguaLanguage.Ukrainian, (LinguaLanguageCode)99); - var code4 = lingua.GetLanguageCode((LinguaLanguage)255, LinguaLanguageCode.Alpha3); + // var code3 = lingua.GetLanguageCode(LinguaLanguage.Ukrainian, (LinguaLanguageCode)99); + // var code4 = lingua.GetLanguageCode((LinguaLanguage)255, LinguaLanguageCode.Alpha3); ; diff --git a/src/LanguageIdentification.Lingua.ConsoleTest/Properties/launchSettings.json b/src/LanguageIdentification.Lingua.ConsoleTest/Properties/launchSettings.json index 65b8965a..515a8d0d 100644 --- a/src/LanguageIdentification.Lingua.ConsoleTest/Properties/launchSettings.json +++ b/src/LanguageIdentification.Lingua.ConsoleTest/Properties/launchSettings.json @@ -2,6 +2,14 @@ "profiles": { "Docker": { "commandName": "Docker" + }, + "WSL": { + "commandName": "WSL2", + "distributionName": "" + }, + "Project": { + "commandName": "Project", + "nativeDebugging": true } } } \ No newline at end of file diff --git a/src/LanguageIdentification.Lingua.Native/.gitignore b/src/LanguageIdentification.Lingua.Native/.gitignore index 0a6c6231..c43b1fba 100644 --- a/src/LanguageIdentification.Lingua.Native/.gitignore +++ b/src/LanguageIdentification.Lingua.Native/.gitignore @@ -1,2 +1,4 @@ liblingua.so +liblingua.arm64.dylib +lingua.dll build_temp/** diff --git a/src/LanguageIdentification.Lingua/Internal/LinguaDetectorWrapper.cs b/src/LanguageIdentification.Lingua/Internal/LinguaDetectorWrapper.cs index 832e938c..53e9d3ed 100644 --- a/src/LanguageIdentification.Lingua/Internal/LinguaDetectorWrapper.cs +++ b/src/LanguageIdentification.Lingua/Internal/LinguaDetectorWrapper.cs @@ -9,7 +9,10 @@ namespace Panlingo.LanguageIdentification.Lingua.Internal internal static class LinguaDetectorWrapper { [DllImport(LinguaNativeLibrary.Name, EntryPoint = "lingua_language_detector_builder_create", CallingConvention = CallingConvention.Cdecl)] - public static extern IntPtr LinguaLanguageDetectorBuilderCreate(LinguaLanguage[] languages, UIntPtr languageCount); + public static extern IntPtr LinguaLanguageDetectorBuilderCreate( + LinguaLanguage[] languages, + UIntPtr languageCount + ); [DllImport(LinguaNativeLibrary.Name, EntryPoint = "lingua_language_detector_builder_with_low_accuracy_mode", CallingConvention = CallingConvention.Cdecl)] public static extern IntPtr LinguaLanguageDetectorBuilderWithLowAccuracyMode(IntPtr builder); @@ -38,14 +41,14 @@ internal static class LinguaDetectorWrapper [DllImport(LinguaNativeLibrary.Name, EntryPoint = "lingua_detect_single", CallingConvention = CallingConvention.Cdecl)] public static extern LinguaStatus LinguaDetectSingle( IntPtr detector, - string text, + [MarshalAs(UnmanagedType.LPUTF8Str)] string text, out LinguaPredictionListResult result ); [DllImport(LinguaNativeLibrary.Name, EntryPoint = "lingua_detect_mixed", CallingConvention = CallingConvention.Cdecl)] public static extern LinguaStatus LinguaDetectMixed( IntPtr detector, - string text, + [MarshalAs(UnmanagedType.LPUTF8Str)] string text, out LinguaPredictionRangeListResult result ); @@ -53,7 +56,7 @@ out LinguaPredictionRangeListResult result public static extern int LinguaLangCode( LinguaLanguage lang, LinguaLanguageCode code, - [MarshalAs(UnmanagedType.LPStr)] StringBuilder buffer, + [MarshalAs(UnmanagedType.LPUTF8Str)] StringBuilder buffer, UIntPtr bufferSize ); } diff --git a/src/LanguageIdentification.Lingua/LinguaDetector.cs b/src/LanguageIdentification.Lingua/LinguaDetector.cs index 4a5c27ad..9f4ed4d9 100644 --- a/src/LanguageIdentification.Lingua/LinguaDetector.cs +++ b/src/LanguageIdentification.Lingua/LinguaDetector.cs @@ -137,6 +137,16 @@ public IEnumerable PredictMixedLanguages(string text) public string GetLanguageCode(LinguaLanguage language, LinguaLanguageCode code) { + if (!Enum.IsDefined(typeof(LinguaLanguage), language)) + { + throw new LinguaDetectorException($"Language code '{language}' is not found"); + } + + if (!Enum.IsDefined(typeof(LinguaLanguageCode), code)) + { + throw new LinguaDetectorException($"Language code type '{code}' is not found"); + } + var stringBuider = new StringBuilder(10); try From 4051eb8f8e12800053bfc3a86675b38a507a53c6 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Fri, 8 Nov 2024 09:37:50 +0400 Subject: [PATCH 19/34] Whatlang: using custom Cargo.toml --- .../Native/whatlang-ffi/Cargo.toml | 12 ++++++++++++ .../Scripts/run-build.sh | 5 +++-- 2 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 src/LanguageIdentification.Whatlang.Native/Native/whatlang-ffi/Cargo.toml diff --git a/src/LanguageIdentification.Whatlang.Native/Native/whatlang-ffi/Cargo.toml b/src/LanguageIdentification.Whatlang.Native/Native/whatlang-ffi/Cargo.toml new file mode 100644 index 00000000..ea60e50e --- /dev/null +++ b/src/LanguageIdentification.Whatlang.Native/Native/whatlang-ffi/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "whatlang-ffi" +version = "0.1.0" +edition = "2021" + +[dependencies] +whatlang = "0.16.4" +libc = "0.2.157" + +[lib] +name = "whatlang" +crate-type = ["staticlib", "cdylib"] \ No newline at end of file diff --git a/src/LanguageIdentification.Whatlang.Native/Scripts/run-build.sh b/src/LanguageIdentification.Whatlang.Native/Scripts/run-build.sh index 1d4be4c4..c8814365 100644 --- a/src/LanguageIdentification.Whatlang.Native/Scripts/run-build.sh +++ b/src/LanguageIdentification.Whatlang.Native/Scripts/run-build.sh @@ -7,6 +7,7 @@ workspace="build_temp" mkdir "$workspace" -p cp -a ../../third_party/whatlang-ffi/. $workspace/. +cp -a Native/whatlang-ffi/. $workspace/. ls -R . @@ -18,9 +19,9 @@ ls -R . cd .. -find "$workspace/target/release" -name "libwhatlang.so" -exec cp {} libwhatlang.so \; -rm -rf "$workspace" +cd ./target/release ldd libwhatlang.so +cp libwhatlang.so ../../../libwhatlang.so echo "Goodbye world"; From 21f3f56d4f985483b3109c6b037b50eecf01a3cf Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Fri, 8 Nov 2024 09:44:33 +0400 Subject: [PATCH 20/34] Update run-build.sh --- .../Scripts/run-build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/LanguageIdentification.Whatlang.Native/Scripts/run-build.sh b/src/LanguageIdentification.Whatlang.Native/Scripts/run-build.sh index c8814365..e15db692 100644 --- a/src/LanguageIdentification.Whatlang.Native/Scripts/run-build.sh +++ b/src/LanguageIdentification.Whatlang.Native/Scripts/run-build.sh @@ -17,11 +17,11 @@ cargo build --release ls -R . -cd .. - cd ./target/release ldd libwhatlang.so cp libwhatlang.so ../../../libwhatlang.so +# Clean up +rm -rf "$workspace" echo "Goodbye world"; From b4ff5b5525747dfefbe56902d8405e0012098095 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Fri, 8 Nov 2024 09:54:30 +0400 Subject: [PATCH 21/34] Whatlang: build for Windows and Mac --- .github/workflows/github-ci.yml | 3 ++ .../Program.cs | 1 - .../.gitignore | 2 + ...guageIdentification.Whatlang.Native.csproj | 30 +++++++++++++ .../Scripts/run-build.macos.sh | 42 +++++++++++++++++++ .../Scripts/run-build.ps1 | 28 +++++++++++++ .../WhatlangDetector.cs | 3 ++ 7 files changed, 108 insertions(+), 1 deletion(-) create mode 100644 src/LanguageIdentification.Whatlang.Native/Scripts/run-build.macos.sh create mode 100644 src/LanguageIdentification.Whatlang.Native/Scripts/run-build.ps1 diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index e4e999e5..b5a06b33 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -24,6 +24,9 @@ jobs: - { title: CLD3, projectName: LanguageIdentification.CLD3.Native, os: macos-14, arch: arm64, script: ./Scripts/run-build.macos.sh, artifact: libcld3.arm64.dylib } - { title: Lingua, projectName: LanguageIdentification.Lingua.Native, os: windows-latest, arch: x86_64, script: ./Scripts/run-build.ps1, artifact: lingua.dll } - { title: Lingua, projectName: LanguageIdentification.Lingua.Native, os: macos-14, arch: arm64, script: ./Scripts/run-build.macos.sh, artifact: liblingua.arm64.dylib } + - { title: Whatlang, projectName: LanguageIdentification.Whatlang.Native, os: windows-latest, arch: x86_64, script: ./Scripts/run-build.ps1, artifact: lingua.dll } + - { title: Whatlang, projectName: LanguageIdentification.Whatlang.Native, os: macos-13, arch: x86_64, script: ./Scripts/run-build.macos.sh, artifact: liblingua.x86_64.dylib } + - { title: Whatlang, projectName: LanguageIdentification.Whatlang.Native, os: macos-14, arch: arm64, script: ./Scripts/run-build.macos.sh, artifact: liblingua.arm64.dylib } permissions: contents: read packages: write diff --git a/src/LanguageIdentification.Lingua.ConsoleTest/Program.cs b/src/LanguageIdentification.Lingua.ConsoleTest/Program.cs index 441156a4..d95686bb 100644 --- a/src/LanguageIdentification.Lingua.ConsoleTest/Program.cs +++ b/src/LanguageIdentification.Lingua.ConsoleTest/Program.cs @@ -16,7 +16,6 @@ static void Main(string[] args) "Привіт", "Привет", "Hello", - "Hello, how are you? Привіт, як справи? Hello, how are you? Привет, как дела?", "Hello, how are you?", "Привіт, як справи?", "Привет, как дела?", diff --git a/src/LanguageIdentification.Whatlang.Native/.gitignore b/src/LanguageIdentification.Whatlang.Native/.gitignore index ec2a4c48..e04f0140 100644 --- a/src/LanguageIdentification.Whatlang.Native/.gitignore +++ b/src/LanguageIdentification.Whatlang.Native/.gitignore @@ -1,2 +1,4 @@ libwhatlang.so +libwhatlang.arm64.dylib +whatlang.dll build_temp/** diff --git a/src/LanguageIdentification.Whatlang.Native/LanguageIdentification.Whatlang.Native.csproj b/src/LanguageIdentification.Whatlang.Native/LanguageIdentification.Whatlang.Native.csproj index 3206f2d0..35bcab00 100644 --- a/src/LanguageIdentification.Whatlang.Native/LanguageIdentification.Whatlang.Native.csproj +++ b/src/LanguageIdentification.Whatlang.Native/LanguageIdentification.Whatlang.Native.csproj @@ -62,6 +62,36 @@ + + + PreserveNewest + true + runtimes/win-x64/native + true + false + + + + + + PreserveNewest + true + runtimes/osx-x64/native/libwhatlang.dylib + true + false + + + + + + PreserveNewest + true + runtimes/osx-arm64/native/libwhatlang.dylib + true + false + + + diff --git a/src/LanguageIdentification.Whatlang.Native/Scripts/run-build.macos.sh b/src/LanguageIdentification.Whatlang.Native/Scripts/run-build.macos.sh new file mode 100644 index 00000000..e993f673 --- /dev/null +++ b/src/LanguageIdentification.Whatlang.Native/Scripts/run-build.macos.sh @@ -0,0 +1,42 @@ +#!/bin/bash +set -e + +if [ -z "$1" ]; then + echo "Error: No architecture specified." + echo "Usage: $0 " + exit 1 +fi + +ARCH=$1 + +if [[ "$ARCH" != "x86_64" && "$ARCH" != "arm64" ]]; then + echo "Error: Invalid architecture specified. Use 'x86_64' or 'arm64'." + exit 1 +fi + +echo "Hello world $ARCH"; + +brew install llvm + +workspace="build_temp" + +mkdir "$workspace" -p +cp -a ../../third_party/whatlang-ffi/. $workspace/. +cp -a Native/whatlang-ffi/. $workspace/. + +ls -R . + +cd "$workspace" + +echo "Build for MacOS on $ARCH"; +cargo build --release + +ls -R + +cd ./target/release +otool -L libwhatlang.dylib +cp libwhatlang.dylib ../../../libwhatlang.$ARCH.dylib + +# Clean up +rm -rf "$workspace" +echo "Goodbye world"; diff --git a/src/LanguageIdentification.Whatlang.Native/Scripts/run-build.ps1 b/src/LanguageIdentification.Whatlang.Native/Scripts/run-build.ps1 new file mode 100644 index 00000000..e4552e99 --- /dev/null +++ b/src/LanguageIdentification.Whatlang.Native/Scripts/run-build.ps1 @@ -0,0 +1,28 @@ +# PowerShell equivalent script +# Ensure script stops if any command fails +$ErrorActionPreference = "Stop" + +Write-Output "Hello world" + +$workspace = "build_temp" + +# Copy directories +Copy-Item -Path "..\..\third_party\whatlang-ffi\." -Destination $workspace -Recurse -Force +Copy-Item -Path "Native\whatlang-ffi\." -Destination $workspace -Recurse -Force + +# List directory contents recursively +Get-ChildItem -Recurse -Path . + +Set-Location $workspace + +# Build for Windows +cargo build --release + +# List directory contents recursively +Get-ChildItem -Recurse -Path . + +# Display shared library dependencies +Copy-Item -Path ".\target\release\whatlang.dll" -Destination "..\whatlang.dll" + +# Clean up +Write-Output "Goodbye world" \ No newline at end of file diff --git a/src/LanguageIdentification.Whatlang/WhatlangDetector.cs b/src/LanguageIdentification.Whatlang/WhatlangDetector.cs index 00b9c8b0..ba044ea2 100644 --- a/src/LanguageIdentification.Whatlang/WhatlangDetector.cs +++ b/src/LanguageIdentification.Whatlang/WhatlangDetector.cs @@ -25,6 +25,9 @@ public static bool IsSupported() return RuntimeInformation.OSArchitecture switch { Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.Linux) => true, + Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.Windows) => true, + Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, + Architecture.Arm64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, _ => false, }; } From b7fe3c60ad0cadcbf4e6c574a5c02413f919449c Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Fri, 8 Nov 2024 10:05:06 +0400 Subject: [PATCH 22/34] Update github-ci.yml --- .github/workflows/github-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index b5a06b33..d51d8c17 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -24,9 +24,9 @@ jobs: - { title: CLD3, projectName: LanguageIdentification.CLD3.Native, os: macos-14, arch: arm64, script: ./Scripts/run-build.macos.sh, artifact: libcld3.arm64.dylib } - { title: Lingua, projectName: LanguageIdentification.Lingua.Native, os: windows-latest, arch: x86_64, script: ./Scripts/run-build.ps1, artifact: lingua.dll } - { title: Lingua, projectName: LanguageIdentification.Lingua.Native, os: macos-14, arch: arm64, script: ./Scripts/run-build.macos.sh, artifact: liblingua.arm64.dylib } - - { title: Whatlang, projectName: LanguageIdentification.Whatlang.Native, os: windows-latest, arch: x86_64, script: ./Scripts/run-build.ps1, artifact: lingua.dll } - - { title: Whatlang, projectName: LanguageIdentification.Whatlang.Native, os: macos-13, arch: x86_64, script: ./Scripts/run-build.macos.sh, artifact: liblingua.x86_64.dylib } - - { title: Whatlang, projectName: LanguageIdentification.Whatlang.Native, os: macos-14, arch: arm64, script: ./Scripts/run-build.macos.sh, artifact: liblingua.arm64.dylib } + - { title: Whatlang, projectName: LanguageIdentification.Whatlang.Native, os: windows-latest, arch: x86_64, script: ./Scripts/run-build.ps1, artifact: whatlang.dll } + - { title: Whatlang, projectName: LanguageIdentification.Whatlang.Native, os: macos-13, arch: x86_64, script: ./Scripts/run-build.macos.sh, artifact: libwhatlang.x86_64.dylib } + - { title: Whatlang, projectName: LanguageIdentification.Whatlang.Native, os: macos-14, arch: arm64, script: ./Scripts/run-build.macos.sh, artifact: libwhatlang.arm64.dylib } permissions: contents: read packages: write From fc6e296727cf528102ee5b930f7e5ab95ba0986d Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Fri, 8 Nov 2024 10:20:19 +0400 Subject: [PATCH 23/34] Whatlang: fixes for Windows --- .../Properties/launchSettings.json | 8 ++++++++ .../.gitignore | 1 + .../Internal/WhatlangDetectorWrapper.cs | 12 ++++++------ 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/LanguageIdentification.Whatlang.ConsoleTest/Properties/launchSettings.json b/src/LanguageIdentification.Whatlang.ConsoleTest/Properties/launchSettings.json index 65b8965a..515a8d0d 100644 --- a/src/LanguageIdentification.Whatlang.ConsoleTest/Properties/launchSettings.json +++ b/src/LanguageIdentification.Whatlang.ConsoleTest/Properties/launchSettings.json @@ -2,6 +2,14 @@ "profiles": { "Docker": { "commandName": "Docker" + }, + "WSL": { + "commandName": "WSL2", + "distributionName": "" + }, + "Project": { + "commandName": "Project", + "nativeDebugging": true } } } \ No newline at end of file diff --git a/src/LanguageIdentification.Whatlang.Native/.gitignore b/src/LanguageIdentification.Whatlang.Native/.gitignore index e04f0140..5cc57232 100644 --- a/src/LanguageIdentification.Whatlang.Native/.gitignore +++ b/src/LanguageIdentification.Whatlang.Native/.gitignore @@ -1,4 +1,5 @@ libwhatlang.so libwhatlang.arm64.dylib +libwhatlang.x86_64.dylib whatlang.dll build_temp/** diff --git a/src/LanguageIdentification.Whatlang/Internal/WhatlangDetectorWrapper.cs b/src/LanguageIdentification.Whatlang/Internal/WhatlangDetectorWrapper.cs index 0bce38e7..31cd2cff 100644 --- a/src/LanguageIdentification.Whatlang/Internal/WhatlangDetectorWrapper.cs +++ b/src/LanguageIdentification.Whatlang/Internal/WhatlangDetectorWrapper.cs @@ -10,13 +10,13 @@ internal static class WhatlangDetectorWrapper { [DllImport(WhatlangNativeLibrary.Name, EntryPoint = "whatlang_detect", CallingConvention = CallingConvention.Cdecl)] public static extern WhatlangStatus WhatlangDetect( - string text, + [MarshalAs(UnmanagedType.LPUTF8Str)] string text, out WhatlangPredictionResult result ); [DllImport(WhatlangNativeLibrary.Name, EntryPoint = "whatlang_detectn", CallingConvention = CallingConvention.Cdecl)] public static extern WhatlangStatus WhatlangDetectN( - string text, + [MarshalAs(UnmanagedType.LPUTF8Str)] string text, int len, out WhatlangPredictionResult info ); @@ -24,28 +24,28 @@ out WhatlangPredictionResult info [DllImport(WhatlangNativeLibrary.Name, EntryPoint = "whatlang_lang_code", CallingConvention = CallingConvention.Cdecl)] public static extern int WhatlangLangCode( WhatlangLanguage lang, - [MarshalAs(UnmanagedType.LPStr)] StringBuilder buffer, + [MarshalAs(UnmanagedType.LPUTF8Str)] StringBuilder buffer, UIntPtr bufferSize ); [DllImport(WhatlangNativeLibrary.Name, EntryPoint = "whatlang_lang_eng_name", CallingConvention = CallingConvention.Cdecl)] public static extern int WhatlangLangEngName( WhatlangLanguage lang, - [MarshalAs(UnmanagedType.LPStr)] StringBuilder buffer, + [MarshalAs(UnmanagedType.LPUTF8Str)] StringBuilder buffer, UIntPtr bufferSize ); [DllImport(WhatlangNativeLibrary.Name, EntryPoint = "whatlang_lang_name", CallingConvention = CallingConvention.Cdecl)] public static extern int WhatlangLangName( WhatlangLanguage lang, - [MarshalAs(UnmanagedType.LPStr)] StringBuilder buffer, + [MarshalAs(UnmanagedType.LPUTF8Str)] StringBuilder buffer, UIntPtr bufferSize ); [DllImport(WhatlangNativeLibrary.Name, EntryPoint = "whatlang_script_name", CallingConvention = CallingConvention.Cdecl)] public static extern int WhatlangScriptName( WhatlangScript script, - [MarshalAs(UnmanagedType.LPStr)] StringBuilder buffer, + [MarshalAs(UnmanagedType.LPUTF8Str)] StringBuilder buffer, UIntPtr bufferSize ); } From 00ea29753d25f67435f3515c96f7a1efef5351ac Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sat, 9 Nov 2024 18:18:38 +0400 Subject: [PATCH 24/34] Update run-build.sh --- .../Scripts/run-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LanguageIdentification.MediaPipe.Native/Scripts/run-build.sh b/src/LanguageIdentification.MediaPipe.Native/Scripts/run-build.sh index 3f0b79ec..8292ee32 100644 --- a/src/LanguageIdentification.MediaPipe.Native/Scripts/run-build.sh +++ b/src/LanguageIdentification.MediaPipe.Native/Scripts/run-build.sh @@ -22,7 +22,7 @@ bazel-6.1.1 build -c opt \ cd .. -find "$workspace/bazel-bin/mediapipe/tasks/c/text/language_detector" -name "liblanguage_detector.so" -exec cp {} liblanguage_detector.so \; +cp $workspace/bazel-bin/mediapipe/tasks/c/text/language_detector/liblanguage_detector.so liblanguage_detector.so rm -rf "$workspace" ldd liblanguage_detector.so From 830e3cdc90b0d7c0703f150f0f9e506a53d902d9 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sat, 9 Nov 2024 21:43:20 +0400 Subject: [PATCH 25/34] Update run-build.sh --- .../Scripts/run-build.sh | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/LanguageIdentification.MediaPipe.Native/Scripts/run-build.sh b/src/LanguageIdentification.MediaPipe.Native/Scripts/run-build.sh index 8292ee32..7168b0d3 100644 --- a/src/LanguageIdentification.MediaPipe.Native/Scripts/run-build.sh +++ b/src/LanguageIdentification.MediaPipe.Native/Scripts/run-build.sh @@ -18,11 +18,19 @@ bazel-6.1.1 build -c opt \ --define MEDIAPIPE_DISABLE_GPU=1 \ --define='absl=0' \ --sandbox_debug --verbose_failures \ - //mediapipe/tasks/c/text/language_detector:liblanguage_detector.so + --config darwin_arm64 \ + //mediapipe/tasks/c/text/language_detector:liblanguage_detector.dylib + +cp ./bazel-bin/mediapipe/tasks/c/text/language_detector/liblanguage_detector.dylib ../liblanguage_detector.dylib -cd .. +bazel-6.1.1 build -c opt \ + --linkopt -s --strip always \ + --define MEDIAPIPE_DISABLE_GPU=1 \ + --define='absl=0' \ + --sandbox_debug --verbose_failures \ + //mediapipe/tasks/c/text/language_detector:liblanguage_detector.so -cp $workspace/bazel-bin/mediapipe/tasks/c/text/language_detector/liblanguage_detector.so liblanguage_detector.so +cp ./bazel-bin/mediapipe/tasks/c/text/language_detector/liblanguage_detector.so ../liblanguage_detector.so rm -rf "$workspace" ldd liblanguage_detector.so From 8d8a242c45ad22bc389b1a141c4be64707137a95 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 10 Nov 2024 00:38:41 +0400 Subject: [PATCH 26/34] Update run-build.sh --- .../Scripts/run-build.sh | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/src/LanguageIdentification.MediaPipe.Native/Scripts/run-build.sh b/src/LanguageIdentification.MediaPipe.Native/Scripts/run-build.sh index 7168b0d3..513eae0e 100644 --- a/src/LanguageIdentification.MediaPipe.Native/Scripts/run-build.sh +++ b/src/LanguageIdentification.MediaPipe.Native/Scripts/run-build.sh @@ -13,16 +13,6 @@ cd "$workspace" zx ../Native/monkey-patch.mjs -bazel-6.1.1 build -c opt \ - --linkopt -s --strip always \ - --define MEDIAPIPE_DISABLE_GPU=1 \ - --define='absl=0' \ - --sandbox_debug --verbose_failures \ - --config darwin_arm64 \ - //mediapipe/tasks/c/text/language_detector:liblanguage_detector.dylib - -cp ./bazel-bin/mediapipe/tasks/c/text/language_detector/liblanguage_detector.dylib ../liblanguage_detector.dylib - bazel-6.1.1 build -c opt \ --linkopt -s --strip always \ --define MEDIAPIPE_DISABLE_GPU=1 \ @@ -31,6 +21,7 @@ bazel-6.1.1 build -c opt \ //mediapipe/tasks/c/text/language_detector:liblanguage_detector.so cp ./bazel-bin/mediapipe/tasks/c/text/language_detector/liblanguage_detector.so ../liblanguage_detector.so +cd .. rm -rf "$workspace" ldd liblanguage_detector.so From 1fb8b3bb1647087901df002ada48aca857dfdd08 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 10 Nov 2024 01:47:07 +0400 Subject: [PATCH 27/34] Update readme --- README.md | 6 ++++-- README_LINGUA.md | 4 ++-- README_WHATLANG.md | 4 ++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 73e52774..f4093383 100644 --- a/README.md +++ b/README.md @@ -94,14 +94,16 @@ functionality into their applications. | CLD2 | :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | | CLD3 | :white_check_mark: | :white_check_mark: | :construction: | :x: | | FastText | :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | -| Whatlang | :white_check_mark: | :construction: | :construction: | :x: | +| Whatlang | :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | | MediaPipe | :white_check_mark: | :construction: | :construction: | :x: | -| Lingua | :white_check_mark: | :construction: | :construction: | :x: | +| Lingua | :white_check_mark: | :white_check_mark: | :white_check_mark:\* | :x: | :white_check_mark: — Full support | :x: — No support | :construction: — Under research +\* **ARM** CPU only + ## TODO - [ ] Research support for other platforms (Windows, macOS). diff --git a/README_LINGUA.md b/README_LINGUA.md index eeef561a..64fc39bd 100644 --- a/README_LINGUA.md +++ b/README_LINGUA.md @@ -5,8 +5,8 @@ Welcome to **Panlingo.LanguageIdentification.Lingua**, a .NET wrapper for the Li ## Requirements - Runtime: **.NET >= 5.0** -- OS: **Linux** -- Arch: **AMD64** +- OS: **Linux (Ubuntu, Debian)**, **Windows 10+** or **Windows Server 2019+**, **macOS** +- Arch: **AMD64** (or **ARM** for macOS) ## Installation diff --git a/README_WHATLANG.md b/README_WHATLANG.md index f3fd5e62..4a07afd7 100644 --- a/README_WHATLANG.md +++ b/README_WHATLANG.md @@ -5,8 +5,8 @@ Welcome to **Panlingo.LanguageIdentification.Whatlang**, a .NET wrapper for the ## Requirements - Runtime: **.NET >= 5.0** -- OS: **Linux** -- Arch: **AMD64** +- OS: **Linux (Ubuntu, Debian)**, **Windows 10+** or **Windows Server 2019+**, **macOS** +- Arch: **AMD64** (or **ARM** for macOS) ## Installation From 21c8e3bb6b9474e1ba1e6ab5c44a8fbfd096bb91 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 10 Nov 2024 01:47:20 +0400 Subject: [PATCH 28/34] Bump version to 0.2.0 --- src/LanguageCode/LanguageCode.csproj | 2 +- .../LanguageIdentification.CLD2.Native.csproj | 2 +- .../LanguageIdentification.CLD2.csproj | 2 +- .../LanguageIdentification.CLD3.Native.csproj | 2 +- .../LanguageIdentification.CLD3.csproj | 2 +- .../LanguageIdentification.FastText.Native.csproj | 2 +- .../LanguageIdentification.FastText.csproj | 2 +- .../LanguageIdentification.Lingua.Native.csproj | 2 +- .../LanguageIdentification.Lingua.csproj | 8 ++++++-- .../LanguageIdentification.MediaPipe.Native.csproj | 2 +- .../LanguageIdentification.MediaPipe.csproj | 2 +- .../LanguageIdentification.Whatlang.Native.csproj | 2 +- .../LanguageIdentification.Whatlang.csproj | 9 +++++++-- 13 files changed, 24 insertions(+), 15 deletions(-) diff --git a/src/LanguageCode/LanguageCode.csproj b/src/LanguageCode/LanguageCode.csproj index 23da55e2..47aa7e70 100644 --- a/src/LanguageCode/LanguageCode.csproj +++ b/src/LanguageCode/LanguageCode.csproj @@ -2,7 +2,7 @@ netstandard2.1;net5.0;net6.0;net7.0;net8.0 - 0.1.2 + 0.2.0 Panlingo.LanguageCode Panlingo.LanguageCode Panlingo.LanguageCode diff --git a/src/LanguageIdentification.CLD2.Native/LanguageIdentification.CLD2.Native.csproj b/src/LanguageIdentification.CLD2.Native/LanguageIdentification.CLD2.Native.csproj index 8ecef5a0..50937410 100644 --- a/src/LanguageIdentification.CLD2.Native/LanguageIdentification.CLD2.Native.csproj +++ b/src/LanguageIdentification.CLD2.Native/LanguageIdentification.CLD2.Native.csproj @@ -2,7 +2,7 @@ netstandard2.1 - 0.1.1 + 0.2.0 Panlingo.LanguageIdentification.CLD2.Native Panlingo.LanguageIdentification.CLD2.Native Panlingo.LanguageIdentification.CLD2.Native diff --git a/src/LanguageIdentification.CLD2/LanguageIdentification.CLD2.csproj b/src/LanguageIdentification.CLD2/LanguageIdentification.CLD2.csproj index 8085caf9..505be4b2 100644 --- a/src/LanguageIdentification.CLD2/LanguageIdentification.CLD2.csproj +++ b/src/LanguageIdentification.CLD2/LanguageIdentification.CLD2.csproj @@ -2,7 +2,7 @@ net5.0;net6.0;net7.0;net8.0 - 0.1.1 + 0.2.0 Panlingo.LanguageIdentification.CLD2 Panlingo.LanguageIdentification.CLD2 Panlingo.LanguageIdentification.CLD2 diff --git a/src/LanguageIdentification.CLD3.Native/LanguageIdentification.CLD3.Native.csproj b/src/LanguageIdentification.CLD3.Native/LanguageIdentification.CLD3.Native.csproj index 39dd2d92..f06efc3a 100644 --- a/src/LanguageIdentification.CLD3.Native/LanguageIdentification.CLD3.Native.csproj +++ b/src/LanguageIdentification.CLD3.Native/LanguageIdentification.CLD3.Native.csproj @@ -2,7 +2,7 @@ netstandard2.1 - 0.1.1 + 0.2.0 Panlingo.LanguageIdentification.CLD3.Native Panlingo.LanguageIdentification.CLD3.Native Panlingo.LanguageIdentification.CLD3.Native diff --git a/src/LanguageIdentification.CLD3/LanguageIdentification.CLD3.csproj b/src/LanguageIdentification.CLD3/LanguageIdentification.CLD3.csproj index 7f22918e..9f891c4f 100644 --- a/src/LanguageIdentification.CLD3/LanguageIdentification.CLD3.csproj +++ b/src/LanguageIdentification.CLD3/LanguageIdentification.CLD3.csproj @@ -2,7 +2,7 @@ net5.0;net6.0;net7.0;net8.0 - 0.1.1 + 0.2.0 Panlingo.LanguageIdentification.CLD3 Panlingo.LanguageIdentification.CLD3 Panlingo.LanguageIdentification.CLD3 diff --git a/src/LanguageIdentification.FastText.Native/LanguageIdentification.FastText.Native.csproj b/src/LanguageIdentification.FastText.Native/LanguageIdentification.FastText.Native.csproj index 51d0e482..0666b064 100644 --- a/src/LanguageIdentification.FastText.Native/LanguageIdentification.FastText.Native.csproj +++ b/src/LanguageIdentification.FastText.Native/LanguageIdentification.FastText.Native.csproj @@ -2,7 +2,7 @@ netstandard2.1 - 0.1.1 + 0.2.0 Panlingo.LanguageIdentification.FastText.Native Panlingo.LanguageIdentification.FastText.Native Panlingo.LanguageIdentification.FastText.Native diff --git a/src/LanguageIdentification.FastText/LanguageIdentification.FastText.csproj b/src/LanguageIdentification.FastText/LanguageIdentification.FastText.csproj index 647b0ee2..1ecce103 100644 --- a/src/LanguageIdentification.FastText/LanguageIdentification.FastText.csproj +++ b/src/LanguageIdentification.FastText/LanguageIdentification.FastText.csproj @@ -2,7 +2,7 @@ net5.0;net6.0;net7.0;net8.0 - 0.1.1 + 0.2.0 Panlingo.LanguageIdentification.FastText Panlingo.LanguageIdentification.FastText Panlingo.LanguageIdentification.FastText diff --git a/src/LanguageIdentification.Lingua.Native/LanguageIdentification.Lingua.Native.csproj b/src/LanguageIdentification.Lingua.Native/LanguageIdentification.Lingua.Native.csproj index 1d6288ea..70c42be4 100644 --- a/src/LanguageIdentification.Lingua.Native/LanguageIdentification.Lingua.Native.csproj +++ b/src/LanguageIdentification.Lingua.Native/LanguageIdentification.Lingua.Native.csproj @@ -2,7 +2,7 @@ netstandard2.1 - 0.1.1 + 0.2.0 Panlingo.LanguageIdentification.Lingua.Native Panlingo.LanguageIdentification.Lingua.Native Panlingo.LanguageIdentification.Lingua.Native diff --git a/src/LanguageIdentification.Lingua/LanguageIdentification.Lingua.csproj b/src/LanguageIdentification.Lingua/LanguageIdentification.Lingua.csproj index ad5faaf0..f2ab19da 100644 --- a/src/LanguageIdentification.Lingua/LanguageIdentification.Lingua.csproj +++ b/src/LanguageIdentification.Lingua/LanguageIdentification.Lingua.csproj @@ -2,7 +2,7 @@ net5.0;net6.0;net7.0;net8.0 - 0.1.1 + 0.2.0 Panlingo.LanguageIdentification.Lingua Panlingo.LanguageIdentification.Lingua Panlingo.LanguageIdentification.Lingua @@ -15,7 +15,11 @@ nlp lid language-identification language-detection lingua README_LINGUA.md - - Initial release +0.2.0.0 +- Windows and MacOS support + +0.0.0.10: +- Initial release This is a .NET wrapper for the Lingua library, which is an efficient and easy-to-use language detection library. diff --git a/src/LanguageIdentification.MediaPipe.Native/LanguageIdentification.MediaPipe.Native.csproj b/src/LanguageIdentification.MediaPipe.Native/LanguageIdentification.MediaPipe.Native.csproj index 4877c4b8..bb15b061 100644 --- a/src/LanguageIdentification.MediaPipe.Native/LanguageIdentification.MediaPipe.Native.csproj +++ b/src/LanguageIdentification.MediaPipe.Native/LanguageIdentification.MediaPipe.Native.csproj @@ -2,7 +2,7 @@ netstandard2.1 - 0.1.1 + 0.2.0 Panlingo.LanguageIdentification.MediaPipe.Native Panlingo.LanguageIdentification.MediaPipe.Native Panlingo.LanguageIdentification.MediaPipe.Native diff --git a/src/LanguageIdentification.MediaPipe/LanguageIdentification.MediaPipe.csproj b/src/LanguageIdentification.MediaPipe/LanguageIdentification.MediaPipe.csproj index 15d3375f..3e641532 100644 --- a/src/LanguageIdentification.MediaPipe/LanguageIdentification.MediaPipe.csproj +++ b/src/LanguageIdentification.MediaPipe/LanguageIdentification.MediaPipe.csproj @@ -2,7 +2,7 @@ net5.0;net6.0;net7.0;net8.0 - 0.1.1 + 0.2.0 Panlingo.LanguageIdentification.MediaPipe Panlingo.LanguageIdentification.MediaPipe Panlingo.LanguageIdentification.MediaPipe diff --git a/src/LanguageIdentification.Whatlang.Native/LanguageIdentification.Whatlang.Native.csproj b/src/LanguageIdentification.Whatlang.Native/LanguageIdentification.Whatlang.Native.csproj index 35bcab00..f1ec6fd6 100644 --- a/src/LanguageIdentification.Whatlang.Native/LanguageIdentification.Whatlang.Native.csproj +++ b/src/LanguageIdentification.Whatlang.Native/LanguageIdentification.Whatlang.Native.csproj @@ -2,7 +2,7 @@ netstandard2.1 - 0.1.1 + 0.2.0 Panlingo.LanguageIdentification.Whatlang.Native Panlingo.LanguageIdentification.Whatlang.Native Panlingo.LanguageIdentification.Whatlang.Native diff --git a/src/LanguageIdentification.Whatlang/LanguageIdentification.Whatlang.csproj b/src/LanguageIdentification.Whatlang/LanguageIdentification.Whatlang.csproj index 9f8e66a0..5cadd6f5 100644 --- a/src/LanguageIdentification.Whatlang/LanguageIdentification.Whatlang.csproj +++ b/src/LanguageIdentification.Whatlang/LanguageIdentification.Whatlang.csproj @@ -2,7 +2,7 @@ net5.0;net6.0;net7.0;net8.0 - 0.1.1 + 0.2.0 Panlingo.LanguageIdentification.Whatlang Panlingo.LanguageIdentification.Whatlang Panlingo.LanguageIdentification.Whatlang @@ -15,7 +15,12 @@ nlp lid language-identification language-detection whatlang README_WHATLANG.md - - Initial release +0.2.0.0 +- Windows and MacOS support +- Armenian support + +0.0.0.10: +- Initial release This is a .NET wrapper for the Whatlang library, which is an efficient and easy-to-use language detection library. From e6976fd5fb5b7b73b9ca19315cd889e41150d02c Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 10 Nov 2024 01:55:21 +0400 Subject: [PATCH 29/34] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f4093383..1b76ca4e 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,7 @@ functionality into their applications. | CLD3 | :white_check_mark: | :white_check_mark: | :construction: | :x: | | FastText | :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | | Whatlang | :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | -| MediaPipe | :white_check_mark: | :construction: | :construction: | :x: | +| MediaPipe | :white_check_mark: | :x: | :x: | :x: | | Lingua | :white_check_mark: | :white_check_mark: | :white_check_mark:\* | :x: | :white_check_mark: — Full support | From e7f8d36babc5c952d7586eb2734fe4ba54e0cbb9 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 10 Nov 2024 14:56:10 +0400 Subject: [PATCH 30/34] CLD3: try to fix for MacOS --- src/LanguageIdentification.CLD3.Native/.gitignore | 3 ++- src/LanguageIdentification.CLD3/CLD3Detector.cs | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/LanguageIdentification.CLD3.Native/.gitignore b/src/LanguageIdentification.CLD3.Native/.gitignore index 2af070e4..a3fdabe0 100644 --- a/src/LanguageIdentification.CLD3.Native/.gitignore +++ b/src/LanguageIdentification.CLD3.Native/.gitignore @@ -1,4 +1,5 @@ libcld3.so libcld3.dll -libcld3.dylib +libcld3.arm64.dylib +libcld3.x86_64.dylib build_temp/** diff --git a/src/LanguageIdentification.CLD3/CLD3Detector.cs b/src/LanguageIdentification.CLD3/CLD3Detector.cs index 02aac06b..915a70e0 100644 --- a/src/LanguageIdentification.CLD3/CLD3Detector.cs +++ b/src/LanguageIdentification.CLD3/CLD3Detector.cs @@ -34,8 +34,8 @@ public static bool IsSupported() { Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.Linux) => true, Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.Windows) => true, - //Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, - //Architecture.Arm64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, + Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, + Architecture.Arm64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, _ => false, }; } From a1de37d1a1e859ba8c407ad571330e8ce305c662 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 10 Nov 2024 14:56:43 +0400 Subject: [PATCH 31/34] Update CLD3Detector.cs --- src/LanguageIdentification.CLD3/CLD3Detector.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/LanguageIdentification.CLD3/CLD3Detector.cs b/src/LanguageIdentification.CLD3/CLD3Detector.cs index 915a70e0..806c36a0 100644 --- a/src/LanguageIdentification.CLD3/CLD3Detector.cs +++ b/src/LanguageIdentification.CLD3/CLD3Detector.cs @@ -34,8 +34,8 @@ public static bool IsSupported() { Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.Linux) => true, Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.Windows) => true, - Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, - Architecture.Arm64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, + // Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, + // Architecture.Arm64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, _ => false, }; } From 798815609b38f28ebe691539c361b056dd86ee35 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 10 Nov 2024 15:22:18 +0400 Subject: [PATCH 32/34] Add some binaries to .gitignore --- src/LanguageIdentification.CLD2.Native/.gitignore | 2 ++ src/LanguageIdentification.Whatlang.Native/.gitignore | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/LanguageIdentification.CLD2.Native/.gitignore b/src/LanguageIdentification.CLD2.Native/.gitignore index 585d5a3f..6dca2351 100644 --- a/src/LanguageIdentification.CLD2.Native/.gitignore +++ b/src/LanguageIdentification.CLD2.Native/.gitignore @@ -1,3 +1,5 @@ libcld2.so libcld2.dll +libcld2.arm64.dylib +libcld2.x86_64.dylib build_temp/** diff --git a/src/LanguageIdentification.Whatlang.Native/.gitignore b/src/LanguageIdentification.Whatlang.Native/.gitignore index 5cc57232..1163e574 100644 --- a/src/LanguageIdentification.Whatlang.Native/.gitignore +++ b/src/LanguageIdentification.Whatlang.Native/.gitignore @@ -1,5 +1,5 @@ libwhatlang.so +whatlang.dll libwhatlang.arm64.dylib libwhatlang.x86_64.dylib -whatlang.dll build_temp/** From 19f0f95705e907133460b86c9b0052334a2de8eb Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 10 Nov 2024 15:24:52 +0400 Subject: [PATCH 33/34] Add .NET 9.0 support --- .github/workflows/github-ci.yml | 2 ++ src/LanguageCode.Benchmark/LanguageCode.Benchmark.csproj | 2 +- src/LanguageCode.Generator/LanguageCode.Generator.csproj | 2 +- src/LanguageCode.Tests/LanguageCode.Tests.csproj | 2 +- .../LanguageIdentification.CLD2.ConsoleTest.csproj | 2 +- .../LanguageIdentification.CLD2.csproj | 2 +- .../LanguageIdentification.CLD3.ConsoleTest.csproj | 2 +- .../LanguageIdentification.CLD3.csproj | 2 +- .../LanguageIdentification.FastText.ConsoleTest.csproj | 2 +- .../LanguageIdentification.FastText.csproj | 2 +- .../LanguageIdentification.Lingua.ConsoleTest.csproj | 2 +- .../LanguageIdentification.Lingua.csproj | 2 +- .../LanguageIdentification.MediaPipe.ConsoleTest.csproj | 2 +- .../LanguageIdentification.MediaPipe.csproj | 2 +- .../LanguageIdentification.Tests.csproj | 2 +- .../LanguageIdentification.Whatlang.ConsoleTest.csproj | 2 +- .../LanguageIdentification.Whatlang.csproj | 2 +- 17 files changed, 18 insertions(+), 16 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index d51d8c17..13f97c34 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -125,6 +125,7 @@ jobs: 6.0.x 7.0.x 8.0.x + 9.0.x source-url: https://nuget.pkg.github.com/${{github.repository_owner}}/index.json - name: 🚀 Pack ${{matrix.task.projectName}}.Native @@ -274,6 +275,7 @@ jobs: 6.0.x 7.0.x 8.0.x + 9.0.x source-url: https://nuget.pkg.github.com/${{github.repository_owner}}/index.json - name: 📂 Files diff --git a/src/LanguageCode.Benchmark/LanguageCode.Benchmark.csproj b/src/LanguageCode.Benchmark/LanguageCode.Benchmark.csproj index 601f81fc..bdee3122 100644 --- a/src/LanguageCode.Benchmark/LanguageCode.Benchmark.csproj +++ b/src/LanguageCode.Benchmark/LanguageCode.Benchmark.csproj @@ -2,7 +2,7 @@ Exe - net8.0 + net9.0 enable enable diff --git a/src/LanguageCode.Generator/LanguageCode.Generator.csproj b/src/LanguageCode.Generator/LanguageCode.Generator.csproj index d6f99602..106c099f 100644 --- a/src/LanguageCode.Generator/LanguageCode.Generator.csproj +++ b/src/LanguageCode.Generator/LanguageCode.Generator.csproj @@ -2,7 +2,7 @@ Exe - net8.0 + net9.0 enable enable Panlingo.LanguageCode.Generator diff --git a/src/LanguageCode.Tests/LanguageCode.Tests.csproj b/src/LanguageCode.Tests/LanguageCode.Tests.csproj index 47b741d4..e3614dc2 100644 --- a/src/LanguageCode.Tests/LanguageCode.Tests.csproj +++ b/src/LanguageCode.Tests/LanguageCode.Tests.csproj @@ -1,7 +1,7 @@  - net8.0 + net9.0 enable enable diff --git a/src/LanguageIdentification.CLD2.ConsoleTest/LanguageIdentification.CLD2.ConsoleTest.csproj b/src/LanguageIdentification.CLD2.ConsoleTest/LanguageIdentification.CLD2.ConsoleTest.csproj index 9898c2e4..0315656a 100644 --- a/src/LanguageIdentification.CLD2.ConsoleTest/LanguageIdentification.CLD2.ConsoleTest.csproj +++ b/src/LanguageIdentification.CLD2.ConsoleTest/LanguageIdentification.CLD2.ConsoleTest.csproj @@ -3,7 +3,7 @@ Panlingo.LanguageIdentification.CLD2.ConsoleTest Exe - net8.0 + net9.0 enable enable Linux diff --git a/src/LanguageIdentification.CLD2/LanguageIdentification.CLD2.csproj b/src/LanguageIdentification.CLD2/LanguageIdentification.CLD2.csproj index 505be4b2..8b019b1d 100644 --- a/src/LanguageIdentification.CLD2/LanguageIdentification.CLD2.csproj +++ b/src/LanguageIdentification.CLD2/LanguageIdentification.CLD2.csproj @@ -1,7 +1,7 @@  - net5.0;net6.0;net7.0;net8.0 + net5.0;net6.0;net7.0;net8.0;net9.0 0.2.0 Panlingo.LanguageIdentification.CLD2 Panlingo.LanguageIdentification.CLD2 diff --git a/src/LanguageIdentification.CLD3.ConsoleTest/LanguageIdentification.CLD3.ConsoleTest.csproj b/src/LanguageIdentification.CLD3.ConsoleTest/LanguageIdentification.CLD3.ConsoleTest.csproj index edea5066..edd40144 100644 --- a/src/LanguageIdentification.CLD3.ConsoleTest/LanguageIdentification.CLD3.ConsoleTest.csproj +++ b/src/LanguageIdentification.CLD3.ConsoleTest/LanguageIdentification.CLD3.ConsoleTest.csproj @@ -3,7 +3,7 @@ Panlingo.LanguageIdentification.CLD3.ConsoleTest Exe - net8.0 + net9.0 enable enable Linux diff --git a/src/LanguageIdentification.CLD3/LanguageIdentification.CLD3.csproj b/src/LanguageIdentification.CLD3/LanguageIdentification.CLD3.csproj index 9f891c4f..67684668 100644 --- a/src/LanguageIdentification.CLD3/LanguageIdentification.CLD3.csproj +++ b/src/LanguageIdentification.CLD3/LanguageIdentification.CLD3.csproj @@ -1,7 +1,7 @@  - net5.0;net6.0;net7.0;net8.0 + net5.0;net6.0;net7.0;net8.0;net9.0 0.2.0 Panlingo.LanguageIdentification.CLD3 Panlingo.LanguageIdentification.CLD3 diff --git a/src/LanguageIdentification.FastText.ConsoleTest/LanguageIdentification.FastText.ConsoleTest.csproj b/src/LanguageIdentification.FastText.ConsoleTest/LanguageIdentification.FastText.ConsoleTest.csproj index 166a99fb..339e8d51 100644 --- a/src/LanguageIdentification.FastText.ConsoleTest/LanguageIdentification.FastText.ConsoleTest.csproj +++ b/src/LanguageIdentification.FastText.ConsoleTest/LanguageIdentification.FastText.ConsoleTest.csproj @@ -3,7 +3,7 @@ Panlingo.LanguageIdentification.FastText.ConsoleTest Exe - net8.0 + net9.0 enable enable Linux diff --git a/src/LanguageIdentification.FastText/LanguageIdentification.FastText.csproj b/src/LanguageIdentification.FastText/LanguageIdentification.FastText.csproj index 1ecce103..03caf788 100644 --- a/src/LanguageIdentification.FastText/LanguageIdentification.FastText.csproj +++ b/src/LanguageIdentification.FastText/LanguageIdentification.FastText.csproj @@ -1,7 +1,7 @@  - net5.0;net6.0;net7.0;net8.0 + net5.0;net6.0;net7.0;net8.0;net9.0 0.2.0 Panlingo.LanguageIdentification.FastText Panlingo.LanguageIdentification.FastText diff --git a/src/LanguageIdentification.Lingua.ConsoleTest/LanguageIdentification.Lingua.ConsoleTest.csproj b/src/LanguageIdentification.Lingua.ConsoleTest/LanguageIdentification.Lingua.ConsoleTest.csproj index fa081388..f988f062 100644 --- a/src/LanguageIdentification.Lingua.ConsoleTest/LanguageIdentification.Lingua.ConsoleTest.csproj +++ b/src/LanguageIdentification.Lingua.ConsoleTest/LanguageIdentification.Lingua.ConsoleTest.csproj @@ -3,7 +3,7 @@ Panlingo.LanguageIdentification.Lingua.ConsoleTest Exe - net8.0 + net9.0 enable enable Linux diff --git a/src/LanguageIdentification.Lingua/LanguageIdentification.Lingua.csproj b/src/LanguageIdentification.Lingua/LanguageIdentification.Lingua.csproj index f2ab19da..2805fd2b 100644 --- a/src/LanguageIdentification.Lingua/LanguageIdentification.Lingua.csproj +++ b/src/LanguageIdentification.Lingua/LanguageIdentification.Lingua.csproj @@ -1,7 +1,7 @@  - net5.0;net6.0;net7.0;net8.0 + net5.0;net6.0;net7.0;net8.0;net9.0 0.2.0 Panlingo.LanguageIdentification.Lingua Panlingo.LanguageIdentification.Lingua diff --git a/src/LanguageIdentification.MediaPipe.ConsoleTest/LanguageIdentification.MediaPipe.ConsoleTest.csproj b/src/LanguageIdentification.MediaPipe.ConsoleTest/LanguageIdentification.MediaPipe.ConsoleTest.csproj index daa535e9..7937397a 100644 --- a/src/LanguageIdentification.MediaPipe.ConsoleTest/LanguageIdentification.MediaPipe.ConsoleTest.csproj +++ b/src/LanguageIdentification.MediaPipe.ConsoleTest/LanguageIdentification.MediaPipe.ConsoleTest.csproj @@ -3,7 +3,7 @@ Panlingo.LanguageIdentification.MediaPipe.ConsoleTest Exe - net8.0 + net9.0 enable enable Linux diff --git a/src/LanguageIdentification.MediaPipe/LanguageIdentification.MediaPipe.csproj b/src/LanguageIdentification.MediaPipe/LanguageIdentification.MediaPipe.csproj index 3e641532..dbb02aa1 100644 --- a/src/LanguageIdentification.MediaPipe/LanguageIdentification.MediaPipe.csproj +++ b/src/LanguageIdentification.MediaPipe/LanguageIdentification.MediaPipe.csproj @@ -1,7 +1,7 @@  - net5.0;net6.0;net7.0;net8.0 + net5.0;net6.0;net7.0;net8.0;net9.0 0.2.0 Panlingo.LanguageIdentification.MediaPipe Panlingo.LanguageIdentification.MediaPipe diff --git a/src/LanguageIdentification.Tests/LanguageIdentification.Tests.csproj b/src/LanguageIdentification.Tests/LanguageIdentification.Tests.csproj index 49e2490b..2e2b0712 100644 --- a/src/LanguageIdentification.Tests/LanguageIdentification.Tests.csproj +++ b/src/LanguageIdentification.Tests/LanguageIdentification.Tests.csproj @@ -1,7 +1,7 @@  - net8.0 + net9.0 enable enable diff --git a/src/LanguageIdentification.Whatlang.ConsoleTest/LanguageIdentification.Whatlang.ConsoleTest.csproj b/src/LanguageIdentification.Whatlang.ConsoleTest/LanguageIdentification.Whatlang.ConsoleTest.csproj index e6698621..879a1a35 100644 --- a/src/LanguageIdentification.Whatlang.ConsoleTest/LanguageIdentification.Whatlang.ConsoleTest.csproj +++ b/src/LanguageIdentification.Whatlang.ConsoleTest/LanguageIdentification.Whatlang.ConsoleTest.csproj @@ -3,7 +3,7 @@ Panlingo.LanguageIdentification.Whatlang.ConsoleTest Exe - net8.0 + net9.0 enable enable Linux diff --git a/src/LanguageIdentification.Whatlang/LanguageIdentification.Whatlang.csproj b/src/LanguageIdentification.Whatlang/LanguageIdentification.Whatlang.csproj index 5cadd6f5..09e05122 100644 --- a/src/LanguageIdentification.Whatlang/LanguageIdentification.Whatlang.csproj +++ b/src/LanguageIdentification.Whatlang/LanguageIdentification.Whatlang.csproj @@ -1,7 +1,7 @@  - net5.0;net6.0;net7.0;net8.0 + net5.0;net6.0;net7.0;net8.0;net9.0 0.2.0 Panlingo.LanguageIdentification.Whatlang Panlingo.LanguageIdentification.Whatlang From c94705d7eded38626fd34f7f88fb05839b8266f5 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 10 Nov 2024 15:48:45 +0400 Subject: [PATCH 34/34] net9.0 -> net8.0 --- src/LanguageCode.Benchmark/LanguageCode.Benchmark.csproj | 2 +- src/LanguageCode.Generator/LanguageCode.Generator.csproj | 2 +- src/LanguageCode.Tests/LanguageCode.Tests.csproj | 2 +- .../LanguageIdentification.CLD2.ConsoleTest.csproj | 2 +- .../LanguageIdentification.CLD3.ConsoleTest.csproj | 2 +- .../LanguageIdentification.FastText.ConsoleTest.csproj | 2 +- .../LanguageIdentification.Lingua.ConsoleTest.csproj | 2 +- .../LanguageIdentification.MediaPipe.ConsoleTest.csproj | 2 +- .../LanguageIdentification.Tests.csproj | 2 +- .../LanguageIdentification.Whatlang.ConsoleTest.csproj | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/LanguageCode.Benchmark/LanguageCode.Benchmark.csproj b/src/LanguageCode.Benchmark/LanguageCode.Benchmark.csproj index bdee3122..601f81fc 100644 --- a/src/LanguageCode.Benchmark/LanguageCode.Benchmark.csproj +++ b/src/LanguageCode.Benchmark/LanguageCode.Benchmark.csproj @@ -2,7 +2,7 @@ Exe - net9.0 + net8.0 enable enable diff --git a/src/LanguageCode.Generator/LanguageCode.Generator.csproj b/src/LanguageCode.Generator/LanguageCode.Generator.csproj index 106c099f..d6f99602 100644 --- a/src/LanguageCode.Generator/LanguageCode.Generator.csproj +++ b/src/LanguageCode.Generator/LanguageCode.Generator.csproj @@ -2,7 +2,7 @@ Exe - net9.0 + net8.0 enable enable Panlingo.LanguageCode.Generator diff --git a/src/LanguageCode.Tests/LanguageCode.Tests.csproj b/src/LanguageCode.Tests/LanguageCode.Tests.csproj index e3614dc2..47b741d4 100644 --- a/src/LanguageCode.Tests/LanguageCode.Tests.csproj +++ b/src/LanguageCode.Tests/LanguageCode.Tests.csproj @@ -1,7 +1,7 @@  - net9.0 + net8.0 enable enable diff --git a/src/LanguageIdentification.CLD2.ConsoleTest/LanguageIdentification.CLD2.ConsoleTest.csproj b/src/LanguageIdentification.CLD2.ConsoleTest/LanguageIdentification.CLD2.ConsoleTest.csproj index 0315656a..9898c2e4 100644 --- a/src/LanguageIdentification.CLD2.ConsoleTest/LanguageIdentification.CLD2.ConsoleTest.csproj +++ b/src/LanguageIdentification.CLD2.ConsoleTest/LanguageIdentification.CLD2.ConsoleTest.csproj @@ -3,7 +3,7 @@ Panlingo.LanguageIdentification.CLD2.ConsoleTest Exe - net9.0 + net8.0 enable enable Linux diff --git a/src/LanguageIdentification.CLD3.ConsoleTest/LanguageIdentification.CLD3.ConsoleTest.csproj b/src/LanguageIdentification.CLD3.ConsoleTest/LanguageIdentification.CLD3.ConsoleTest.csproj index edd40144..edea5066 100644 --- a/src/LanguageIdentification.CLD3.ConsoleTest/LanguageIdentification.CLD3.ConsoleTest.csproj +++ b/src/LanguageIdentification.CLD3.ConsoleTest/LanguageIdentification.CLD3.ConsoleTest.csproj @@ -3,7 +3,7 @@ Panlingo.LanguageIdentification.CLD3.ConsoleTest Exe - net9.0 + net8.0 enable enable Linux diff --git a/src/LanguageIdentification.FastText.ConsoleTest/LanguageIdentification.FastText.ConsoleTest.csproj b/src/LanguageIdentification.FastText.ConsoleTest/LanguageIdentification.FastText.ConsoleTest.csproj index 339e8d51..166a99fb 100644 --- a/src/LanguageIdentification.FastText.ConsoleTest/LanguageIdentification.FastText.ConsoleTest.csproj +++ b/src/LanguageIdentification.FastText.ConsoleTest/LanguageIdentification.FastText.ConsoleTest.csproj @@ -3,7 +3,7 @@ Panlingo.LanguageIdentification.FastText.ConsoleTest Exe - net9.0 + net8.0 enable enable Linux diff --git a/src/LanguageIdentification.Lingua.ConsoleTest/LanguageIdentification.Lingua.ConsoleTest.csproj b/src/LanguageIdentification.Lingua.ConsoleTest/LanguageIdentification.Lingua.ConsoleTest.csproj index f988f062..fa081388 100644 --- a/src/LanguageIdentification.Lingua.ConsoleTest/LanguageIdentification.Lingua.ConsoleTest.csproj +++ b/src/LanguageIdentification.Lingua.ConsoleTest/LanguageIdentification.Lingua.ConsoleTest.csproj @@ -3,7 +3,7 @@ Panlingo.LanguageIdentification.Lingua.ConsoleTest Exe - net9.0 + net8.0 enable enable Linux diff --git a/src/LanguageIdentification.MediaPipe.ConsoleTest/LanguageIdentification.MediaPipe.ConsoleTest.csproj b/src/LanguageIdentification.MediaPipe.ConsoleTest/LanguageIdentification.MediaPipe.ConsoleTest.csproj index 7937397a..daa535e9 100644 --- a/src/LanguageIdentification.MediaPipe.ConsoleTest/LanguageIdentification.MediaPipe.ConsoleTest.csproj +++ b/src/LanguageIdentification.MediaPipe.ConsoleTest/LanguageIdentification.MediaPipe.ConsoleTest.csproj @@ -3,7 +3,7 @@ Panlingo.LanguageIdentification.MediaPipe.ConsoleTest Exe - net9.0 + net8.0 enable enable Linux diff --git a/src/LanguageIdentification.Tests/LanguageIdentification.Tests.csproj b/src/LanguageIdentification.Tests/LanguageIdentification.Tests.csproj index 2e2b0712..49e2490b 100644 --- a/src/LanguageIdentification.Tests/LanguageIdentification.Tests.csproj +++ b/src/LanguageIdentification.Tests/LanguageIdentification.Tests.csproj @@ -1,7 +1,7 @@  - net9.0 + net8.0 enable enable diff --git a/src/LanguageIdentification.Whatlang.ConsoleTest/LanguageIdentification.Whatlang.ConsoleTest.csproj b/src/LanguageIdentification.Whatlang.ConsoleTest/LanguageIdentification.Whatlang.ConsoleTest.csproj index 879a1a35..e6698621 100644 --- a/src/LanguageIdentification.Whatlang.ConsoleTest/LanguageIdentification.Whatlang.ConsoleTest.csproj +++ b/src/LanguageIdentification.Whatlang.ConsoleTest/LanguageIdentification.Whatlang.ConsoleTest.csproj @@ -3,7 +3,7 @@ Panlingo.LanguageIdentification.Whatlang.ConsoleTest Exe - net9.0 + net8.0 enable enable Linux