Merge pull request #22 from gluschenko/cross-platform-part2

Cross-platfrom #2
gluschenko · Nov 10, 2024 · 3eea82e · 3eea82e
2 parents 28270ef + c94705d
commit 3eea82e
Show file tree

Hide file tree

Showing 43 changed files with 372 additions and 90 deletions.
diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml
@@ -22,6 +22,11 @@ jobs:
           - { title: CLD2, projectName: LanguageIdentification.CLD2.Native, os: macos-14, arch: arm64, script: ./Scripts/run-build.macos.sh, artifact: libcld2.arm64.dylib }
           - { title: CLD3, projectName: LanguageIdentification.CLD3.Native, os: macos-13, arch: x86_64, script: ./Scripts/run-build.macos.sh, artifact: libcld3.x86_64.dylib }
           - { title: CLD3, projectName: LanguageIdentification.CLD3.Native, os: macos-14, arch: arm64, script: ./Scripts/run-build.macos.sh, artifact: libcld3.arm64.dylib }
+          - { title: Lingua, projectName: LanguageIdentification.Lingua.Native, os: windows-latest, arch: x86_64, script: ./Scripts/run-build.ps1, artifact: lingua.dll }
+          - { title: Lingua, projectName: LanguageIdentification.Lingua.Native, os: macos-14, arch: arm64, script: ./Scripts/run-build.macos.sh, artifact: liblingua.arm64.dylib }
+          - { title: Whatlang, projectName: LanguageIdentification.Whatlang.Native, os: windows-latest, arch: x86_64, script: ./Scripts/run-build.ps1, artifact: whatlang.dll }
+          - { title: Whatlang, projectName: LanguageIdentification.Whatlang.Native, os: macos-13, arch: x86_64, script: ./Scripts/run-build.macos.sh, artifact: libwhatlang.x86_64.dylib }
+          - { title: Whatlang, projectName: LanguageIdentification.Whatlang.Native, os: macos-14, arch: arm64, script: ./Scripts/run-build.macos.sh, artifact: libwhatlang.arm64.dylib }
     permissions:
       contents: read
       packages: write
@@ -120,6 +125,7 @@ jobs:
           6.0.x
           7.0.x
           8.0.x
+          9.0.x
         source-url: https://nuget.pkg.github.com/${{github.repository_owner}}/index.json
 
     - name: 🚀 Pack ${{matrix.task.projectName}}.Native
@@ -269,6 +275,7 @@ jobs:
           6.0.x
           7.0.x
           8.0.x
+          9.0.x
         source-url: https://nuget.pkg.github.com/${{github.repository_owner}}/index.json
 
     - name: 📂 Files

diff --git a/README.md b/README.md
@@ -94,14 +94,16 @@ functionality into their applications.
 | CLD2      | :white_check_mark: | :white_check_mark: | :white_check_mark: | :x:            |
 | CLD3      | :white_check_mark: | :white_check_mark: | :construction:     | :x:            |
 | FastText  | :white_check_mark: | :white_check_mark: | :white_check_mark: | :x:            |
-| Whatlang  | :white_check_mark: | :construction:     | :construction:     | :x:            |
-| MediaPipe | :white_check_mark: | :construction:     | :construction:     | :x:            |
-| Lingua    | :white_check_mark: | :construction:     | :construction:     | :x:            |
+| Whatlang  | :white_check_mark: | :white_check_mark: | :white_check_mark: | :x:            |
+| MediaPipe | :white_check_mark: | :x:                | :x:                | :x:            |
+| Lingua    | :white_check_mark: | :white_check_mark: | :white_check_mark:\* | :x:            |
 
 :white_check_mark: — Full support |
 :x: — No support |
 :construction: — Under research
 
+\* **ARM** CPU only
+
 ## TODO
 
 - [ ] Research support for other platforms (Windows, macOS).

diff --git a/README_LINGUA.md b/README_LINGUA.md
@@ -5,8 +5,8 @@ Welcome to **Panlingo.LanguageIdentification.Lingua**, a .NET wrapper for the Li
 ## Requirements
 
 - Runtime: **.NET >= 5.0**
-- OS: **Linux**
-- Arch: **AMD64**
+- OS: **Linux (Ubuntu, Debian)**, **Windows 10+** or **Windows Server 2019+**, **macOS**
+- Arch: **AMD64** (or **ARM** for macOS)
 
 ## Installation
 

diff --git a/README_WHATLANG.md b/README_WHATLANG.md
@@ -5,8 +5,8 @@ Welcome to **Panlingo.LanguageIdentification.Whatlang**, a .NET wrapper for the
 ## Requirements
 
 - Runtime: **.NET >= 5.0**
-- OS: **Linux**
-- Arch: **AMD64**
+- OS: **Linux (Ubuntu, Debian)**, **Windows 10+** or **Windows Server 2019+**, **macOS**
+- Arch: **AMD64** (or **ARM** for macOS)
 
 ## Installation
 

diff --git a/src/LanguageCode/LanguageCode.csproj b/src/LanguageCode/LanguageCode.csproj
@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <TargetFrameworks>netstandard2.1;net5.0;net6.0;net7.0;net8.0</TargetFrameworks>
-    <Version>0.1.2</Version>
+    <Version>0.2.0</Version>
     <PackageId>Panlingo.LanguageCode</PackageId>
     <AssemblyName>Panlingo.LanguageCode</AssemblyName>
     <RootNamespace>Panlingo.LanguageCode</RootNamespace>

diff --git a/src/LanguageIdentification.CLD2.Native/.gitignore b/src/LanguageIdentification.CLD2.Native/.gitignore
@@ -1,3 +1,5 @@
 libcld2.so
 libcld2.dll
+libcld2.arm64.dylib
+libcld2.x86_64.dylib
 build_temp/**
diff --git a/src/LanguageIdentification.CLD2.Native/LanguageIdentification.CLD2.Native.csproj b/src/LanguageIdentification.CLD2.Native/LanguageIdentification.CLD2.Native.csproj
@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <TargetFramework>netstandard2.1</TargetFramework>
-    <Version>0.1.1</Version>
+    <Version>0.2.0</Version>
     <PackageId>Panlingo.LanguageIdentification.CLD2.Native</PackageId>
     <AssemblyName>Panlingo.LanguageIdentification.CLD2.Native</AssemblyName>
     <RootNamespace>Panlingo.LanguageIdentification.CLD2.Native</RootNamespace>

diff --git a/src/LanguageIdentification.CLD2/LanguageIdentification.CLD2.csproj b/src/LanguageIdentification.CLD2/LanguageIdentification.CLD2.csproj
@@ -1,8 +1,8 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFrameworks>net5.0;net6.0;net7.0;net8.0</TargetFrameworks>
-    <Version>0.1.1</Version>
+    <TargetFrameworks>net5.0;net6.0;net7.0;net8.0;net9.0</TargetFrameworks>
+    <Version>0.2.0</Version>
     <PackageId>Panlingo.LanguageIdentification.CLD2</PackageId>
     <AssemblyName>Panlingo.LanguageIdentification.CLD2</AssemblyName>
     <RootNamespace>Panlingo.LanguageIdentification.CLD2</RootNamespace>

diff --git a/src/LanguageIdentification.CLD3.Native/.gitignore b/src/LanguageIdentification.CLD3.Native/.gitignore
@@ -1,4 +1,5 @@
 libcld3.so
 libcld3.dll
-libcld3.dylib
+libcld3.arm64.dylib
+libcld3.x86_64.dylib
 build_temp/**
diff --git a/src/LanguageIdentification.CLD3.Native/LanguageIdentification.CLD3.Native.csproj b/src/LanguageIdentification.CLD3.Native/LanguageIdentification.CLD3.Native.csproj
@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <TargetFramework>netstandard2.1</TargetFramework>
-    <Version>0.1.1</Version>
+    <Version>0.2.0</Version>
     <PackageId>Panlingo.LanguageIdentification.CLD3.Native</PackageId>
     <AssemblyName>Panlingo.LanguageIdentification.CLD3.Native</AssemblyName>
     <RootNamespace>Panlingo.LanguageIdentification.CLD3.Native</RootNamespace>

diff --git a/src/LanguageIdentification.CLD3/CLD3Detector.cs b/src/LanguageIdentification.CLD3/CLD3Detector.cs
@@ -34,8 +34,8 @@ public static bool IsSupported()
             {
                 Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.Linux) => true,
                 Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.Windows) => true,
-                //Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true,
-                //Architecture.Arm64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true,
+                // Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true,
+                // Architecture.Arm64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true,
                 _ => false,
             };
         }

diff --git a/src/LanguageIdentification.CLD3/LanguageIdentification.CLD3.csproj b/src/LanguageIdentification.CLD3/LanguageIdentification.CLD3.csproj
@@ -1,8 +1,8 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFrameworks>net5.0;net6.0;net7.0;net8.0</TargetFrameworks>
-    <Version>0.1.1</Version>
+    <TargetFrameworks>net5.0;net6.0;net7.0;net8.0;net9.0</TargetFrameworks>
+    <Version>0.2.0</Version>
     <PackageId>Panlingo.LanguageIdentification.CLD3</PackageId>
     <AssemblyName>Panlingo.LanguageIdentification.CLD3</AssemblyName>
     <RootNamespace>Panlingo.LanguageIdentification.CLD3</RootNamespace>

diff --git a/src/LanguageIdentification.FastText.Native/LanguageIdentification.FastText.Native.csproj b/src/LanguageIdentification.FastText.Native/LanguageIdentification.FastText.Native.csproj
@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <TargetFramework>netstandard2.1</TargetFramework>
-    <Version>0.1.1</Version>
+    <Version>0.2.0</Version>
     <PackageId>Panlingo.LanguageIdentification.FastText.Native</PackageId>
     <AssemblyName>Panlingo.LanguageIdentification.FastText.Native</AssemblyName>
     <RootNamespace>Panlingo.LanguageIdentification.FastText.Native</RootNamespace>

diff --git a/src/LanguageIdentification.FastText.Native/Scripts/run-build.ps1 b/src/LanguageIdentification.FastText.Native/Scripts/run-build.ps1
@@ -35,7 +35,7 @@ cmake --build .
 Get-ChildItem -Recurse -Path .
 
 # Display shared library dependencies
-Copy-Item -Path ".\Debug\fasttext.dll" -Destination "../../fasttext.dll"
+Copy-Item -Path ".\Debug\fasttext.dll" -Destination "..\..\fasttext.dll"
 
 # List directory contents recursively
 Get-ChildItem -Recurse -Path .

diff --git a/src/LanguageIdentification.FastText/LanguageIdentification.FastText.csproj b/src/LanguageIdentification.FastText/LanguageIdentification.FastText.csproj
@@ -1,8 +1,8 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFrameworks>net5.0;net6.0;net7.0;net8.0</TargetFrameworks>
-    <Version>0.1.1</Version>
+    <TargetFrameworks>net5.0;net6.0;net7.0;net8.0;net9.0</TargetFrameworks>
+    <Version>0.2.0</Version>
     <PackageId>Panlingo.LanguageIdentification.FastText</PackageId>
     <AssemblyName>Panlingo.LanguageIdentification.FastText</AssemblyName>
     <RootNamespace>Panlingo.LanguageIdentification.FastText</RootNamespace>

diff --git a/src/LanguageIdentification.Lingua.ConsoleTest/Program.cs b/src/LanguageIdentification.Lingua.ConsoleTest/Program.cs
@@ -16,7 +16,6 @@ static void Main(string[] args)
                 "Привіт",
                 "Привет",
                 "Hello",
-                "Hello, how are you? Привіт, як справи? Hello, how are you? Привет, как дела?",
                 "Hello, how are you?",
                 "Привіт, як справи?",
                 "Привет, как дела?",
@@ -45,8 +44,8 @@ static void Main(string[] args)
 
             var code1 = lingua.GetLanguageCode(LinguaLanguage.Ukrainian, LinguaLanguageCode.Alpha2);
             var code2 = lingua.GetLanguageCode(LinguaLanguage.Ukrainian, LinguaLanguageCode.Alpha3);
-            var code3 = lingua.GetLanguageCode(LinguaLanguage.Ukrainian, (LinguaLanguageCode)99);
-            var code4 = lingua.GetLanguageCode((LinguaLanguage)255, LinguaLanguageCode.Alpha3);
+            // var code3 = lingua.GetLanguageCode(LinguaLanguage.Ukrainian, (LinguaLanguageCode)99);
+            // var code4 = lingua.GetLanguageCode((LinguaLanguage)255, LinguaLanguageCode.Alpha3);
 
 
             ;

diff --git a/src/LanguageIdentification.Lingua.ConsoleTest/Properties/launchSettings.json b/src/LanguageIdentification.Lingua.ConsoleTest/Properties/launchSettings.json
@@ -2,6 +2,14 @@
   "profiles": {
     "Docker": {
       "commandName": "Docker"
+    },
+    "WSL": {
+      "commandName": "WSL2",
+      "distributionName": ""
+    },
+    "Project": {
+      "commandName": "Project",
+      "nativeDebugging": true
     }
   }
 }
diff --git a/src/LanguageIdentification.Lingua.Native/.gitignore b/src/LanguageIdentification.Lingua.Native/.gitignore
@@ -1,2 +1,4 @@
 liblingua.so
+liblingua.arm64.dylib
+lingua.dll
 build_temp/**
diff --git a/src/LanguageIdentification.Lingua.Native/LanguageIdentification.Lingua.Native.csproj b/src/LanguageIdentification.Lingua.Native/LanguageIdentification.Lingua.Native.csproj
@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <TargetFramework>netstandard2.1</TargetFramework>
-    <Version>0.1.1</Version>
+    <Version>0.2.0</Version>
     <PackageId>Panlingo.LanguageIdentification.Lingua.Native</PackageId>
     <AssemblyName>Panlingo.LanguageIdentification.Lingua.Native</AssemblyName>
     <RootNamespace>Panlingo.LanguageIdentification.Lingua.Native</RootNamespace>
@@ -61,6 +61,26 @@
     </Content>
   </ItemGroup>
 
+  <ItemGroup>
+    <Content Include="lingua.dll">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+      <PackageCopyToOutput>true</PackageCopyToOutput>
+      <PackagePath>runtimes/win-x64/native</PackagePath>
+      <pack>true</pack>
+      <Visible>false</Visible>
+    </Content>
+  </ItemGroup>
+
+  <ItemGroup>
+    <Content Include="liblingua.arm64.dylib">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+      <PackageCopyToOutput>true</PackageCopyToOutput>
+      <PackagePath>runtimes/osx-arm64/native/liblingua.dylib</PackagePath>
+      <pack>true</pack>
+      <Visible>false</Visible>
+    </Content>
+  </ItemGroup>
+
   <ItemGroup>
     <None Include="..\..\README_LINGUA.md" Pack="true" PackagePath="\" />
   </ItemGroup>

diff --git a/src/LanguageIdentification.Lingua.Native/Native/lingua-ffi/Cargo.toml b/src/LanguageIdentification.Lingua.Native/Native/lingua-ffi/Cargo.toml
@@ -9,4 +9,13 @@ libc = "0.2.157"
 
 [lib]
 name = "lingua"
+crate-type = ["cdylib"]
+
+[target.'cfg(target_os = "linux")'.lib]
+crate-type = ["cdylib"]
+
+[target.'cfg(target_os = "windows")'.lib]
+crate-type = ["staticlib"]
+
+[target.'cfg(target_os = "macos")'.lib]
 crate-type = ["staticlib", "cdylib"]
diff --git a/src/LanguageIdentification.Lingua.Native/Scripts/run-build.macos.sh b/src/LanguageIdentification.Lingua.Native/Scripts/run-build.macos.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+set -e
+
+if [ -z "$1" ]; then
+    echo "Error: No architecture specified."
+    echo "Usage: $0 <arch>"
+    exit 1
+fi
+
+ARCH=$1
+
+if [[ "$ARCH" != "x86_64" && "$ARCH" != "arm64" ]]; then
+    echo "Error: Invalid architecture specified. Use 'x86_64' or 'arm64'."
+    exit 1
+fi
+
+echo "Hello world $ARCH";
+
+brew install llvm
+
+workspace="build_temp"
+
+mkdir "$workspace" -p
+cp -a Native/lingua-ffi/. $workspace/.
+
+ls -R .
+
+cd "$workspace"
+
+echo "Build for MacOS on $ARCH";
+cargo build --release
+
+ls -R
+
+cd ./target/release
+otool -L liblingua.dylib
+cp liblingua.dylib ../../../liblingua.$ARCH.dylib
+
+# Clean up
+rm -rf "$workspace"
+echo "Goodbye world";
diff --git a/src/LanguageIdentification.Lingua.Native/Scripts/run-build.ps1 b/src/LanguageIdentification.Lingua.Native/Scripts/run-build.ps1
@@ -0,0 +1,27 @@
+# PowerShell equivalent script
+# Ensure script stops if any command fails
+$ErrorActionPreference = "Stop"
+
+Write-Output "Hello world"
+
+$workspace = "build_temp"
+
+# Copy directories
+Copy-Item -Path "Native\lingua-ffi\." -Destination $workspace -Recurse -Force
+
+# List directory contents recursively
+Get-ChildItem -Recurse -Path .
+
+Set-Location $workspace
+
+# Build for Windows
+cargo build --release
+
+# List directory contents recursively
+Get-ChildItem -Recurse -Path .
+
+# Display shared library dependencies
+Copy-Item -Path ".\target\release\lingua.dll" -Destination "..\lingua.dll"
+
+# Clean up
+Write-Output "Goodbye world"
diff --git a/src/LanguageIdentification.Lingua.Native/Scripts/run-build.sh b/src/LanguageIdentification.Lingua.Native/Scripts/run-build.sh
@@ -16,11 +16,11 @@ cargo build --release
 
 ls -R .
 
-cd ..
-
-find "$workspace/target/release" -name "liblingua.so" -exec cp {} liblingua.so \;
-rm -rf "$workspace"
+cd ./target/release
 ldd liblingua.so
+cp liblingua.so ../../../liblingua.so
 
+# Clean up
+rm -rf "$workspace"
 echo "Goodbye world";
 
diff --git a/src/LanguageIdentification.Lingua/Internal/LinguaDetectorWrapper.cs b/src/LanguageIdentification.Lingua/Internal/LinguaDetectorWrapper.cs
@@ -9,7 +9,10 @@ namespace Panlingo.LanguageIdentification.Lingua.Internal
     internal static class LinguaDetectorWrapper
     {
         [DllImport(LinguaNativeLibrary.Name, EntryPoint = "lingua_language_detector_builder_create", CallingConvention = CallingConvention.Cdecl)]
-        public static extern IntPtr LinguaLanguageDetectorBuilderCreate(LinguaLanguage[] languages, UIntPtr languageCount);
+        public static extern IntPtr LinguaLanguageDetectorBuilderCreate(
+            LinguaLanguage[] languages, 
+            UIntPtr languageCount
+        );
 
         [DllImport(LinguaNativeLibrary.Name, EntryPoint = "lingua_language_detector_builder_with_low_accuracy_mode", CallingConvention = CallingConvention.Cdecl)]
         public static extern IntPtr LinguaLanguageDetectorBuilderWithLowAccuracyMode(IntPtr builder);
@@ -38,22 +41,22 @@ internal static class LinguaDetectorWrapper
         [DllImport(LinguaNativeLibrary.Name, EntryPoint = "lingua_detect_single", CallingConvention = CallingConvention.Cdecl)]
         public static extern LinguaStatus LinguaDetectSingle(
             IntPtr detector,
-            string text,
+            [MarshalAs(UnmanagedType.LPUTF8Str)] string text,
             out LinguaPredictionListResult result
         );
 
         [DllImport(LinguaNativeLibrary.Name, EntryPoint = "lingua_detect_mixed", CallingConvention = CallingConvention.Cdecl)]
         public static extern LinguaStatus LinguaDetectMixed(
             IntPtr detector,
-            string text,
+            [MarshalAs(UnmanagedType.LPUTF8Str)] string text,
             out LinguaPredictionRangeListResult result
         );
 
         [DllImport(LinguaNativeLibrary.Name, EntryPoint = "lingua_language_code", CallingConvention = CallingConvention.Cdecl)]
         public static extern int LinguaLangCode(
             LinguaLanguage lang,
             LinguaLanguageCode code,
-            [MarshalAs(UnmanagedType.LPStr)] StringBuilder buffer,
+            [MarshalAs(UnmanagedType.LPUTF8Str)] StringBuilder buffer,
             UIntPtr bufferSize
         );
     }