Merge branch 'master' into migrate-to-we-cant-spell

.github/workflows/ci-cd.yml

@@ -28,9 +28,9 @@ jobs:
 
     # Install the .NET Core workload
     - name: Install .NET Core
-      uses: actions/setup-dotnet@v2
+      uses: actions/setup-dotnet@v4
       with:
-        dotnet-version: 6.0.x
+        dotnet-version: 8.0.x
 
     - name: Install mono-devel package
       run: sudo apt-get install mono-devel
@@ -67,11 +67,11 @@ jobs:
     - name: Test on Linux
       run: |
         . environ
-        dotnet test --no-restore --no-build -p:ParallelizeAssembly=false --configuration Release --logger:"trx" --results-directory ./test-results
+        dotnet test --no-restore --no-build -p:ParallelizeAssembly=false --configuration Release --logger:"trx;LogFilePrefix=results" --results-directory ./test-results
       if: matrix.os == 'ubuntu-latest'
 
     - name: Test on Windows
-      run: dotnet test --no-restore --no-build -p:ParallelizeAssembly=false --configuration Release --logger:"trx" --results-directory ./test-results
+      run: dotnet test --no-restore --no-build -p:ParallelizeAssembly=false --configuration Release --logger:"trx;LogFilePrefix=results" --results-directory ./test-results
       if: matrix.os != 'ubuntu-latest'
     - name: Upload test results
       if: always()
@@ -94,7 +94,7 @@ jobs:
     - name: Publish Artifacts
       uses: actions/upload-artifact@v4
       with:
-        name: NugetPackages
+        name: NugetPackages (${{ matrix.os }})
         path: artifacts/*.nupkg
       if: github.event_name == 'pull_request'
   publish-test-results:

.gitignore

@@ -12,6 +12,7 @@ artifacts/
 lib/downloads/
 .idea/
 .vs/
+*.bak
 
 src/SIL.LCModel.Core/KernelInterfaces/Kernel.cs
 src/SIL.LCModel/Infrastructure/Impl/Generated*.cs

Directory.Build.props

@@ -11,7 +11,8 @@
     <PackageProjectUrl>https://github.com/sillsdev/liblcm</PackageProjectUrl>
     <PackageRequireLicenseAcceptance>false</PackageRequireLicenseAcceptance>
     <Platform>Any CPU</Platform>
-    <OutputPath>$(MSBuildThisFileDirectory)/artifacts/$(Configuration)/$(TargetFramework)</OutputPath>
+	<Configuration Condition="'$(Configuration)' == ''">Debug</Configuration>
+    <OutputPath>$(MSBuildThisFileDirectory)artifacts/$(Configuration)/$(TargetFramework)</OutputPath>
     <AppendTargetFrameworkToOutputPath>false</AppendTargetFrameworkToOutputPath>
     <PackageOutputPath>$(MSBuildThisFileDirectory)/artifacts</PackageOutputPath>
     <SignAssembly>true</SignAssembly>

src/SIL.LCModel.Core/SIL.LCModel.Core.csproj

@@ -1,24 +1,24 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFrameworks>netstandard2.0;net461</TargetFrameworks>
+    <TargetFrameworks>netstandard2.0;net461;net8.0</TargetFrameworks>
     <RootNamespace>SIL.LCModel.Core</RootNamespace>
     <Description>The liblcm library is the core FieldWorks model for linguistic analyses of languages. Tools in this library provide the ability to store and interact with language and culture data, including anthropological, text corpus, and linguistics data.
 SIL.LCModel.Core provides a base library with core functionality.</Description>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="GitVersion.MsBuild" Version="5.6.10" PrivateAssets="All" />
-    <PackageReference Include="icu.net" Version="2.*-*" />
+    <PackageReference Include="icu.net" Version="3.0.0-*" />
     <PackageReference Include="Icu4c.Win.Fw.Bin" Version="70.1.152" IncludeAssets="build" />
     <PackageReference Include="Icu4c.Win.Fw.Lib" Version="70.1.152" IncludeAssets="build" PrivateAssets="runtime" />
     <PackageReference Include="Microsoft.DotNet.PlatformAbstractions" Version="2.0.4" />
     <PackageReference Include="Microsoft.Extensions.DependencyModel" Version="2.0.4" />
     <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.0.0" PrivateAssets="All" />
     <PackageReference Include="NHunspell" Version="1.2.5554.16953" GeneratePropertyPath="true" />
-    <PackageReference Include="SIL.Lexicon" Version="12.0.0-*" />
+    <PackageReference Include="SIL.Lexicon" Version="14.2.0-*" />
     <PackageReference Include="SIL.ReleaseTasks" Version="2.5.0" PrivateAssets="All" />
-    <PackageReference Include="SIL.WritingSystems" Version="12.0.0-*" />
+    <PackageReference Include="SIL.WritingSystems" Version="14.2.0-*" />
     <PackageReference Include="System.Drawing.Common" Version="6.0.0" />
     <PackageReference Include="System.Runtime.InteropServices.RuntimeInformation" Version="4.3.0" />
     <PackageReference Include="vswhere" Version="2.8.4" PrivateAssets="all" />
@@ -164,12 +164,12 @@ SIL.LCModel.Core provides a base library with core functionality.</Description>
   <Target Name="BeforeCoreClean" BeforeTargets="CoreClean">
     <!-- Prevent the deletion of IDLImporter.xml. This seems like a bug in msbuild - it shouldn't
     delete files it didn't copy, at least not in the middle of a rebuild -->
-    <Copy SourceFiles="$(OutputPath)/IDLImporter.xml" DestinationFiles="$(OutputPath)/../IDLImporter.xml" Condition="Exists('$(OutputPath)/IDLImporter.xml')" />
+    <Copy SourceFiles="$(OutputPath)IDLImporter.xml" DestinationFiles="$(OutputPath)../IDLImporter.xml" Condition="Exists('$(OutputPath)IDLImporter.xml')" />
   </Target>
 
   <Target Name="AfterCoreClean" AfterTargets="CoreClean">
     <!-- Restore IDLImporter.xml. -->
-    <Copy SourceFiles="$(OutputPath)/../IDLImporter.xml" DestinationFiles="$(OutputPath)/IDLImporter.xml" />
+    <Copy SourceFiles="$(OutputPath)../IDLImporter.xml" DestinationFiles="$(OutputPath)IDLImporter.xml" Condition="Exists('$(OutputPath)../IDLImporter.xml')" />
   </Target>
 
   <Target Name="CollectRuntimeOutputs" BeforeTargets="_GetPackageFiles">

src/SIL.LCModel.Utils/FileUtils.cs

@@ -319,7 +319,7 @@ public static bool AreFilesIdentical(string file1, string file2)
 			byte[] fileHash2;
 
 			// compute the hashes for comparison
-			using(var hash = HashAlgorithm.Create())
+			using(var hash = HashAlgorithm.Create("SHA1"))
 			using(Stream fileStream1 = OpenStreamForRead(file1),
 							 fileStream2 = OpenStreamForRead(file2))
 			{
@@ -473,16 +473,8 @@ public static string ActualFilePath(string sPathname)
 		/// ------------------------------------------------------------------------------------
 		public static void AssertValidFilePath(string filename)
 		{
-			try
-			{
-				new FileInfo(filename);
-			}
-			catch (SecurityException)
-			{
-			}
-			catch (UnauthorizedAccessException)
-			{
-			}
+			if (!IsFilePathValid(filename))
+				throw new ArgumentException("Illegal characters in path.");
 		}
 
 		/// ------------------------------------------------------------------------------------
@@ -492,11 +484,34 @@ public static void AssertValidFilePath(string filename)
 		/// ------------------------------------------------------------------------------------
 		public static bool IsFilePathValid(string filename)
 		{
+			if (filename == null)
+				return false;
 			try
 			{
-				AssertValidFilePath(filename);
+				// will throw in .net framework, but not newer versions
+				_ = Path.GetFullPath(filename);
+				var name = Path.GetFileName(filename);
+				var invalidFileNameChars = Path.GetInvalidFileNameChars();
+				if (name.IndexOfAny(invalidFileNameChars) >= 0)
+					return false;
+				if (name == filename) return true;
+				var directoryPath = filename.Substring(0, filename.Length - name.Length);
+				if (directoryPath.IndexOfAny(Path.GetInvalidPathChars()) >= 0)
+					return false;
+				if (Platform.IsWindows)
+				{
+					// some paths like "C:\bla" are valid, but not C\:bla, we want to catch those by excluding the drive letter and checking for invalid file name chars in each directory
+
+					//trim off the drive letter if it exists
+					directoryPath = directoryPath.Substring(Path.GetPathRoot(directoryPath).Length);
+					//each directory must be a valid file name. Using both / and \ because it could be a mixed path which usually works fine
+					if (directoryPath.Split('\\', '/').Any(dir => dir.IndexOfAny(invalidFileNameChars) >= 0))
+					{
+						return false;
+					}
+				}
 			}
-			catch (Exception)
+			catch
 			{
 				return false;
 			}
@@ -936,7 +951,7 @@ public static string ChangeWindowsPathIfLinux(string windowsPath)
 		public static string ChangeWindowsPathIfLinuxPreservingPrefix(string windowsPath,
 			string prefix)
 		{
-			if (windowsPath == null || prefix == null || !windowsPath.StartsWith(prefix))
+			if (windowsPath == null || prefix == null || !windowsPath.StartsWith(prefix, StringComparison.Ordinal))
 				return ChangeWindowsPathIfLinux(windowsPath);
 			// Preserve prefix
 			windowsPath = windowsPath.Substring(prefix.Length);
@@ -975,7 +990,7 @@ public static string ChangeLinuxPathIfWindows(string linuxPath)
 		public static string ChangeLinuxPathIfWindowsPreservingPrefix(string linuxPath,
 			string prefix)
 		{
-			if (linuxPath == null || prefix == null || !linuxPath.StartsWith(prefix))
+			if (linuxPath == null || prefix == null || !linuxPath.StartsWith(prefix, StringComparison.Ordinal))
 				return ChangeLinuxPathIfWindows(linuxPath);
 			// Preserve prefix
 			linuxPath = linuxPath.Substring(prefix.Length);

src/SIL.LCModel.Utils/SIL.LCModel.Utils.csproj

@@ -10,11 +10,11 @@ SIL.LCModel.Utils provides utility classes.</Description>
   <ItemGroup>
     <PackageReference Include="GitVersion.MsBuild" Version="5.6.10" PrivateAssets="All" />
     <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.0.0" PrivateAssets="All" />
-    <PackageReference Include="SIL.Core" Version="12.0.0-*" />
+    <PackageReference Include="SIL.Core" Version="14.2.0-*" />
     <PackageReference Include="SIL.ReleaseTasks" Version="2.5.0" PrivateAssets="All" />
     <PackageReference Include="Mono.Unix" Version="7.1.0-final.1.21458.1" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.2" />
-    <PackageReference Include="System.Configuration.ConfigurationManager" Version="5.0.0" />
+    <PackageReference Include="System.Configuration.ConfigurationManager" Version="6.0.0" />
     <PackageReference Include="System.Diagnostics.PerformanceCounter" Version="6.0.0" />
     <PackageReference Include="System.Management" Version="6.0.0" />
     <Reference Include="System.Management" />

src/SIL.LCModel/Application/ApplicationServices/XmlImportData.cs

@@ -995,6 +995,18 @@ private void ReadXmlObject(XmlReader xrdr, FieldInfo fi, ICmObject objToUse)
 					cmo = m_cache.LangProject.LexDbOA;
 					Debug.Assert(cmo != null);
 					break;
+				case "PhBdryMarker":
+					IPhBdryMarkerRepository repository = m_cache.ServiceLocator.GetInstance<IPhBdryMarkerRepository>();
+					IPhBdryMarkerFactory factory = m_cache.ServiceLocator.GetInstance<IPhBdryMarkerFactory>();
+					Guid guid = Guid.Parse(xrdr.GetAttribute("Guid"));
+					IPhBdryMarker marker;
+					if (repository.TryGetObject(guid, out marker))
+						cmo = marker;
+					else
+						cmo = factory.Create(guid, (IPhPhonemeSet)fi.Owner);
+					// Remove the default code added by PhTerminalUnit.SetDefaultValuesAfterInit in OverridesLing_Lex.
+					(cmo as PhBdryMarker).CodesOS.Clear();
+					break;
 				case "PhPhonData":
 					cmo = m_cache.LangProject.PhonologicalDataOA;
 					Debug.Assert(cmo != null);
@@ -1003,6 +1015,10 @@ private void ReadXmlObject(XmlReader xrdr, FieldInfo fi, ICmObject objToUse)
 					cmo = m_cache.LangProject.PhFeatureSystemOA;
 					Debug.Assert(cmo != null);
 					break;
+				case "PhPhonemeSet":
+					cmo = m_cache.LangProject.PhonologicalDataOA.GetPhonemeSet();
+					Debug.Assert(cmo != null);
+					break;
 				default:
 					int clid = m_mdc.GetClassId(sClass);
 					if (fi != null)
@@ -1048,7 +1064,6 @@ private void ReadXmlObject(XmlReader xrdr, FieldInfo fi, ICmObject objToUse)
 						cmo = m_repoCmObject.GetObject(hvo);
 						// Remove the default code added by PhTerminalUnit.SetDefaultValuesAfterInit in OverridesLing_Lex.
 						(cmo as PhPhoneme)?.CodesOS.Clear();
-						(cmo as PhBdryMarker)?.CodesOS.Clear();
 						// Remove default values.
 						(cmo as PhRegularRule)?.RightHandSidesOS.Clear();
 					}

src/SIL.LCModel/DomainImpl/OverridesLing_Lex.cs

@@ -6858,6 +6858,21 @@ public List<string> AllPhonemes()
 			return phReps;
 		}
 
+		/// <summary>
+		/// Get the PhonemeSet.
+		/// Hides the fact that multiple PhonemeSets used to be allowed.
+		/// </summary>
+		/// <returns></returns>
+		/// <exception cref="Exception"></exception>
+		public IPhPhonemeSet GetPhonemeSet()
+		{
+			if (this.PhonemeSetsOS.Count == 0)
+				throw new Exception("Missing PhonemeSet.");
+			if (this.PhonemeSetsOS.Count > 1)
+				throw new Exception("Too many PhonemeSets.");
+			return this.PhonemeSetsOS[0];
+		}
+
 		/// <summary>
 		/// Return the list of abbreviations for all the natural classes defined, each in the
 		/// default analysis writing system.

src/SIL.LCModel/DomainServices/DomainObjectServices.cs

@@ -2298,9 +2298,7 @@ public static ICmFile FindOrCreateFile(ICmFolder folder, string srcFile)
 			if (String.IsNullOrEmpty(srcFile))
 				throw new ArgumentException("File path not specified.", "srcFile");
 
-			char[] bad = Path.GetInvalidPathChars();
-			int idx = srcFile.IndexOfAny(bad);
-			if (idx >= 0)
+			if (!FileUtils.IsFilePathValid(srcFile))
 				throw new ArgumentException("File path (" + srcFile + ") contains at least one invalid character.", "srcFile");
 
 			foreach (ICmFile file in folder.FilesOC)

src/SIL.LCModel/DomainServices/M3ModelExportServices.cs

@@ -591,12 +591,16 @@ select ExportItemAsReference(constraint, constraints.IndexOf(constraint), "Featu
 							new XAttribute("Id", rhs.Hvo),
 							new XElement("StrucChange", from structChange in rhs.StrucChangeOS
 														select ExportContext(structChange)),
-							new XElement("InputPOSes", from pos in rhs.InputPOSesRC
-														select ExportItemAsReference(pos, "RequiredPOS")),
-							new XElement("ReqRuleFeats", from rrf in rhs.ReqRuleFeatsRC
-														select ExportItemAsReference(rrf, "RuleFeat")),
-							new XElement("ExclRuleFeats", from erf in rhs.ExclRuleFeatsRC
-															select ExportItemAsReference(erf, "RuleFeat")),
+							// RuleFeats and POS are not part of the phonology.
+							phonology ? null
+								: new XElement("InputPOSes", from pos in rhs.InputPOSesRC
+															select ExportItemAsReference(pos, "RequiredPOS")),
+							phonology ? null
+								: new XElement("ReqRuleFeats", from rrf in rhs.ReqRuleFeatsRC
+															select ExportItemAsReference(rrf, "RuleFeat")),
+							phonology ? null
+								: new XElement("ExclRuleFeats", from erf in rhs.ExclRuleFeatsRC
+																select ExportItemAsReference(erf, "RuleFeat")),
 							new XElement("LeftContext", ExportContext(rhs.LeftContextOA)),
 							new XElement("RightContext", ExportContext(rhs.RightContextOA)))));
 					break;

src/SIL.LCModel/DomainServices/PhonologyServices.cs

@@ -11,6 +11,7 @@
 using SIL.LCModel.Core.Text;
 using SIL.LCModel.Infrastructure.Impl;
 using static Icu.Normalization.Normalizer2;
+using SIL.LCModel.Core.KernelInterfaces;
 
 namespace SIL.LCModel.DomainServices
 {
@@ -74,42 +75,27 @@ public void ImportPhonologyFromXml(TextReader rdr)
 				// () => AssignVernacularWritingSystemToDefaultPhPhonemes(Cache));
 		}
 
-		private void AssignVernacularWritingSystemToDefaultPhPhonemes(LcmCache cache)
+		/// <summary>
+		/// Clear PhonologicalData and Phonological Features.
+		/// Don't clear boundary markers.
+		/// </summary>
+		public void DeletePhonology()
 		{
-			// For all PhCodes in the default phoneme set, change the writing system from "en" to icuLocale
-			if (cache.LanguageProject.PhonologicalDataOA.PhonemeSetsOS.Count == 0)
-				return;
-			var phSet = cache.LanguageProject.PhonologicalDataOA.PhonemeSetsOS[0];
-			int wsVern = m_wsVernId == null
-				? cache.DefaultVernWs
-				: cache.ServiceLocator.WritingSystemManager.Get(m_wsVernId).Handle;
-			foreach (var phone in phSet.PhonemesOC)
-			{
-				foreach (var code in phone.CodesOS)
-				{
-
-					if (code.Representation.VernacularDefaultWritingSystem.Length == 0)
-						code.Representation.VernacularDefaultWritingSystem =
-							TsStringUtils.MakeString(code.Representation.UserDefaultWritingSystem.Text, wsVern);
-				}
-				if (phone.Name.VernacularDefaultWritingSystem.Length == 0)
-					phone.Name.VernacularDefaultWritingSystem =
-						TsStringUtils.MakeString(phone.Name.UserDefaultWritingSystem.Text, wsVern);
-			}
-			foreach (var mrkr in phSet.BoundaryMarkersOC)
+			NonUndoableUnitOfWorkHelper.Do(Cache.ServiceLocator.GetInstance<IActionHandler>(), () =>
 			{
-				foreach (var code in mrkr.CodesOS)
-				{
-					if (code.Representation.VernacularDefaultWritingSystem.Length == 0)
-						code.Representation.VernacularDefaultWritingSystem =
-							TsStringUtils.MakeString(code.Representation.UserDefaultWritingSystem.Text, wsVern);
-				}
-				if (mrkr.Name.VernacularDefaultWritingSystem.Length == 0)
-					mrkr.Name.VernacularDefaultWritingSystem =
-						TsStringUtils.MakeString(mrkr.Name.UserDefaultWritingSystem.Text, wsVern);
-			}
+				IPhPhonData phonData = Cache.LangProject.PhonologicalDataOA;
+				// Delete what is covered by ImportPhonology.
+				phonData.ContextsOS.Clear();
+				phonData.EnvironmentsOS.Clear();
+				phonData.FeatConstraintsOS.Clear();
+				phonData.NaturalClassesOS.Clear();
+				phonData.GetPhonemeSet().PhonemesOC.Clear();
+				// Don't clear phonData.GetPhonemeSet().BoundaryMarkersOC!
+				// They have GUIDs known to the code.
+				phonData.PhonRulesOS.Clear();
+				Cache.LanguageProject.PhFeatureSystemOA.TypesOC.Clear();
+				Cache.LanguageProject.PhFeatureSystemOA.FeaturesOC.Clear();
+			});
 		}
-
-
 	}
 }

src/SIL.LCModel/InterfaceAdditions.cs

@@ -5626,6 +5626,12 @@ public partial interface IPhPhonData
 		/// </summary>
 		List<string> AllNaturalClassAbbrs();
 
+		/// <summary>
+		/// Get the PhonemeSet.
+		/// Hides the fact that the data structure used to allow more than one.
+		/// </summary>
+		IPhPhonemeSet GetPhonemeSet();
+
 		/// <summary>
 		/// Rebuild the list of PhonRuleFeats
 		/// </summary>

src/SIL.LCModel/LinkedFilesRelativePathHelper.cs

@@ -274,6 +274,8 @@ private static string GetPathWithLowercaseRoot(string path)
 			try
 			{
 				var rootOfPath = Path.GetPathRoot(path);
+				// dotnet 8 on linux, when path contains invalid characters rootOfPath will be null
+				if (rootOfPath is null) return path.ToLowerInvariant();
 				return rootOfPath.ToLowerInvariant()
 					+ path.Substring(rootOfPath.Length, path.Length - rootOfPath.Length);
 			}

src/SIL.LCModel/SIL.LCModel.csproj

@@ -22,9 +22,9 @@
     <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.0.0" PrivateAssets="All" />
     <PackageReference Include="protobuf-net" Version="2.4.6" />
     <PackageReference Include="SharpZipLib" Version="1.4.0" />
-    <PackageReference Include="SIL.Lexicon" Version="12.0.0-*" />
+    <PackageReference Include="SIL.Lexicon" Version="14.2.0-*" />
     <PackageReference Include="SIL.ReleaseTasks" Version="2.5.0" PrivateAssets="All" />
-    <PackageReference Include="SIL.WritingSystems" Version="12.0.0-*" />
+    <PackageReference Include="SIL.WritingSystems" Version="14.2.0-*" />
     <PackageReference Include="structuremap.patched" Version="4.7.3" />
     <PackageReference Include="System.ServiceProcess.ServiceController" Version="6.0.0" />
   </ItemGroup>