feat: refactor file path handling across SFTP, S3, and FTP providers;…

… remove redundant StringEx classes
paillave · Jan 3, 2025 · 2d92767 · 2d92767
1 parent 29d899d
commit 2d92767
Show file tree

Hide file tree

Showing 14 changed files with 40 additions and 66 deletions.
diff --git a/src/Paillave.Etl.Dropbox/String.ex.cs b/src/Paillave.Etl.Dropbox/String.ex.cs
diff --git a/src/Paillave.Etl.Ftp/FtpFileValue.cs b/src/Paillave.Etl.Ftp/FtpFileValue.cs
@@ -24,7 +24,7 @@ public FtpFileValue(IFtpConnectionInfo connectionInfo, string folder, string fil
         protected override void DeleteFile() => ActionRunner.TryExecute(_connectionInfo.MaxAttempts, DeleteFileSingleTime);
         protected void DeleteFileSingleTime()
         {
-            var pathToDelete = Path.Combine(_folder, this.Name).Replace('\\', '/');
+            var pathToDelete = StringEx.ConcatenatePath(_folder, this.Name).Replace('\\', '/');
             using (FtpClient client = _connectionInfo.CreateFtpClient())
                 client.DeleteFile(pathToDelete);
         }
@@ -34,7 +34,7 @@ private Stream GetContentSingleTime()
             using (FtpClient client = _connectionInfo.CreateFtpClient())
             {
                 MemoryStream ms = new MemoryStream();
-                var pathToDownload = Path.Combine(_folder, this.Name).Replace('\\', '/');
+                var pathToDownload = StringEx.ConcatenatePath(_folder, this.Name).Replace('\\', '/');
                 if (!client.DownloadStream(ms, pathToDownload))
                     throw new System.Exception($"File {pathToDownload} failed to be downloaded");
                 ms.Seek(0, SeekOrigin.Begin);
@@ -44,7 +44,7 @@ private Stream GetContentSingleTime()
         public override StreamWithResource OpenContent()
         {
             FtpClient client = _connectionInfo.CreateFtpClient();
-            var pathToDownload = Path.Combine(_folder, this.Name).Replace('\\', '/');
+            var pathToDownload = StringEx.ConcatenatePath(_folder, this.Name).Replace('\\', '/');
             var targetPath = Path.GetTempFileName();
             if (client.DownloadFile(targetPath, pathToDownload) != FtpStatus.Success)
                 throw new System.Exception($"File {pathToDownload} failed to be downloaded");

diff --git a/src/Paillave.Etl.Ftp/FtpFileValueProcessor.cs b/src/Paillave.Etl.Ftp/FtpFileValueProcessor.cs
@@ -14,10 +14,10 @@ public FtpFileValueProcessor(string code, string name, string connectionName, Ft
         public override ProcessImpact MemoryFootPrint => ProcessImpact.Average;
         protected override void Process(IFileValue fileValue, FtpAdapterConnectionParameters connectionParameters, FtpAdapterProcessorParameters processorParameters, Action<IFileValue> push, CancellationToken cancellationToken, IExecutionContext context)
         {
-            var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (processorParameters.SubFolder ?? "") : Path.Combine(connectionParameters.RootFolder, processorParameters.SubFolder ?? "");
+            var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (processorParameters.SubFolder ?? "") : StringEx.ConcatenatePath(connectionParameters.RootFolder, processorParameters.SubFolder ?? "");
 
-            // var folder = Path.Combine(connectionParameters.RootFolder ?? "", processorParameters.SubFolder ?? "");
-            var filePath = Path.Combine(folder, fileValue.Name);
+            // var folder = StringEx.ConcatenatePath(connectionParameters.RootFolder ?? "", processorParameters.SubFolder ?? "");
+            var filePath = StringEx.ConcatenatePath(folder, fileValue.Name);
             filePath = SmartFormat.Smart.Format(filePath.Replace(@"\",@"\\"), fileValue.Metadata);
             using var stream = fileValue.Get(processorParameters.UseStreamCopy);
             byte[] fileContents;
@@ -46,8 +46,8 @@ private void UploadSingleTime(FtpAdapterConnectionParameters connectionParameter
 
         protected override void Test(FtpAdapterConnectionParameters connectionParameters, FtpAdapterProcessorParameters processorParameters)
         {
-            var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (processorParameters.SubFolder ?? "") : Path.Combine(connectionParameters.RootFolder, processorParameters.SubFolder ?? "");
-            var testFilePath = Path.Combine(folder, Guid.NewGuid().ToString());
+            var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (processorParameters.SubFolder ?? "") : StringEx.ConcatenatePath(connectionParameters.RootFolder, processorParameters.SubFolder ?? "");
+            var testFilePath = StringEx.ConcatenatePath(folder, Guid.NewGuid().ToString());
             using (FtpClient client = connectionParameters.CreateFtpClient())
             {
                 var stream = new MemoryStream();

diff --git a/src/Paillave.Etl.Ftp/FtpFileValueProvider.cs b/src/Paillave.Etl.Ftp/FtpFileValueProvider.cs
@@ -31,15 +31,15 @@ protected override void Provide(Action<IFileValue> pushFileValue, FtpAdapterConn
         }
         private FtpListItem[] GetFileList(FtpAdapterConnectionParameters connectionParameters, FtpAdapterProviderParameters providerParameters)
         {
-            var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (providerParameters.SubFolder ?? "") : Path.Combine(connectionParameters.RootFolder, providerParameters.SubFolder ?? "");
+            var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (providerParameters.SubFolder ?? "") : StringEx.ConcatenatePath(connectionParameters.RootFolder, providerParameters.SubFolder ?? "");
             using (FtpClient client = connectionParameters.CreateFtpClient())
             {
                 return (providerParameters.Recursive ? client.GetListing(folder, FtpListOption.Recursive) : client.GetListing(folder)).Where(i => i.Type == FtpObjectType.File).ToArray();
             }
         }
         protected override void Test(FtpAdapterConnectionParameters connectionParameters, FtpAdapterProviderParameters providerParameters)
         {
-            var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (providerParameters.SubFolder ?? "") : Path.Combine(connectionParameters.RootFolder, providerParameters.SubFolder ?? "");
+            var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (providerParameters.SubFolder ?? "") : StringEx.ConcatenatePath(connectionParameters.RootFolder, providerParameters.SubFolder ?? "");
             using (FtpClient client = connectionParameters.CreateFtpClient())
             {
                 if (providerParameters.Recursive)

diff --git a/src/Paillave.Etl.S3/S3Bucket.cs b/src/Paillave.Etl.S3/S3Bucket.cs
@@ -6,6 +6,7 @@
 using Amazon.Runtime;
 using Amazon.S3;
 using Amazon.S3.Model;
+using Paillave.Etl.Core;
 
 namespace Paillave.Etl.S3;
 public class S3Bucket : IDisposable
@@ -24,7 +25,7 @@ public async Task UploadAsync(string objectName, Stream stream, string? folder =
         var response = await _client.PutObjectAsync(new PutObjectRequest
         {
             BucketName = _bucketName,
-            Key = string.IsNullOrWhiteSpace(folder) ? objectName : Path.Combine(folder, objectName),
+            Key = string.IsNullOrWhiteSpace(folder) ? objectName : StringEx.ConcatenatePath(folder, objectName),
             InputStream = stream
             // FilePath = filePath,
         });
@@ -35,7 +36,7 @@ public async Task UploadAsync(string objectName, Stream stream, string? folder =
     }
 
     public Task<Stream> DownloadAsync(string objectName, string? folder = null)
-        => DownloadFromKeyAsync(string.IsNullOrWhiteSpace(folder) ? objectName : Path.Combine(folder, objectName));
+        => DownloadFromKeyAsync(string.IsNullOrWhiteSpace(folder) ? objectName : StringEx.ConcatenatePath(folder, objectName));
     public async Task<Stream> DownloadFromKeyAsync(string objectKey)
     {
         GetObjectResponse response = await _client.GetObjectAsync(new GetObjectRequest

diff --git a/src/Paillave.Etl.S3/S3FileValue.cs b/src/Paillave.Etl.S3/S3FileValue.cs
@@ -26,7 +26,7 @@ protected void DeleteFileSingleTime()
     {
         using (var client = _connectionInfo.CreateBucketConnection())
         {
-            client.DeleteAsync(Path.Combine(_folder, Name)).Wait();
+            client.DeleteAsync(StringEx.ConcatenatePath(_folder, Name)).Wait();
         }
     }
     public override Stream GetContent() => ActionRunner.TryExecute(_connectionInfo.MaxAttempts, GetContentSingleTime);
@@ -44,7 +44,7 @@ private Stream GetContentSingleTime()
     private StreamWithResource OpenContentSingleTime()
     {
         var client = _connectionInfo.CreateBucketConnection();
-        return new StreamWithResource(client.DownloadFromKeyAsync(Path.Combine(_folder, Name)).Result, client);
+        return new StreamWithResource(client.DownloadFromKeyAsync(StringEx.ConcatenatePath(_folder, Name)).Result, client);
     }
 
     public override StreamWithResource OpenContent() => ActionRunner.TryExecute(_connectionInfo.MaxAttempts, OpenContentSingleTime);

diff --git a/src/Paillave.Etl.S3/S3FileValueProcessor.cs b/src/Paillave.Etl.S3/S3FileValueProcessor.cs
@@ -22,9 +22,9 @@ public S3FileValueProcessor(string code, string name, string serviceUrl, string
     public override ProcessImpact MemoryFootPrint => ProcessImpact.Average;
     protected override void Process(IFileValue fileValue, S3AdapterConnectionParameters connectionParameters, S3AdapterProcessorParameters processorParameters, Action<IFileValue> push, CancellationToken cancellationToken, IExecutionContext context)
     {
-        var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (processorParameters.SubFolder ?? "") : Path.Combine(connectionParameters.RootFolder, processorParameters.SubFolder ?? "");
+        var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (processorParameters.SubFolder ?? "") : StringEx.ConcatenatePath(connectionParameters.RootFolder, processorParameters.SubFolder ?? "");
         using var stream = fileValue.Get(processorParameters.UseStreamCopy);
-        ActionRunner.TryExecute(connectionParameters.MaxAttempts, () => UploadSingleTime(connectionParameters, stream, Path.Combine(folder, fileValue.Name)));
+        ActionRunner.TryExecute(connectionParameters.MaxAttempts, () => UploadSingleTime(connectionParameters, stream, StringEx.ConcatenatePath(folder, fileValue.Name)));
         push(fileValue);
     }
     private void UploadSingleTime(S3AdapterConnectionParameters connectionParameters, Stream stream, string filePath)
@@ -37,14 +37,14 @@ private void UploadSingleTime(S3AdapterConnectionParameters connectionParameters
     protected override void Test(S3AdapterConnectionParameters connectionParameters, S3AdapterProcessorParameters processorParameters)
     {
         var fileName = Guid.NewGuid().ToString();
-        var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (processorParameters.SubFolder ?? "") : Path.Combine(connectionParameters.RootFolder, processorParameters.SubFolder ?? "");
+        var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (processorParameters.SubFolder ?? "") : StringEx.ConcatenatePath(connectionParameters.RootFolder, processorParameters.SubFolder ?? "");
         using (var client = connectionParameters.CreateBucketConnection())
         {
-            client.UploadAsync(Path.Combine(folder, fileName), new MemoryStream()).Wait();
+            client.UploadAsync(StringEx.ConcatenatePath(folder, fileName), new MemoryStream()).Wait();
         }
         using (var client = connectionParameters.CreateBucketConnection())
         {
-            client.DeleteAsync(Path.Combine(folder, fileName)).Wait();
+            client.DeleteAsync(StringEx.ConcatenatePath(folder, fileName)).Wait();
         }
     }
 }
diff --git a/src/Paillave.Etl.S3/S3FileValueProvider.cs b/src/Paillave.Etl.S3/S3FileValueProvider.cs
@@ -28,7 +28,7 @@ protected override void Provide(Action<IFileValue> pushFileValue, S3AdapterConne
     {
         var searchPattern = string.IsNullOrEmpty(providerParameters.FileNamePattern) ? "*" : providerParameters.FileNamePattern;
         var matcher = new Matcher().AddInclude(searchPattern);
-        var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (providerParameters.SubFolder ?? "") : Path.Combine(connectionParameters.RootFolder, providerParameters.SubFolder ?? "");
+        var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (providerParameters.SubFolder ?? "") : StringEx.ConcatenatePath(connectionParameters.RootFolder, providerParameters.SubFolder ?? "");
 
         var files = ActionRunner.TryExecute(connectionParameters.MaxAttempts, () => GetFileList(connectionParameters, providerParameters));
         foreach (var file in files)
@@ -40,7 +40,7 @@ protected override void Provide(Action<IFileValue> pushFileValue, S3AdapterConne
     }
     private List<S3FileItem> GetFileList(S3AdapterConnectionParameters connectionParameters, S3AdapterProviderParameters providerParameters)
     {
-        var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (providerParameters.SubFolder ?? "") : Path.Combine(connectionParameters.RootFolder, providerParameters.SubFolder ?? "");
+        var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (providerParameters.SubFolder ?? "") : StringEx.ConcatenatePath(connectionParameters.RootFolder, providerParameters.SubFolder ?? "");
         using (var client = connectionParameters.CreateBucketConnection())
             return client.ListAsync(folder, providerParameters.Recursive).Result;
     }

diff --git a/src/Paillave.Etl.Sftp/SftpConnectionInfo.ex.cs b/src/Paillave.Etl.Sftp/SftpConnectionInfo.ex.cs
@@ -1,5 +1,6 @@
 using System.Collections.Generic;
 using Renci.SshNet;
+using Paillave.Etl.Core;
 
 namespace Paillave.Etl.Sftp
 {

diff --git a/src/Paillave.Etl.Sftp/SftpFileValue.cs b/src/Paillave.Etl.Sftp/SftpFileValue.cs
@@ -29,7 +29,7 @@ protected void DeleteFileSingleTime()
             using (var client = new SftpClient(connectionInfo))
             {
                 client.Connect();
-                client.DeleteFile(Path.Combine(_folder, Name));
+                client.DeleteFile(StringEx.ConcatenatePath(_folder, Name));
             }
         }
         public override Stream GetContent() => ActionRunner.TryExecute(_connectionInfo.MaxAttempts, GetContentSingleTime);
@@ -39,15 +39,15 @@ private Stream GetContentSingleTime()
             using (var client = new SftpClient(connectionInfo))
             {
                 client.Connect();
-                return new MemoryStream(client.ReadAllBytes(Path.Combine(_folder, Name)));
+                return new MemoryStream(client.ReadAllBytes(StringEx.ConcatenatePath(_folder, Name)));
             }
         }
         private StreamWithResource OpenContentSingleTime()
         {
             var connectionInfo = _connectionInfo.CreateConnectionInfo();
             var client = new SftpClient(connectionInfo);
             client.Connect();
-            return new StreamWithResource(client.OpenRead(Path.Combine(_folder, Name)), client);
+            return new StreamWithResource(client.OpenRead(StringEx.ConcatenatePath(_folder, Name)), client);
         }
 
         public override StreamWithResource OpenContent() => ActionRunner.TryExecute(_connectionInfo.MaxAttempts, OpenContentSingleTime);

diff --git a/src/Paillave.Etl.Sftp/SftpFileValueProcessor.cs b/src/Paillave.Etl.Sftp/SftpFileValueProcessor.cs
@@ -14,15 +14,15 @@ public SftpFileValueProcessor(string code, string name, string connectionName, S
         public override ProcessImpact MemoryFootPrint => ProcessImpact.Average;
         protected override void Process(IFileValue fileValue, SftpAdapterConnectionParameters connectionParameters, SftpAdapterProcessorParameters processorParameters, Action<IFileValue> push, CancellationToken cancellationToken, IExecutionContext context)
         {
-            var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (processorParameters.SubFolder ?? "") : Path.Combine(connectionParameters.RootFolder, processorParameters.SubFolder ?? "");
+            var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (processorParameters.SubFolder ?? "") : StringEx.ConcatenatePath(connectionParameters.RootFolder, processorParameters.SubFolder ?? "");
             using var stream = fileValue.Get(processorParameters.UseStreamCopy);
             byte[] fileContents;
             using (MemoryStream ms = new MemoryStream())
             {
                 stream.CopyTo(ms);
                 fileContents = ms.ToArray();
             }
-            ActionRunner.TryExecute(connectionParameters.MaxAttempts, () => UploadSingleTime(connectionParameters, fileContents, Path.Combine(folder, fileValue.Name)));
+            ActionRunner.TryExecute(connectionParameters.MaxAttempts, () => UploadSingleTime(connectionParameters, fileContents, StringEx.ConcatenatePath(folder, fileValue.Name)));
             push(fileValue);
         }
         private void UploadSingleTime(SftpAdapterConnectionParameters connectionParameters, byte[] fileContents, string filePath)
@@ -39,7 +39,7 @@ private void UploadSingleTime(SftpAdapterConnectionParameters connectionParamete
         protected override void Test(SftpAdapterConnectionParameters connectionParameters, SftpAdapterProcessorParameters processorParameters)
         {
             var fileName = Guid.NewGuid().ToString();
-            var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (processorParameters.SubFolder ?? "") : Path.Combine(connectionParameters.RootFolder, processorParameters.SubFolder ?? "");
+            var folder = string.IsNullOrWhiteSpace(connectionParameters.RootFolder) ? (processorParameters.SubFolder ?? "") : StringEx.ConcatenatePath(connectionParameters.RootFolder, processorParameters.SubFolder ?? "");
             var connectionInfo = connectionParameters.CreateConnectionInfo();
             using (var client = new SftpClient(connectionInfo))
             {
@@ -53,12 +53,12 @@ protected override void Test(SftpAdapterConnectionParameters connectionParameter
                     fileContents = ms.ToArray();
                 }
 
-                client.WriteAllBytes(Path.Combine(folder, fileName), fileContents);
+                client.WriteAllBytes(StringEx.ConcatenatePath(folder, fileName), fileContents);
             }
             using (var client = new SftpClient(connectionInfo))
             {
                 client.Connect();
-                client.DeleteFile(Path.Combine(folder, fileName));
+                client.DeleteFile(StringEx.ConcatenatePath(folder, fileName));
             }
         }
     }

diff --git a/src/Paillave.Etl.Sftp/String.ex.cs b/src/Paillave.Etl.Sftp/String.ex.cs
diff --git a/src/Paillave.Etl.S3/String.ex.cs → src/Paillave.Etl/Core/String.ex.cs b/src/Paillave.Etl.S3/String.ex.cs → src/Paillave.Etl/Core/String.ex.cs
@@ -1,6 +1,8 @@
 using System.IO;
+using System.Linq;
+
+namespace Paillave.Etl.Core;
 
-namespace Paillave.Etl.S3;
 public static class StringEx
 {
     public static Stream ToStream(this string s)
@@ -12,4 +14,8 @@ public static Stream ToStream(this string s)
         stream.Position = 0;
         return stream;
     }
-}
+    public static string ConcatenatePath(params string[] segments)
+    {
+        return string.Join("/", segments.Where(i => !string.IsNullOrWhiteSpace(i))).Replace("//", "/");
+    }
+}
diff --git a/src/SharedSettings.props b/src/SharedSettings.props
@@ -2,7 +2,7 @@
   <PropertyGroup>
     <LangVersion>latest</LangVersion>
     <Nullable>enable</Nullable>
-    <Version>2.2.3-beta</Version>
+    <Version>2.2.4-beta</Version>
     <PackageIcon>NugetIcon.png</PackageIcon>
     <PackageReadmeFile>README.md</PackageReadmeFile>
     <Authors>Stéphane Royer</Authors>