Merge pull request #202 from sebagomez/aws

Support for Aws and GCP

.gitignore

@@ -13,3 +13,6 @@ __azurite*.json
 private*.data
 
 azurestorageexplorer.sln
+credentials.json
+.vscode/
+

README.md

@@ -37,6 +37,16 @@ These variables are `AZURE_STORAGE_CONNECTIONSTRING`, `AZURE_STORAGE_ACCOUNT`, `
 
 If you want to test it against [Azurite](https://github.com/Azure/Azurite) (either locally, via Docker, Docker Compose, or Kubernetes) you'll have to add the `AZURITE` variable set to `true`.
 
+#### AWS & GCP
+
+You can now also manager your AWS and GCP buckets. Just make sure you set the environment variable `CLOUD_PROVIDER` to either `AWS` or `GCP` and you also need to set up a few other environments depending the cloud provider.  
+If you want to connect to your AWS S3 account, you need to provider three environment variables `AWS_ACCESS_KEY`, `AWS_SECRET_KEY`, and `AWS_REGION`.  
+If in the other hand you want to connect to GCP, you need to donload the credentials file for your service account and set the `GCP_CREDENTIALS_FILE` environment variable to the full path to that file. 
+
+![](./res/AWSExplorer.png)
+
+**This feature is in beta so feel free to provide feedback**
+
 ## Exploring
 
 **Blobs**: Create public or private Containers and Blobs (only BlockBlobs for now). Download or delete your blobs.

src/StorageLibrary/AWS/AWSBucket.cs

@@ -0,0 +1,164 @@
+using System.Collections.Generic;
+using System.IO;
+using System.Text.Json;
+using System.Threading.Tasks;
+
+using Amazon;
+using Amazon.Runtime;
+using Amazon.S3;
+using Amazon.S3.Model;
+
+using StorageLibrary.Common;
+using StorageLibrary.Interfaces;
+
+namespace StorageLibrary.AWS
+{
+	internal class AWSBucket : StorageObject, IContainer
+	{
+		protected AmazonS3Client _s3Client;
+		public AWSBucket(StorageFactoryConfig config): base(config)
+		{
+			var credentials = new BasicAWSCredentials(config.AwsKey, config.AwsSecret);
+			_s3Client = new AmazonS3Client(credentials, RegionEndpoint.GetBySystemName(config.AwsRegion));
+		}
+
+		public async Task CreateAsync(string bucket, bool publicAccess)
+		{
+			var putBucketRequest = new PutBucketRequest
+			{
+				BucketName = bucket,
+			};
+
+			await _s3Client.PutBucketAsync(putBucketRequest);
+
+			if (publicAccess)
+			{
+				await SetBucketPolicyAsync(bucket);
+			}
+		}
+
+		public async Task SetBucketPolicyAsync(string bucket)
+		{
+			var bucketPolicy = new
+			{
+				Version = "2012-10-17",
+				Statement = new[]
+				{
+				new
+				{
+					Sid = "AddPerm",
+					Effect = "Allow",
+					Principal = "*",
+					Action = "s3:GetObject",
+					Resource = $"arn:aws:s3:::{bucket}/*"
+				}
+			}
+			};
+
+			var policyJson = JsonSerializer.Serialize(bucketPolicy);
+
+			var putBucketPolicyRequest = new PutBucketPolicyRequest
+			{
+				BucketName = bucket,
+				Policy = policyJson
+			};
+
+			await _s3Client.PutBucketPolicyAsync(putBucketPolicyRequest);
+		}
+
+		public async Task CreateBlobAsync(string bucket, string key, Stream fileContent)
+		{
+			var putRequest = new PutObjectRequest
+			{
+				BucketName = bucket,
+				Key = key,
+				InputStream = fileContent
+			};
+
+			await _s3Client.PutObjectAsync(putRequest);
+		}
+
+		public async Task DeleteAsync(string bucket)
+		{
+			var deleteBucketRequest = new DeleteBucketRequest
+			{
+				BucketName = bucket
+			};
+
+			await _s3Client.DeleteBucketAsync(deleteBucketRequest);
+		}
+
+		public async Task DeleteBlobAsync(string bucket, string key)
+		{
+			var deleteObjectRequest = new DeleteObjectRequest
+			{
+				BucketName = bucket,
+				Key = key
+			};
+
+			await _s3Client.DeleteObjectAsync(deleteObjectRequest);
+		}
+
+		public async Task<string> GetBlobAsync(string bucket, string key)
+		{
+			string tmpPath = Util.File.GetTempFileName();
+
+			var getRequest = new GetObjectRequest
+			{
+				BucketName = bucket,
+				Key = key
+			};
+
+			using (GetObjectResponse response = await _s3Client.GetObjectAsync(getRequest))
+			using (Stream responseStream = response.ResponseStream)
+			using (FileStream fileStream = File.Create(tmpPath))
+			{
+				await responseStream.CopyToAsync(fileStream);
+			}
+
+			return tmpPath;
+		}
+
+		public async Task<List<BlobItemWrapper>> ListBlobsAsync(string bucket, string path)
+		{
+			var request = new ListObjectsV2Request
+			{
+				BucketName = bucket,
+				Prefix = path,
+				Delimiter = Path.AltDirectorySeparatorChar.ToString()
+			};
+
+
+			var blobs = new List<BlobItemWrapper>();
+			var response = await _s3Client.ListObjectsV2Async(request);
+
+			var uriTemplate = $"https://{bucket}.s3.{RegionEndpoint.USEast1.SystemName}.amazonaws.com/";
+
+			foreach (S3Object entry in response.S3Objects)
+			{
+				if (entry.Key == path)
+					continue;
+
+				blobs.Add(new BlobItemWrapper($"{uriTemplate}{entry.Key}", entry.Size, CloudProvider.AWS));
+			}
+
+			foreach (string commonPrefix in response.CommonPrefixes)
+				blobs.Add(new BlobItemWrapper($"{uriTemplate}{commonPrefix}", 0, CloudProvider.AWS));
+
+			return blobs;
+		}
+
+		public async Task<List<CloudBlobContainerWrapper>> ListContainersAsync()
+		{
+			var buckets = new List<CloudBlobContainerWrapper>();
+
+			ListBucketsResponse response = await _s3Client.ListBucketsAsync();
+			foreach (S3Bucket bucket in response.Buckets)
+			{
+				buckets.Add(new CloudBlobContainerWrapper() { Name = bucket.BucketName });
+			}
+
+			return buckets;
+		}
+	}
+}

src/StorageLibrary/AzureContainer.cs → src/StorageLibrary/Azure/AzureContainer.cs

@@ -9,7 +9,7 @@
 using StorageLibrary.Common;
 using StorageLibrary.Interfaces;
 
-namespace StorageLibrary
+namespace StorageLibrary.Azure
 {
 	internal class AzureContainer : StorageObject, IContainer
 	{
@@ -45,11 +45,11 @@ public async Task<List<BlobItemWrapper>> ListBlobsAsync(string containerName, st
 				{
 					BlobClient blobClient = container.GetBlobClient(blobItem.Blob.Name);
 
-					wrapper = new BlobItemWrapper(blobClient.Uri.AbsoluteUri, blobItem.Blob.Properties.ContentLength.HasValue ? blobItem.Blob.Properties.ContentLength.Value : 0, IsAzurite);
+					wrapper = new BlobItemWrapper(blobClient.Uri.AbsoluteUri, blobItem.Blob.Properties.ContentLength.HasValue ? blobItem.Blob.Properties.ContentLength.Value : 0, CloudProvider.Azure, IsAzurite);
 				}
 				else if (blobItem.IsPrefix)
 				{
-					wrapper = new BlobItemWrapper($"{container.Uri}/{blobItem.Prefix}", 0, IsAzurite);
+					wrapper = new BlobItemWrapper($"{container.Uri}/{blobItem.Prefix}", 0, CloudProvider.Azure, IsAzurite);
 				}
 
 				if (wrapper != null && !results.Contains(wrapper))

src/StorageLibrary/AzureFile.cs → src/StorageLibrary/Azure/AzureFile.cs

@@ -10,7 +10,7 @@
 using StorageLibrary.Common;
 using StorageLibrary.Interfaces;
 
-namespace StorageLibrary
+namespace StorageLibrary.Azure
 {
 	internal class AzureFile : StorageObject, IFile
 	{

src/StorageLibrary/AzureQueue.cs → src/StorageLibrary/Azure/AzureQueue.cs

@@ -7,7 +7,7 @@
 using StorageLibrary.Common;
 using StorageLibrary.Interfaces;
 
-namespace StorageLibrary
+namespace StorageLibrary.Azure
 {
 	internal class AzureQueue : StorageObject, IQueue
 	{

src/StorageLibrary/AzureTable.cs → src/StorageLibrary/Azure/AzureTable.cs

@@ -6,7 +6,7 @@
 using StorageLibrary.Common;
 using StorageLibrary.Interfaces;
 
-namespace StorageLibrary
+namespace StorageLibrary.Azure
 {
 	internal class AzureTable : StorageObject, ITable
 	{

src/StorageLibrary/Common/BlobItemWrapper.cs

@@ -20,22 +20,43 @@ public string Url
 
 		public long Size { get; private set; }
 
+		public CloudProvider Provider { get; private set; }
+
 		public decimal SizeInKBs { get => (decimal)Size / 1024; }
 
 		public decimal SizeInMBs { get => (decimal)Size / 1024 / 1024; }
 
-		public BlobItemWrapper(string url) : this(url, 0) { }
+		public BlobItemWrapper(string url) : this(url, 0, CloudProvider.Azure) { }
 
-		public BlobItemWrapper(string url, long size, bool fromAzurite = false)
+		public BlobItemWrapper(string url, long size, CloudProvider provider, bool fromAzurite = false)
 		{
 			Url = url;
 			Size = size;
+			Provider = provider;
 			m_isAzurite = fromAzurite;
 			IsFile = !m_internalUri.Segments[m_internalUri.Segments.Length - 1].EndsWith(System.IO.Path.AltDirectorySeparatorChar);
-			Container = m_isAzurite ? m_internalUri.Segments[2] : m_internalUri.Segments[1];
 			Name = HttpUtility.UrlDecode(m_internalUri.Segments[m_internalUri.Segments.Length - 1]);
-			int containerPos = m_internalUri.LocalPath.IndexOf(Container) + Container.Length;
-			Path = m_internalUri.LocalPath.Substring(containerPos, (m_internalUri.LocalPath.Length) - (containerPos) - Name.Length);
+
+			switch (provider)
+			{
+				case CloudProvider.Azure:
+					Container = m_isAzurite ? m_internalUri.Segments[2] : m_internalUri.Segments[1];
+					int containerPos = m_internalUri.LocalPath.IndexOf(Container) + Container.Length;
+					Path = m_internalUri.LocalPath.Substring(containerPos, (m_internalUri.LocalPath.Length) - (containerPos) - Name.Length);
+					break;
+				case CloudProvider.AWS:
+					Container = m_internalUri.Host.Split('.')[0];
+					Path = m_internalUri.LocalPath.Substring(1, m_internalUri.LocalPath.Length - 1 - Name.Length);
+					break;
+				case CloudProvider.GCP:
+					Container = m_internalUri.Segments[1];
+					int bucketPos = m_internalUri.LocalPath.IndexOf(Container) + Container.Length;
+					Path = m_internalUri.LocalPath.Substring(bucketPos, (m_internalUri.LocalPath.Length) - (bucketPos) - Name.Length);
+					break;
+				default:
+					throw new ApplicationException($"Invalid provider: {provider}");
+			}
+
 		}
 
 		public int CompareTo(BlobItemWrapper other)