From a5382e2968ea3bb6eeb446450bb32ca5760bc8b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=B6tz=20Jensen?= Date: Sun, 12 Jan 2020 21:08:39 +0100 Subject: [PATCH 1/8] Refactor: Update requried module from Azure.Storage to Az.Storage --- d365fo.tools/d365fo.tools.psd1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/d365fo.tools/d365fo.tools.psd1 b/d365fo.tools/d365fo.tools.psd1 index e9bdc0ea..bd65f5e4 100644 --- a/d365fo.tools/d365fo.tools.psd1 +++ b/d365fo.tools/d365fo.tools.psd1 @@ -27,7 +27,7 @@ # this module RequiredModules = @( @{ ModuleName = 'PSFramework'; ModuleVersion = '1.0.12' } - , @{ ModuleName = 'Azure.Storage'; ModuleVersion = '4.4.0' } + , @{ ModuleName = 'Az.Storage'; ModuleVersion = '1.11.0' } , @{ ModuleName = 'AzureAd'; ModuleVersion = '2.0.1.16' } , @{ ModuleName = 'PSNotification'; ModuleVersion = '0.5.3' } , @{ ModuleName = 'PSOAuthHelper'; ModuleVersion = '0.2.3' } From f5819a38f2ff1a41d962e01270d3a5c59ebf6e01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=B6tz=20Jensen?= Date: Sun, 12 Jan 2020 21:09:02 +0100 Subject: [PATCH 2/8] Refactor: Use cmdlets from the Az.Storage module directly --- .../invoke-d365azurestorageupload.ps1 | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/d365fo.tools/functions/invoke-d365azurestorageupload.ps1 b/d365fo.tools/functions/invoke-d365azurestorageupload.ps1 index 2406080a..4da7828a 100644 --- a/d365fo.tools/functions/invoke-d365azurestorageupload.ps1 +++ b/d365fo.tools/functions/invoke-d365azurestorageupload.ps1 @@ -84,6 +84,8 @@ function Invoke-D365AzureStorageUpload { [Alias('Path')] [string] $Filepath, + [switch] $Force, + [switch] $DeleteOnUpload, [switch] $EnableException @@ -101,32 +103,26 @@ function Invoke-D365AzureStorageUpload { if (Test-PSFFunctionInterrupt) { return } Invoke-TimeSignal -Start + + $FileName = Split-Path -Path $Filepath -Leaf try { if ([string]::IsNullOrEmpty($SAS)) { Write-PSFMessage -Level Verbose -Message "Working against Azure Storage Account with AccessToken" - $storageContext = new-AzureStorageContext -StorageAccountName $AccountId.ToLower() -StorageAccountKey $AccessToken + $storageContext = New-AzStorageContext -StorageAccountName $AccountId.ToLower() -StorageAccountKey $AccessToken } else { $conString = $("BlobEndpoint=https://{0}.blob.core.windows.net/;QueueEndpoint=https://{0}.queue.core.windows.net/;FileEndpoint=https://{0}.file.core.windows.net/;TableEndpoint=https://{0}.table.core.windows.net/;SharedAccessSignature={1}" -f $AccountId.ToLower(), $SAS) Write-PSFMessage -Level Verbose -Message "Working against Azure Storage Account with SAS" -Target $conString - $storageContext = new-AzureStorageContext -ConnectionString $conString + $storageContext = New-AzStorageContext -ConnectionString $conString } - $cloudStorageAccount = [Microsoft.WindowsAzure.Storage.CloudStorageAccount]::Parse($storageContext.ConnectionString) - - $blobClient = $cloudStorageAccount.CreateCloudBlobClient() - - $blobContainer = $blobClient.GetContainerReference($Container.ToLower()); - Write-PSFMessage -Level Verbose -Message "Start uploading the file to Azure" - $FileName = Split-Path $Filepath -Leaf - $blockBlob = $blobContainer.GetBlockBlobReference($FileName) - $blockBlob.UploadFromFile($Filepath) + Set-AzStorageBlobContent -Context $storageContext -File $Filepath -Container $($Container.ToLower()) -Force:$Force if ($DeleteOnUpload) { Remove-Item $Filepath -Force From 5182b26ba0ce792751bfc70e4bc693c6ca0e3034 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=B6tz=20Jensen?= Date: Sun, 12 Jan 2020 21:38:38 +0100 Subject: [PATCH 3/8] Refactor: Update to use native cmdlets from Az.Storage module --- .../functions/get-d365azurestoragefile.ps1 | 16 +++++----------- d365fo.tools/xml/d365fo.tools.Format.ps1xml | 6 ------ 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/d365fo.tools/functions/get-d365azurestoragefile.ps1 b/d365fo.tools/functions/get-d365azurestoragefile.ps1 index 64829829..68ae0688 100644 --- a/d365fo.tools/functions/get-d365azurestoragefile.ps1 +++ b/d365fo.tools/functions/get-d365azurestoragefile.ps1 @@ -96,33 +96,27 @@ function Get-D365AzureStorageFile { if ([string]::IsNullOrEmpty($SAS)) { Write-PSFMessage -Level Verbose -Message "Working against Azure Storage Account with AccessToken" - $storageContext = new-AzureStorageContext -StorageAccountName $AccountId.ToLower() -StorageAccountKey $AccessToken + $storageContext = New-AzStorageContext -StorageAccountName $AccountId.ToLower() -StorageAccountKey $AccessToken } else { Write-PSFMessage -Level Verbose -Message "Working against Azure Storage Account with SAS" $conString = $("BlobEndpoint=https://{0}.blob.core.windows.net/;QueueEndpoint=https://{0}.queue.core.windows.net/;FileEndpoint=https://{0}.file.core.windows.net/;TableEndpoint=https://{0}.table.core.windows.net/;SharedAccessSignature={1}" -f $AccountId.ToLower(), $SAS) - $storageContext = new-AzureStorageContext -ConnectionString $conString + $storageContext = New-AzStorageContext -ConnectionString $conString } - $cloudStorageAccount = [Microsoft.WindowsAzure.Storage.CloudStorageAccount]::Parse($storageContext.ConnectionString) - - $blobClient = $cloudStorageAccount.CreateCloudBlobClient() - - $blobcontainer = $blobClient.GetContainerReference($Container); - try { - $files = $blobcontainer.ListBlobs() | Sort-Object -Descending { $_.Properties.LastModified } + $files = Get-AzStorageBlob -Container $($Container.ToLower()) -Context $storageContext| Sort-Object -Descending { $_.Properties.LastModified } if ($Latest) { - $files | Select-Object -First 1 | Select-PSFObject -TypeName D365FO.TOOLS.Azure.Blob "name", @{Name = "Size"; Expression = {[PSFSize]$_.Properties.Length}}, "IsDeleted", @{Name = "LastModified"; Expression = {[Datetime]::Parse($_.Properties.LastModified)}} + $files | Select-Object -First 1 | Select-PSFObject -TypeName D365FO.TOOLS.Azure.Blob "name", @{Name = "Size"; Expression = {[PSFSize]$_.Length}}, @{Name = "LastModified"; Expression = {[Datetime]::Parse($_.LastModified)}} } else { foreach ($obj in $files) { if ($obj.Name -NotLike $Name) { continue } - $obj | Select-PSFObject -TypeName D365FO.TOOLS.Azure.Blob "name", @{Name = "Size"; Expression = {[PSFSize]$_.Properties.Length}}, "IsDeleted", @{Name = "LastModified"; Expression = {[Datetime]::Parse($_.Properties.LastModified)}} + $obj | Select-PSFObject -TypeName D365FO.TOOLS.Azure.Blob "name", @{Name = "Size"; Expression = {[PSFSize]$_.Length}}, @{Name = "LastModified"; Expression = {[Datetime]::Parse($_.LastModified)}} } } } diff --git a/d365fo.tools/xml/d365fo.tools.Format.ps1xml b/d365fo.tools/xml/d365fo.tools.Format.ps1xml index 1bb2e90a..9f793bd8 100644 --- a/d365fo.tools/xml/d365fo.tools.Format.ps1xml +++ b/d365fo.tools/xml/d365fo.tools.Format.ps1xml @@ -135,9 +135,6 @@ 12 Right - - 9 - 30 @@ -151,9 +148,6 @@ Size - - IsDeleted - LastModified From 16975ce35943a9556729375901dfe34d00ed0d52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=B6tz=20Jensen?= Date: Sun, 12 Jan 2020 21:53:14 +0100 Subject: [PATCH 4/8] Formatting: missing space --- d365fo.tools/functions/get-d365azurestoragefile.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/d365fo.tools/functions/get-d365azurestoragefile.ps1 b/d365fo.tools/functions/get-d365azurestoragefile.ps1 index 68ae0688..d288899e 100644 --- a/d365fo.tools/functions/get-d365azurestoragefile.ps1 +++ b/d365fo.tools/functions/get-d365azurestoragefile.ps1 @@ -106,7 +106,7 @@ function Get-D365AzureStorageFile { } try { - $files = Get-AzStorageBlob -Container $($Container.ToLower()) -Context $storageContext| Sort-Object -Descending { $_.Properties.LastModified } + $files = Get-AzStorageBlob -Container $($Container.ToLower()) -Context $storageContext | Sort-Object -Descending { $_.Properties.LastModified } if ($Latest) { $files | Select-Object -First 1 | Select-PSFObject -TypeName D365FO.TOOLS.Azure.Blob "name", @{Name = "Size"; Expression = {[PSFSize]$_.Length}}, @{Name = "LastModified"; Expression = {[Datetime]::Parse($_.LastModified)}} From 642a429c120c182802ff1210374ee9023662f4f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=B6tz=20Jensen?= Date: Sun, 12 Jan 2020 21:53:51 +0100 Subject: [PATCH 5/8] Formatting: Spaces included --- d365fo.tools/functions/get-d365azurestoragefile.ps1 | 4 ++-- d365fo.tools/functions/invoke-d365azurestorageupload.ps1 | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/d365fo.tools/functions/get-d365azurestoragefile.ps1 b/d365fo.tools/functions/get-d365azurestoragefile.ps1 index d288899e..a7313fb9 100644 --- a/d365fo.tools/functions/get-d365azurestoragefile.ps1 +++ b/d365fo.tools/functions/get-d365azurestoragefile.ps1 @@ -109,14 +109,14 @@ function Get-D365AzureStorageFile { $files = Get-AzStorageBlob -Container $($Container.ToLower()) -Context $storageContext | Sort-Object -Descending { $_.Properties.LastModified } if ($Latest) { - $files | Select-Object -First 1 | Select-PSFObject -TypeName D365FO.TOOLS.Azure.Blob "name", @{Name = "Size"; Expression = {[PSFSize]$_.Length}}, @{Name = "LastModified"; Expression = {[Datetime]::Parse($_.LastModified)}} + $files | Select-Object -First 1 | Select-PSFObject -TypeName D365FO.TOOLS.Azure.Blob "name", @{Name = "Size"; Expression = { [PSFSize]$_.Length } }, @{Name = "LastModified"; Expression = { [Datetime]::Parse($_.LastModified) } } } else { foreach ($obj in $files) { if ($obj.Name -NotLike $Name) { continue } - $obj | Select-PSFObject -TypeName D365FO.TOOLS.Azure.Blob "name", @{Name = "Size"; Expression = {[PSFSize]$_.Length}}, @{Name = "LastModified"; Expression = {[Datetime]::Parse($_.LastModified)}} + $obj | Select-PSFObject -TypeName D365FO.TOOLS.Azure.Blob "name", @{Name = "Size"; Expression = { [PSFSize]$_.Length } }, @{Name = "LastModified"; Expression = { [Datetime]::Parse($_.LastModified) } } } } } diff --git a/d365fo.tools/functions/invoke-d365azurestorageupload.ps1 b/d365fo.tools/functions/invoke-d365azurestorageupload.ps1 index 4da7828a..76901c71 100644 --- a/d365fo.tools/functions/invoke-d365azurestorageupload.ps1 +++ b/d365fo.tools/functions/invoke-d365azurestorageupload.ps1 @@ -144,5 +144,5 @@ function Invoke-D365AzureStorageUpload { } } - END {} + END { } } \ No newline at end of file From 5b009de4810cee90dc96cbcc0e6a39608d34a318 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=B6tz=20Jensen?= Date: Sun, 12 Jan 2020 21:56:38 +0100 Subject: [PATCH 6/8] Refactor: Use native cmdlets from Az.Storage --- .../invoke-d365azurestoragedownload.ps1 | 42 +++++++++---------- .../invoke-d365azurestorageupload.ps1 | 2 +- 2 files changed, 20 insertions(+), 24 deletions(-) diff --git a/d365fo.tools/functions/invoke-d365azurestoragedownload.ps1 b/d365fo.tools/functions/invoke-d365azurestoragedownload.ps1 index 10b13a8f..27f4c418 100644 --- a/d365fo.tools/functions/invoke-d365azurestoragedownload.ps1 +++ b/d365fo.tools/functions/invoke-d365azurestoragedownload.ps1 @@ -78,16 +78,12 @@ function Invoke-D365AzureStorageDownload { [CmdletBinding(DefaultParameterSetName = 'Default')] param ( - [Parameter(Mandatory = $false)] [string] $AccountId = $Script:AccountId, - [Parameter(Mandatory = $false)] [string] $AccessToken = $Script:AccessToken, - [Parameter(Mandatory = $false)] [string] $SAS = $Script:SAS, - [Parameter(Mandatory = $false)] [Alias('Blob')] [Alias('Blobname')] [string] $Container = $Script:Container, @@ -96,13 +92,14 @@ function Invoke-D365AzureStorageDownload { [Alias('Name')] [string] $FileName, - [Parameter(Mandatory = $false)] [string] $Path = $Script:DefaultTempPath, [Parameter(Mandatory = $true, ParameterSetName = 'Latest', Position = 4 )] [Alias('GetLatest')] [switch] $Latest, + [switch] $Force, + [switch] $EnableException ) @@ -120,7 +117,7 @@ function Invoke-D365AzureStorageDownload { } } PROCESS { - if (Test-PSFFunctionInterrupt) {return} + if (Test-PSFFunctionInterrupt) { return } Invoke-TimeSignal -Start @@ -129,41 +126,40 @@ function Invoke-D365AzureStorageDownload { if ([string]::IsNullOrEmpty($SAS)) { Write-PSFMessage -Level Verbose -Message "Working against Azure Storage Account with AccessToken" - $storageContext = new-AzureStorageContext -StorageAccountName $AccountId.ToLower() -StorageAccountKey $AccessToken + $storageContext = New-AzStorageContext -StorageAccountName $AccountId.ToLower() -StorageAccountKey $AccessToken } else { Write-PSFMessage -Level Verbose -Message "Working against Azure Storage Account with SAS" $conString = $("BlobEndpoint=https://{0}.blob.core.windows.net/;QueueEndpoint=https://{0}.queue.core.windows.net/;FileEndpoint=https://{0}.file.core.windows.net/;TableEndpoint=https://{0}.table.core.windows.net/;SharedAccessSignature={1}" -f $AccountId.ToLower(), $SAS) - $storageContext = new-AzureStorageContext -ConnectionString $conString + $storageContext = New-AzStorageContext -ConnectionString $conString } - $cloudStorageAccount = [Microsoft.WindowsAzure.Storage.CloudStorageAccount]::Parse($storageContext.ConnectionString) - - $blobClient = $cloudStorageAccount.CreateCloudBlobClient() - - $blobContainer = $blobClient.GetContainerReference($Container.ToLower()); - Write-PSFMessage -Level Verbose -Message "Start download from Azure Storage Account" if ($Latest) { - $files = $blobContainer.ListBlobs() - $File = ($files | Sort-Object -Descending { $_.Properties.LastModified } | Select-Object -First 1) - - $NewFile = Join-Path $Path $($File.Name) + $files = Get-AzStorageBlob -Container $($Container.ToLower()) -Context $storageContext - $File.DownloadToFile($NewFile, [System.IO.FileMode]::Create) + $File = ($files | Sort-Object -Descending { $_.LastModified } | Select-Object -First 1) $FileName = $File.Name + + Write-PSFMessage -Level Verbose -Message "Filename is: $FileName" + + $NewFile = Join-Path $Path $($File.Name) + + $null = Get-AzStorageBlobContent -Container $($Container.ToLower()) -Blob $File.Name -Destination $NewFile -Context $storageContext -Force:$Force } else { + + Write-PSFMessage -Level Verbose -Message "Filename is: $FileName" + $NewFile = Join-Path $Path $FileName - $blockBlob = $blobContainer.GetBlockBlobReference($FileName); - $blockBlob.DownloadToFile($NewFile, [System.IO.FileMode]::Create) + $null = Get-AzStorageBlobContent -Container $($Container.ToLower()) -Blob $FileName -Destination $NewFile -Context $storageContext -Force:$Force } - Get-Item -Path $NewFile | Select-PSFObject "Name as Filename", @{Name = "Size"; Expression = {[PSFSize]$_.Length}}, "LastWriteTime as LastModified", "Fullname as File" + Get-Item -Path $NewFile | Select-PSFObject "Name as Filename", @{Name = "Size"; Expression = { [PSFSize]$_.Length } }, "LastWriteTime as LastModified", "Fullname as File" } catch { $messageString = "Something went wrong while downloading the file from Azure." @@ -176,5 +172,5 @@ function Invoke-D365AzureStorageDownload { } } - END {} + END { } } \ No newline at end of file diff --git a/d365fo.tools/functions/invoke-d365azurestorageupload.ps1 b/d365fo.tools/functions/invoke-d365azurestorageupload.ps1 index 76901c71..67bd0d44 100644 --- a/d365fo.tools/functions/invoke-d365azurestorageupload.ps1 +++ b/d365fo.tools/functions/invoke-d365azurestorageupload.ps1 @@ -122,7 +122,7 @@ function Invoke-D365AzureStorageUpload { Write-PSFMessage -Level Verbose -Message "Start uploading the file to Azure" - Set-AzStorageBlobContent -Context $storageContext -File $Filepath -Container $($Container.ToLower()) -Force:$Force + $null = Set-AzStorageBlobContent -Context $storageContext -File $Filepath -Container $($Container.ToLower()) -Force:$Force if ($DeleteOnUpload) { Remove-Item $Filepath -Force From 5cf88fcde8bc9cd8aee3924f1c71fabcfdc575c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=B6tz=20Jensen?= Date: Sun, 12 Jan 2020 22:07:05 +0100 Subject: [PATCH 7/8] Refactor: Use Az.Storage instead of Azure.Storage --- build/vsts-prerequisites.ps1 | 2 +- build/vsts-validate.ps1 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build/vsts-prerequisites.ps1 b/build/vsts-prerequisites.ps1 index c37de7b3..b0646cb8 100644 --- a/build/vsts-prerequisites.ps1 +++ b/build/vsts-prerequisites.ps1 @@ -1,7 +1,7 @@ Write-Host "Working on the machine named: $($env:computername)" Write-Host "The user running is: $($env:UserName)" -$modules = @("Pester", "PSFramework", "PSScriptAnalyzer", "Azure.Storage", "AzureAd", "PSNotification", "PSOAuthHelper", "PowerShellGet", "PackageManagement") +$modules = @("Pester", "PSFramework", "PSScriptAnalyzer", "Az.Storage", "AzureAd", "PSNotification", "PSOAuthHelper", "PowerShellGet", "PackageManagement") foreach ($module in $modules) { Write-Host "Installing $module" -ForegroundColor Cyan diff --git a/build/vsts-validate.ps1 b/build/vsts-validate.ps1 index 9d2fb14e..829a20d2 100644 --- a/build/vsts-validate.ps1 +++ b/build/vsts-validate.ps1 @@ -8,7 +8,7 @@ Write-Host "Working on the machine named: $($env:computername)" Write-Host "The user running is: $($env:UserName)" -$modules = @("Pester", "PSFramework", "PSScriptAnalyzer", "Azure.Storage", "AzureAd", "PSNotification") +$modules = @("Pester", "PSFramework", "PSScriptAnalyzer", "Az.Storage", "AzureAd", "PSNotification") foreach ($module in $modules) { Write-Host "Importing $module" -ForegroundColor Cyan From 80b1edfa2bc81f9921ffff0e53b3e1884822fb3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=B6tz=20Jensen?= Date: Sun, 12 Jan 2020 22:19:49 +0100 Subject: [PATCH 8/8] Tests: Fixing parameter unit tests --- d365fo.tools/functions/invoke-d365azurestoragedownload.ps1 | 3 +++ d365fo.tools/functions/invoke-d365azurestorageupload.ps1 | 3 +++ 2 files changed, 6 insertions(+) diff --git a/d365fo.tools/functions/invoke-d365azurestoragedownload.ps1 b/d365fo.tools/functions/invoke-d365azurestoragedownload.ps1 index 27f4c418..99648439 100644 --- a/d365fo.tools/functions/invoke-d365azurestoragedownload.ps1 +++ b/d365fo.tools/functions/invoke-d365azurestoragedownload.ps1 @@ -29,6 +29,9 @@ .PARAMETER Latest Instruct the cmdlet to download the latest file from Azure regardless of name + .PARAMETER Force + Instruct the cmdlet to overwrite the local file if it already exists + .PARAMETER EnableException This parameters disables user-friendly warnings and enables the throwing of exceptions This is less user friendly, but allows catching exceptions in calling scripts diff --git a/d365fo.tools/functions/invoke-d365azurestorageupload.ps1 b/d365fo.tools/functions/invoke-d365azurestorageupload.ps1 index 67bd0d44..0885d467 100644 --- a/d365fo.tools/functions/invoke-d365azurestorageupload.ps1 +++ b/d365fo.tools/functions/invoke-d365azurestorageupload.ps1 @@ -21,6 +21,9 @@ .PARAMETER Filepath Path to the file you want to upload + .PARAMETER Force + Instruct the cmdlet to overwrite the file in the container if it already exists + .PARAMETER DeleteOnUpload Switch to tell the cmdlet if you want the local file to be deleted after the upload completes