Merge pull request #44 from psrenergy/rs/psrmodels

Add deploy to psrmodels

.github/workflows/test.yml

@@ -23,5 +23,10 @@ jobs:
           fetch-depth: 0
 
       - name: Test
+        env: 
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          SLACK_BOT_USER_OAUTH_ACCESS_TOKEN: ${{ secrets.SLACK_BOT_USER_OAUTH_ACCESS_TOKEN }}      
         run: |
           .\test\test.bat

.gitignore

@@ -26,5 +26,6 @@ format/Manifest.toml
 revise/Manifest.toml
 
 test/Example.jl/compile/build/
+test/Example.jl/compile/setup/
 test/Example.jl/compile/Example.log
 test/Example.jl/compile/build.ok

.vscode/settings.json

@@ -1,3 +1,3 @@
 {
-    "julia.executablePath": "${env:JULIA_194}"
+    "julia.executablePath": "${env:JULIA_1106}"
 }

Project.toml

@@ -19,15 +19,15 @@ UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 p7zip_jll = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"
 
 [compat]
-AWS = "1.89"
+AWS = "1.92"
 Git = "1.3"
 HTTP = "1.9"
 JSON = "0.21"
-LoggingPolyglot = "0.1"
-PackageCompiler = "2.1.16"
+LoggingPolyglot = "0.1 - 0.3"
+PackageCompiler = "2.2"
 SlackAPI = "0.1"
 TOML = "1.0"
-julia = "1.6"
+julia = "1.10"
 
 [extras]
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

format/format.bat

@@ -2,4 +2,4 @@
 
 SET BASEPATH=%~dp0
 
-CALL "%JULIA_194%" --project=%BASEPATH% %BASEPATH%\format.jl
+CALL "%JULIA_1106%" --project=%BASEPATH% %BASEPATH%\format.jl

revise/revise.bat

@@ -2,4 +2,4 @@
 
 SET BASEPATH=%~dp0
 
-CALL "%JULIA_194%" --project=%BASEPATH% --load=%BASEPATH%\revise.jl
+CALL "%JULIA_1106%" --project=%BASEPATH% --load=%BASEPATH%\revise.jl

src/PSRContinuousDeployment.jl

@@ -27,6 +27,7 @@ export Configuration,
     is_release_tag_available,
     bundle_psrhub,
     create_setup,
+    deploy_to_psrmodels,
     deploy_to_psrmodules,
     notify_slack_channel,
     create_release
@@ -46,6 +47,7 @@ include("slack.jl")
 include("setup.jl")
 include("testrunner.jl")
 
+include("deploy/psrmodels.jl")
 include("deploy/psrmodules.jl")
 include("deploy/psrcloud.jl")
 include("deploy/distribution.jl")

src/deploy/psrmodels.jl

@@ -0,0 +1,83 @@
+function generate_unique_key(; bucket::AbstractString, version::AbstractString, target::AbstractString, setup_zip::AbstractString, overwrite::Bool = false)
+    objects = S3.list_objects_v2(bucket, Dict("prefix" => target))
+    if haskey(objects, "Contents")
+        for contents in objects["Contents"]
+            key = contents["Key"]
+            if startswith(key, "$target/$version/")
+                if overwrite
+                    Log.info("PSRMODELS: Overwriting the $setup_zip in the $bucket bucket")
+                    return key
+                else
+                    Log.fatal_error("PSRMODELS: The $setup_zip already exists in the $bucket bucket")
+                end
+            end
+        end
+    end
+
+    for _ in 1:10
+        hash = randstring(['a':'z'; '0':'9'], 4)
+        key = "$target/$version/$hash/$setup_zip"
+
+        objects = S3.list_objects_v2(bucket, Dict("prefix" => target))
+        if haskey(objects, "Contents")
+            unique = true
+            for contents in objects["Contents"]
+                if key == contents["Key"]
+                    unique = false
+                    break
+                end
+            end
+
+            if unique
+                Log.info("PSRMODELS: Generated a unique key: $key")
+                return key
+            end
+        else
+            return key
+        end
+    end
+
+    error("Failed to generate a unique hash")
+end
+
+function deploy_to_psrmodels(;
+    configuration::Configuration,
+    aws_access_key::AbstractString,
+    aws_secret_key::AbstractString,
+    overwrite::Bool = false,
+)
+    bucket = "psr-models"
+
+    target = configuration.target
+    version = configuration.version
+    setup_path = configuration.setup_path
+
+    setup_exe = "$target-$version-setup.exe"
+    setup_exe_path = joinpath(setup_path, setup_exe)
+
+    setup_zip = "$target-$version.zip"
+    setup_zip_path = joinpath(setup_path, setup_zip)
+
+    Log.info("PSRMODELS: Zipping $setup_exe")
+    run(`$(p7zip_jll.p7zip()) a -tzip $setup_zip_path $setup_exe_path`)
+    @assert isfile(setup_zip_path)
+
+    aws_credentials = AWSCredentials(aws_access_key, aws_secret_key)
+    aws_config = AWSConfig(; creds = aws_credentials, region = "us-east-1")
+    global_aws_config(aws_config)
+
+    key = generate_unique_key(
+        bucket = bucket,
+        version = version,
+        target = target,
+        setup_zip = setup_zip,
+        overwrite = overwrite,
+    )
+
+    Log.info("PSRMODELS: Uploading $setup_zip")
+    S3.put_object(bucket, key, Dict("body" => read(setup_zip_path)))
+
+    Log.info("PSRMODELS: Success")
+
+    return "https://models.psr-inc.com/$key"
+end

src/slack.jl

@@ -13,3 +13,19 @@ function notify_slack_channel(configuration::Configuration, slack_token::Abstrac
 
     return nothing
 end
+
+function notify_slack_channel(; configuration::Configuration, slack_token::AbstractString, channel::AbstractString, url::AbstractString)
+    Log.info("NOTIFY: Notifying the Slack channel of code $channel")
+    target = configuration.target
+    version = configuration.version
+    message = "$target v$version has been published: $url"
+
+    context = SlackContext(slack_token)
+    response = SlackAPI.channel_message(context, channel, message)
+    if response.status != 200
+        Log.warn("NOTIFY: Failed to notify the Slack channel of code $channel. Status: $(response.status)")
+        return nothing
+    end
+
+    return nothing
+end

test/Example.jl/Example.bat

@@ -2,4 +2,4 @@
 
 SET BASEPATH=%~dp0
 
-CALL "%JULIA_194%" --project=%BASEPATH% %BASEPATH%\main.jl %*
+CALL "%JULIA_1106%" --project=%BASEPATH% %BASEPATH%\main.jl %*

test/runtests.jl

@@ -2,13 +2,19 @@ using PSRContinuousDeployment
 
 using Test
 
+const SLACK_CHANNEL = "C03SSPFNTJS"
+
+const AWS_ACCESS = ENV["AWS_ACCESS_KEY_ID"]
+const AWS_SECRET = ENV["AWS_SECRET_ACCESS_KEY"]
+const SLACK_TOKEN = ENV["SLACK_BOT_USER_OAUTH_ACCESS_TOKEN"]
+
 function testall()
     package_path = joinpath(@__DIR__, "Example.jl")
     database_path = joinpath(package_path, "database")
 
     configuration = PSRContinuousDeployment.Configuration(
         package_path,
-        development_stage = PSRContinuousDeployment.DevelopmentStage.Alpha,
+        development_stage = PSRContinuousDeployment.DevelopmentStage.StableRelease,
     )
 
     PSRContinuousDeployment.compile(
@@ -22,6 +28,25 @@ function testall()
         skip_version_jl = true,
     )
 
+    create_setup(
+        configuration,
+        sign = true,
+    )
+
+    url = deploy_to_psrmodels(
+        configuration = configuration,
+        aws_access_key = AWS_ACCESS,
+        aws_secret_key = AWS_SECRET,
+        overwrite = true,
+    )
+
+    notify_slack_channel(
+        configuration = configuration,
+        slack_token = SLACK_TOKEN,
+        channel = SLACK_CHANNEL,
+        url = url,
+    )
+
     return 0
 end
 

test/test.bat

@@ -2,4 +2,4 @@
 
 SET BASEPATH=%~dp0
 
-CALL "%JULIA_194%" --project=%BASEPATH%\.. -e "import Pkg; Pkg.test()"
+CALL "%JULIA_1106%" --project=%BASEPATH%\.. -e "import Pkg; Pkg.test()"