Merge pull request #45 from MadhurangaWije/slalpha5

Add build pipeline improvements

.github/workflows/ci.yml

@@ -4,7 +4,7 @@ name: CI
 on:
   pull_request:
     branches:
-      - main
+      - slalpha5
     types:
       - opened
 
@@ -22,4 +22,4 @@ jobs:
           JAVA_OPTS: -DBALLERINA_DEV_COMPILE_BALLERINA_ORG=true
           GRADLE_USER_HOME: ~/.gradle
         run: |
-          ./gradlew build -Prelease=false -Premote=true
+          ./gradlew build -Prelease=false -Premote=true -Ppublishing=false

.github/workflows/daily-build.yml

@@ -0,0 +1,32 @@
+name: Daily build
+
+# Controls when the action will run.
+on: 
+  schedule:
+    - cron: '30 2 * * *'
+
+jobs:
+  dailyBuild:
+    runs-on: ubuntu-latest
+    container:
+      image: ballerina/ballerina:swan-lake-beta1
+      options: --user root
+    steps:
+      - uses: actions/checkout@v1 
+      - name: Build with Gradle
+        env:
+          BALLERINA_CENTRAL_ACCESS_TOKEN: ${{ secrets.BALLERINA_CENTRAL_ACCESS_TOKEN }}
+          JAVA_OPTS: -DBALLERINA_DEV_COMPILE_BALLERINA_ORG=true
+          GRADLE_USER_HOME: ~/.gradle
+        run: |
+          ./gradlew build -Prelease=false -Premote=true -Ppublishing=false
+
+      - name: Publish Github Package
+        env:
+          packageUser: ${{ secrets.BALLERINA_BOT_USERNAME }}
+          packagePAT: ${{ secrets.BALLERINA_BOT_TOKEN }}
+          publishUser: ${{ secrets.BALLERINA_BOT_USERNAME }}
+          publishPAT: ${{ secrets.CONNECTOR_PUBLISH_PAT }}
+          JAVA_OPTS: -DBALLERINA_DEV_COMPILE_BALLERINA_ORG=true
+        run: |
+          ./gradlew publish -Prelease=false -Premote=true -Ppublishing=true   

.github/workflows/release.yml

@@ -21,7 +21,7 @@ jobs:
           BALLERINA_CENTRAL_ACCESS_TOKEN: ${{ secrets.BALLERINA_CENTRAL_ACCESS_TOKEN }}
           JAVA_OPTS: -DBALLERINA_DEV_COMPILE_BALLERINA_ORG=true
         run: |
-           ./gradlew build -Prelease=true -Premote=true
+           ./gradlew build -Prelease=true -Premote=true -Ppublishing=false
       - name: Create Pull Request For Updating Release Files
         id: cpr
         uses: peter-evans/create-pull-request@v3
@@ -32,7 +32,7 @@ jobs:
           author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
           signoff: false
           branch: release/file-update
-          base: main
+          base: slalpha5
           title: Update file hashes
           body: |
             Update file hashes

.gitignore

@@ -28,3 +28,5 @@ gen
 
 # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
 hs_err_pid*
+
+.project

build.gradle

@@ -18,16 +18,28 @@ apply plugin: 'jacoco'
 apply plugin: 'maven-publish'
 
 def ballerinaDistributionPath = System.getenv("BALLERINA_HOME")
+List<String> ballerinaPackages = new ArrayList<>();
 List<String> updatedBallerinaPackages = new ArrayList<>();
-boolean release = new Boolean(project.property("release").toString())
+boolean release = false;
 boolean remote = false;
+boolean isPublishing = false;
 if (project.hasProperty("remote")){
     remote = new Boolean(project.property("remote").toString())
 }
+if (project.hasProperty("release")){
+    release = new Boolean(project.property("release").toString())
+}
+if (project.hasProperty("publishing")){
+    isPublishing = new Boolean(project.property("publishing").toString())
+}
 
 Utils.loadOpenAPIProperties(project.projectDir.absolutePath)
 String openApiPackageDirPath = project.projectDir.absolutePath + "/openapi";
-updatedBallerinaPackages = Utils.findUpdatedBallerinaPackages(openApiPackageDirPath)
+updatedBallerinaPackages = Utils.findUpdatedBallerinaPackages(openApiPackageDirPath);
+ballerinaPackages = Utils.findBallerinaPackages(openApiPackageDirPath);
+if (isPublishing){
+    updatedBallerinaPackages = ballerinaPackages;
+}
 
 // TODO: Remove this cleanup task once the file override enabled
 // task cleanup {
@@ -107,8 +119,8 @@ updatedBallerinaPackages = Utils.findUpdatedBallerinaPackages(openApiPackageDirP
 
 // Temporarily added.
 task codeBuild {
-    // Pulling slalpha5 distribution if not available
     if (project.hasProperty("remote") ) {
+        // Pulling slalpha5 distribution if not available
         if (remote) {
             exec {
                 commandLine 'sh', '-c', "${ballerinaDistributionPath}/bin/bal dist pull slalpha5"
@@ -117,31 +129,46 @@ task codeBuild {
             exec {
                 commandLine 'sh', '-c', "rm -rf ${ballerinaDistributionPath}/bin/../dependencies/jdk-11.0.8+10-jre"
             }
+
+            // Changing the build distribution to SL Alpha5
+            exec {
+                commandLine 'sh', '-c', "${ballerinaDistributionPath}/bin/bal dist use slalpha5"
+            }
         }
     }
 
-    // Changing the build distribution to SL Alpha5
-    exec {
-        commandLine 'sh', '-c', "${ballerinaDistributionPath}/bin/bal dist use slalpha5"
-    }
     for (String path : updatedBallerinaPackages) {
-        println "Code building..."
+        print "Code building..."
+        Utils.removeSnapshotSuffixFromVersion(path);
         exec {
             commandLine 'sh', '-c', "${ballerinaDistributionPath}/bin/bal build -c ${path}"
         }
+        Utils.addSnapshotSuffixToVersion(path);
+    }
+
+    for (String path : updatedBallerinaPackages) {
+        if (!release) {
+            Utils.removeSnapshotSuffixFromVersion(path);
+            exec {
+                workingDir "${path}"
+                commandLine 'sh', '-c', "${ballerinaDistributionPath}/bin/bal push --repository=local"
+            }
+            Utils.addSnapshotSuffixToVersion(path);
+        }
     }
 }
 
 task releaseConnector {
-    //TODO: check and remove the snapshot part in the ballerina toml
     if (project.hasProperty("release") ) {
         if(release){
             for (String path : updatedBallerinaPackages) {
-                println "Pushing to Ballerina central..."
+                Utils.removeSnapshotSuffixFromVersion(path);
+                print "Pushing to Ballerina central..."
                 exec {
                     workingDir "${path}"
                     commandLine 'sh', '-c', "${ballerinaDistributionPath}/bin/bal push"
                 }
+                Utils.addSnapshotSuffixToVersion(path);
                 print "Updating Ballerina.toml version: "
                 Utils.bumpBallerinaTomlVersion(path);
                 println "Success"
@@ -152,3 +179,70 @@ task releaseConnector {
         }
     }
 }
+
+// Daily build configurations
+def packageOrg = "ballerinax"
+def platform = "java11"
+def artifactCacheParent = file("${project.projectDir}/build/cache_parent/")
+
+def packageName = "";
+def packageVersion = "";
+
+task copyDistribution {
+    if (isPublishing) {
+        for(String packagePath: ballerinaPackages) {
+            packageName = Utils.getPackageName(packagePath);
+            packageVersion = Utils.getPackageVersion(packagePath);
+            mkdir "${project.projectDir}/openapi/${packageName}/target/bala"
+            def pathToBala = file("${project.projectDir}/openapi/${packageName}/target/bala")
+
+            inputs.dir file(project.projectDir)
+            pathToBala.eachFileMatch(~/.*.bala/) { balaFile ->
+                copy {
+                    from zipTree(balaFile)
+                    into file("${artifactCacheParent}/bala/${packageOrg}/${packageName}/${packageVersion}/${platform}")
+                }
+            }
+            copy {
+                from file("${packagePath}/target/cache")
+                exclude '**/*-testable.jar'
+                exclude '**/tests_cache/'
+                into file("${artifactCacheParent}/cache/")
+            }
+            outputs.dir artifactCacheParent
+        }
+    }
+}
+
+task createArtifactZip(type: Zip) {
+    if (isPublishing){
+        println("Create Zip Artifact");
+        from "${buildDir}/cache_parent"
+        archiveName 'distribution.zip'
+        destinationDir(file("${buildDir}/distribution"))
+    }    
+}
+
+publishing {
+    if (isPublishing){
+        println("Package published");
+        publications {
+            mavenJava(MavenPublication) {
+                artifact source: createArtifactZip, extension: 'zip'
+            }
+        }
+
+        repositories {
+            maven {
+                name = "GitHubPackages"
+                url = uri("https://maven.pkg.github.com/ballerina-platform/ballerinax-openapi-connectors")
+                credentials {
+                    username = System.getenv("publishUser")
+                    password = System.getenv("publishPAT")
+                }
+            }
+        }
+    }
+}
+
+publish.dependsOn copyDistribution

openapi.properties

@@ -1,4 +0,0 @@
-#Mon Jun 28 08:34:34 GMT 2021
-openapi/openweathermap/openap.yaml=64ca19e5706be121d5617fd8af51fccd
-openapi/worldbank/openapi.yaml=ac5aceb2b7b6dc68316709619fc5a416
-openapi/covid19/openapi.yaml=0086d41e6b6a64ea76a3679587c1dcd8

pull_request_template.md

@@ -0,0 +1,52 @@
+## Purpose
+> Describe the problems, issues, or needs driving this feature/fix and include links to related issues in the following format: Resolves issue1, issue2, etc.
+
+## Goals
+> Describe the solutions that this feature/fix will introduce to resolve the problems described above
+
+## Approach
+> Describe how you are implementing the solutions. Include an animated GIF or screenshot if the change affects the UI (email [email protected] to review all UI text). Include a link to a Markdown file or Google doc if the feature write-up is too long to paste here.
+
+## User stories
+> Summary of user stories addressed by this change>
+
+## Release note
+> Brief description of the new feature or bug fix as it will appear in the release notes
+
+## Documentation
+> Link(s) to product documentation that addresses the changes of this PR. If no doc impact, enter “N/A” plus brief explanation of why there’s no doc impact
+
+## Training
+> Link to the PR for changes to the training content in https://github.com/wso2/WSO2-Training, if applicable
+
+## Certification
+> Type “Sent” when you have provided new/updated certification questions, plus four answers for each question (correct answer highlighted in bold), based on this change. Certification questions/answers should be sent to [email protected] and NOT pasted in this PR. If there is no impact on certification exams, type “N/A” and explain why.
+
+## Marketing
+> Link to drafts of marketing content that will describe and promote this feature, including product page changes, technical articles, blog posts, videos, etc., if applicable
+
+## Automation github4.tests
+ - Unit github4.tests 
+   > Code coverage information
+ - Integration github4.tests
+   > Details about the test cases and coverage
+
+## Security checks
+ - Followed secure coding standards in http://wso2.com/technical-reports/wso2-secure-engineering-guidelines? yes/no
+ - Ran FindSecurityBugs plugin and verified report? yes/no
+ - Confirmed that this PR doesn't commit any keys, passwords, tokens, usernames, or other secrets? yes/no
+
+## Samples
+> Provide high-level details about the samples related to this feature
+
+## Related PRs
+> List any other related PRs
+
+## Migrations (if applicable)
+> Describe migration steps and platforms on which migration has been tested
+
+## Test environment
+> List all JDK versions, operating systems, databases, and browser/versions on which this feature/fix was tested
+ 
+## Learning
+> Describe the research phase and any blog posts, patterns, libraries, or add-ons you used to solve the problem.