Java SDK release script (#406)

* Use back revision variable in pom.xml So user or CI system can easily override revision from external sources such as Git tag name * Add flatten maven plugin This plugin is useful during deployment so the final pom is resolved without parent dependency, i.e. we do not necessarily need to upload parent library * Increase versions for maven source,javadoc,spotless plugins So it has newer features and more fixes * Add gpg-plugin needed to sign releases * Use oss configure for flatten plugin, add developers info in pom.xml (required for releasing library * Add publish-java-sdk script * Add more logs to publish-java-sdk.sh * Add ProwJob publish-java-sdk * Use GPG_KEY_IMPORT_DIR variable * Update revision in pom.xml to 0.4.2-SNAPSHOT
feast-dev · Jan 7, 2020 · ec6b26b · ec6b26b
1 parent 3230bf5
commit ec6b26b
Show file tree

Hide file tree

Showing 8 changed files with 202 additions and 22 deletions.
diff --git a/.gitignore b/.gitignore
@@ -174,4 +174,8 @@ dmypy.json
 .pyre/
 .vscode
 
-sdk/python/docs/html
+# .flattened-pom.xml is generated by flatten-maven-plugin.
+# This pom should not be committed because it is only used during release / deployment.
+.flattened-pom.xml
+
+sdk/python/docs/html
diff --git a/.prow/config.yaml b/.prow/config.yaml
@@ -169,6 +169,34 @@ postsubmits:
     # https://github.com/semver/semver/issues/232
     - ^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$
 
+  - name: publish-java-sdk
+    decorate: true
+    spec:
+      containers:
+      - image: maven:3.6-jdk-8
+        command:
+        - bash
+        - -c
+        - .prow/scripts/publish-java-sdk.sh --revision ${PULL_BASE_REF:1}
+        volumeMounts:
+        - name: gpg-keys
+          mountPath: /etc/gpg
+          readOnly: true
+        - name: maven-settings
+          mountPath: /root/.m2/settings.xml
+          subPath: settings.xml
+          readOnly: true
+      volumes:
+      - name: gpg-keys
+        secret:
+          secretName: gpg-keys
+      - name: maven-settings
+        secret:
+          secretName: maven-settings
+    branches:
+    # Filter on tags with semantic versioning, prefixed with "v"
+    - ^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$
+
   - name: publish-docker-images
     decorate: true
     spec:
@@ -278,4 +306,5 @@ postsubmits:
         secret:
           secretName: feast-service-account
     branches:
+    # Filter on tags with semantic versioning, prefixed with "v"
     - ^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$
diff --git a/.prow/scripts/publish-java-sdk.sh b/.prow/scripts/publish-java-sdk.sh
@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+
+set -e 
+set -o pipefail
+
+GPG_KEY_IMPORT_DIR=/etc/gpg
+
+usage()
+{
+    echo "usage: publish-java-sdk.sh
+
+  --revision            Value for the revision e.g. '0.2.3'
+  --gpg-key-import-dir  Directory containing existing GPG keys to import.
+                        The directory should contain these 2 files:
+                        - public-key
+                        - private-key
+                        The default value is '/etc/gpg'
+  
+  This script assumes the GPG private key is protected by a passphrase.
+  The passphrase can be specified in \$HOME/.m2/settings.xml. In the same xml
+  file, credentials to upload releases to Sonatype must also be provided.
+
+  # Example settings: ~/.m2/settings.xml
+  <settings>
+    <servers>
+      <server>
+        <id>ossrh</id>
+        <username>SONATYPE_USER</username>
+        <password>SONATYPE_PASSWORD</password>
+      </server>
+    </servers>
+    <profiles>
+      <profile>
+        <id>ossrh</id>
+        <properties>
+          <gpg.passphrase>GPG_PASSPHRASE</gpg.passphrase>
+        </properties>
+      </profile>
+    </profiles>
+  </settings>
+"
+}
+
+while [ "$1" != "" ]; do
+  case "$1" in
+      --revision )             REVISION="$2";            shift;;
+      --gpg-key-import-dir )   GPG_KEY_IMPORT_DIR="$2";  shift;;
+      -h | --help )            usage;                    exit;; 
+      * )                      usage;                    exit 1
+  esac
+  shift
+done
+
+if [ -z $REVISION ]; then usage; exit 1; fi
+
+echo "============================================================"
+echo "Checking Maven and GPG versions"
+echo "============================================================"
+mvn --version 
+echo ""
+gpg --version
+
+echo "============================================================"
+echo "Importing GPG keys"
+echo "============================================================"
+gpg --import --batch --yes $GPG_KEY_IMPORT_DIR/public-key
+gpg --import --batch --yes $GPG_KEY_IMPORT_DIR/private-key
+
+echo "============================================================"
+echo "Deploying Java SDK with revision: $REVISION"
+echo "============================================================"
+mvn --projects sdk/java -Drevision=$REVISION --batch-mode clean deploy
diff --git a/core/pom.xml b/core/pom.xml
@@ -23,7 +23,7 @@
     <parent>
         <groupId>dev.feast</groupId>
         <artifactId>feast-parent</artifactId>
-        <version>0.3.6-SNAPSHOT</version>
+        <version>${revision}</version>
     </parent>
 
     <name>Feast Core</name>

diff --git a/ingestion/pom.xml b/ingestion/pom.xml
@@ -23,7 +23,7 @@
   <parent>
     <groupId>dev.feast</groupId>
     <artifactId>feast-parent</artifactId>
-    <version>0.3.6-SNAPSHOT</version>
+    <version>${revision}</version>
   </parent>
 
   <name>Feast Ingestion</name>

diff --git a/pom.xml b/pom.xml
@@ -24,7 +24,7 @@
 
     <groupId>dev.feast</groupId>
     <artifactId>feast-parent</artifactId>
-    <version>0.3.6-SNAPSHOT</version>
+    <version>${revision}</version>
     <packaging>pom</packaging>
 
     <modules>
@@ -35,6 +35,7 @@
     </modules>
 
     <properties>
+        <revision>0.4.2-SNAPSHOT</revision>
         <github.url>https://github.com/gojek/feast</github.url>
 
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@@ -47,7 +48,6 @@
         <org.apache.beam.version>2.16.0</org.apache.beam.version>
         <com.google.cloud.version>1.91.0</com.google.cloud.version>
         <io.prometheus.version>0.8.0</io.prometheus.version>
-
         <byte-buddy.version>1.9.10</byte-buddy.version>
         <hamcrest.version>1.3</hamcrest.version>
         <kafka.version>2.3.0</kafka.version>
@@ -58,9 +58,18 @@
 
     <organization>
         <name>Gojek</name>
-        <url>https://www.gojek.io/</url>
+        <url>https://www.gojek.com</url>
     </organization>
 
+    <developers>
+        <developer>
+            <name>Feast Authors</name>
+            <url>${github.url}</url>
+            <organization>Gojek</organization>
+            <organizationUrl>https://www.gojek.com</organizationUrl>
+        </developer>
+    </developers>
+
     <licenses>
         <license>
             <name>Apache License, Version 2.0</name>
@@ -81,11 +90,16 @@
         <url>${github.url}/issues</url>
     </issueManagement>
 
+    <!-- Release Java library on Sonatype https://central.sonatype.org/pages/apache-maven.html -->
     <distributionManagement>
         <snapshotRepository>
             <id>ossrh</id>
             <url>https://oss.sonatype.org/content/repositories/snapshots</url>
         </snapshotRepository>
+        <repository>
+            <id>ossrh</id>
+            <url>https://oss.sonatype.org/service/local/staging/deploy/maven2/</url>
+        </repository>
     </distributionManagement>
 
     <dependencyManagement>
@@ -274,21 +288,10 @@
         </extensions>
 
         <plugins>
-            <plugin>
-                <groupId>org.sonatype.plugins</groupId>
-                <artifactId>nexus-staging-maven-plugin</artifactId>
-                <version>1.6.7</version>
-                <extensions>true</extensions>
-                <configuration>
-                    <serverId>ossrh</serverId>
-                    <nexusUrl>https://oss.sonatype.org/</nexusUrl>
-                    <autoReleaseAfterClose>false</autoReleaseAfterClose>
-                </configuration>
-            </plugin>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-source-plugin</artifactId>
-                <version>2.2.1</version>
+                <version>3.2.1</version>
                 <executions>
                     <execution>
                         <id>attach-sources</id>
@@ -301,7 +304,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-javadoc-plugin</artifactId>
-                <version>2.9.1</version>
+                <version>3.1.1</version>
                 <executions>
                     <execution>
                         <id>attach-javadocs</id>
@@ -314,7 +317,7 @@
             <plugin>
                 <groupId>com.diffplug.spotless</groupId>
                 <artifactId>spotless-maven-plugin</artifactId>
-                <version>1.26.0</version>
+                <version>1.26.1</version>
                 <configuration>
                     <java>
                         <licenseHeader>
@@ -429,6 +432,78 @@
                     <skip>true</skip>
                 </configuration>
             </plugin>
+            <!-- nexus-staging-maven-plugin configures Maven to deploy to OSSRH Nexus Repository Manager -->
+            <plugin>
+                <groupId>org.sonatype.plugins</groupId>
+                <artifactId>nexus-staging-maven-plugin</artifactId>
+                <version>1.6.8</version>
+                <extensions>true</extensions>
+                <configuration>
+                    <serverId>ossrh</serverId>
+                    <nexusUrl>https://oss.sonatype.org/</nexusUrl>
+                    <!-- autoReleaseAfterClose is true as the release should be automated via continuous integration -->
+                    <autoReleaseAfterClose>true</autoReleaseAfterClose>
+                </configuration>
+            </plugin>
+            <!--
+            flatten-maven-plugin is used to flatten the generated POM during deployment. This is
+            required when releasing a project with a parent dependency where the version
+            is using ${revision} variable. This plugin will resolve and flatten the relationship.
+            https://www.mojohaus.org/flatten-maven-plugin/index.html
+            -->
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>flatten-maven-plugin</artifactId>
+                <version>1.1.0</version>
+                <configuration>
+                    <flattenMode>oss</flattenMode>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>flatten</id>
+                        <phase>process-resources</phase>
+                        <goals>
+                            <goal>flatten</goal>
+                        </goals>
+                    </execution>
+                    <execution>
+                        <id>flatten.clean</id>
+                        <phase>clean</phase>
+                        <goals>
+                            <goal>clean</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <!-- maven-gpg-plugin is used to sign Maven components. This is required when releasing libraries to Maven Central -->
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-gpg-plugin</artifactId>
+                <version>1.6</version>
+                <executions>
+                    <execution>
+                        <id>sign-artifacts</id>
+                        <phase>verify</phase>
+                        <goals>
+                            <goal>sign</goal>
+                        </goals>
+                        <!--
+                        This configuration helps with performing GPG operations non-interactively.
+                        For example when releasing via continuous integration.
+                        It is assumed that the GPG command used is version 2.x.
+                        -->
+                        <configuration>
+                            <gpgArguments>
+                                <arg>--pinentry-mode</arg>
+                                <arg>loopback</arg>
+                            </gpgArguments>
+                            <!-- Setting to allow retrieval of GPG passphrase from ~/.m2/settings.xml, refer to https://maven.apache.org/plugins/maven-gpg-plugin/usage.html -->
+                            <!--suppress MavenModelInspection -->
+                            <passphrase>${gpg.passphrase}</passphrase>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
         </plugins>
 
         <pluginManagement>

diff --git a/sdk/java/pom.xml b/sdk/java/pom.xml
@@ -11,7 +11,7 @@
   <parent>
     <groupId>dev.feast</groupId>
     <artifactId>feast-parent</artifactId>
-    <version>0.3.6-SNAPSHOT</version>
+    <version>${revision}</version>
     <relativePath>../..</relativePath>
   </parent>
 

diff --git a/serving/pom.xml b/serving/pom.xml
@@ -23,7 +23,7 @@
   <parent>
     <groupId>dev.feast</groupId>
     <artifactId>feast-parent</artifactId>
-    <version>0.3.6-SNAPSHOT</version>
+    <version>${revision}</version>
   </parent>
 
   <artifactId>feast-serving</artifactId>