From f1150a3d7a26415d904ea950ce2a52e9bef2d82b Mon Sep 17 00:00:00 2001 From: Joo-Won Jung Date: Mon, 5 Nov 2018 02:26:38 +0900 Subject: [PATCH 1/4] chore: add infos to publish to oss.sonatype.org --- build.sbt | 45 +++++++++++++++++++++++++++++++++++++++++++-- project/plugins.sbt | 1 + 2 files changed, 44 insertions(+), 2 deletions(-) create mode 100644 project/plugins.sbt diff --git a/build.sbt b/build.sbt index c87c843..3002c82 100644 --- a/build.sbt +++ b/build.sbt @@ -1,5 +1,36 @@ ThisBuild / scalaVersion := "2.11.12" -ThisBuild / organization := "net.sanori" +ThisBuild / organization := "net.sanori.spark" +ThisBuild / organizationName := "SanoriNet" +ThisBuild / organizationHomepage := Some(url("http://sanori.github.io/")) + +ThisBuild / scmInfo := Some( + ScmInfo( + url("https://github.com/sanori/spark-access-log"), + "scm:git@github.com:sanori/spark-access-log.git" + ) +) +ThisBuild / developers := List( + Developer( + id = "sanori", + name = "Joo-Won Jung", + email = "sanori@sanori.net", + url = url("https://sanori.github.io/") + ) +) +ThisBuild / licenses := Seq( + "Apache 2" -> new URL("http://www.apache.org/licenses/LICENSE-2.0.txt") +) +ThisBuild / homepage := Some(url("https://github.com/sanori/spark-access-log")) + +ThisBuild / pomIncludeRepository := { _ => false } +ThisBuild / publishTo := { + val nexus = "https://oss.sonatype.org/" + if (isSnapshot.value) + Some("snapshots" at nexus + "content/repositories/snapshots") + else + Some("releases" at nexus + "service/local/staging/deploy/maven2") +} +ThisBuild / publishMavenStyle := true lazy val root = (project in file(".")) .settings( @@ -12,4 +43,14 @@ lazy val root = (project in file(".")) // fork in Test := true // javaOptions ++= Seq("-Xms512M", "-Xmx2048M", "-XX:MaxPermSize=2048M", "-XX:+CMSClassUnloadingEnabled") -// parallelExecution in Test := false \ No newline at end of file +// parallelExecution in Test := false + +credentials += Credentials(Path.userHome / ".sbt" / "sonatype_credential") + +// To specify signing key +credentials += Credentials( + "GnuPG Key ID", + "gpg", + "615633F4F7E94C938A2918DF10CB79CA9DDE74FD", + "ignored" +) diff --git a/project/plugins.sbt b/project/plugins.sbt new file mode 100644 index 0000000..b9c4bb7 --- /dev/null +++ b/project/plugins.sbt @@ -0,0 +1 @@ +addSbtPlugin("io.crashbox" % "sbt-gpg" % "0.2.0") From e6c42f6c4068f22b9edb8ede9a79d123ed419ddb Mon Sep 17 00:00:00 2001 From: Joo-Won Jung Date: Mon, 5 Nov 2018 03:07:04 +0900 Subject: [PATCH 2/4] chore: change package name & version --- README.md | 4 ++-- build.sbt | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 298a7e8..26d4b72 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ log formats are supported. ### SQL (spark-sql) When start spark-sql, include jar file of this project. ```sh -spark-sql --jars accesslog_2.11-0.1.0-SNAPSHOT.jar +spark-sql --jars access-log_2.11-0.1.0.jar ``` In SQL, you can create user defined function and use it: @@ -76,7 +76,7 @@ which has the following meaning: ``` sbt clean package ``` -generates `accesslog_2.11-0.1.0-SNAPSHOT.jar` in `target/scala-2.11`. +generates `access-log_2.11-0.1.0.jar` in `target/scala-2.11`. ### Motivation * To simplify analysis of web server logs diff --git a/build.sbt b/build.sbt index 3002c82..5307608 100644 --- a/build.sbt +++ b/build.sbt @@ -34,7 +34,8 @@ ThisBuild / publishMavenStyle := true lazy val root = (project in file(".")) .settings( - name := "accessLog", + name := "access-log", + version := "0.1.0", libraryDependencies += "org.apache.spark" %% "spark-core" % "2.3.2", libraryDependencies += "org.apache.spark" %% "spark-sql" % "2.3.2", libraryDependencies += "org.apache.spark" %% "spark-hive" % "2.3.2", From 9d53764938684fc8ecaf137a133364052503e48c Mon Sep 17 00:00:00 2001 From: Joo-Won Jung Date: Tue, 6 Nov 2018 00:55:38 +0900 Subject: [PATCH 3/4] fix: change sbt-gpg to sbt-pgp --- build.sbt | 10 ++++------ project/plugins.sbt | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/build.sbt b/build.sbt index 5307608..6830d5a 100644 --- a/build.sbt +++ b/build.sbt @@ -48,10 +48,8 @@ lazy val root = (project in file(".")) credentials += Credentials(Path.userHome / ".sbt" / "sonatype_credential") +useGpg := true +// workaround for sbt/sbt-pgp#126 +pgpSecretRing := pgpPublicRing.value // To specify signing key -credentials += Credentials( - "GnuPG Key ID", - "gpg", - "615633F4F7E94C938A2918DF10CB79CA9DDE74FD", - "ignored" -) +usePgpKeyHex("10CB79CA9DDE74FD") diff --git a/project/plugins.sbt b/project/plugins.sbt index b9c4bb7..a894c71 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1 +1 @@ -addSbtPlugin("io.crashbox" % "sbt-gpg" % "0.2.0") +addSbtPlugin("com.jsuereth" % "sbt-pgp" % "2.0.0-M2") From a4219ea34db508ffbb4bdf046c05dc932a814861 Mon Sep 17 00:00:00 2001 From: Joo-Won Jung Date: Tue, 6 Nov 2018 03:17:30 +0900 Subject: [PATCH 4/4] doc: change README.md to use precompiled package --- README.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 26d4b72..67e0aaa 100644 --- a/README.md +++ b/README.md @@ -9,9 +9,9 @@ log formats are supported. ## How to use ### SQL (spark-sql) -When start spark-sql, include jar file of this project. +When start spark-sql: ```sh -spark-sql --jars access-log_2.11-0.1.0.jar +spark-sql --packages net.sanori.spark:access-log_2.11:0.1.0 ``` In SQL, you can create user defined function and use it: @@ -46,6 +46,16 @@ val logDs = lineDs .as[CombinedLog] ``` +When start spark-shell: +```sh +spark-shell --packages net.sanori.spark:access-log_2.11:0.1.0 +``` + +In build.sbt: +```sbtshell +libraryDependencies += "net.sanori.spark" %% "access-log" % "0.1.0" +``` + ### RDD in Scala ```scala import net.sanori.spark.accessLog.toCombinedLog