build.gradle

/*
 * Licensed to the Apache Software Foundation (ASF) under one (1)
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License. (new fix)
 */

buildscript {
  repositories {
    mavenCentral()
  }

  apply from: file('gradle/buildscript.gradle'), to: buildscript
}

allprojects {
  repositories {
    mavenCentral()
    mavenLocal()
 }
}

apply from: file("gradle/dependency-versions.gradle")
apply from: file("gradle/dependency-versions-scala-" + scalaVersion + ".gradle")
apply from: file("gradle/release.gradle")
apply from: file("gradle/rat.gradle")
apply from: file('gradle/customize.gradle')

rat {
  excludes = [
    '*.patch',
    '**/*.eot',
    '**/*.graffle',
    '**/*.iml',
    '**/*.ipr',
    '**/*.iws',
    '**/*.json',
    '**/*.otf',
    '**/*.svg',
    '**/*.ttf',
    '**/*.woff',
    '**/.classpath',
    '**/.cache/**',
    '**/.git/**',
    '**/.gitignore',
    '**/.gradle/**',
    '**/.idea/**',
    '**/.project',
    '**/.settings/**',
    '**/.DS_Store/**',
    '**/bootstrap.css.map',
    '**/bootstrap.min.css',
    '**/bootstrap.min.js',
    '**/build/**',
    '**/font-awesome.min.css',
    '**/jquery-1.11.1.min.js',
    '**/jquery.tablesorter.min.js',
    '**/non-responsive.less',
    '**/ropa-sans.css',
    '**/syntax.css',
    '.reviewboardrc',
    'docs/_site/**',
    'docs/sitemap.xml',
    'docs/learn/documentation/*/api/javadocs/**',
    'docs/Gemfile.lock',
    'gradle/wrapper/**',
    'gradlew',
    'gradlew.bat',
    'samza-test/state/**',
    'README.md',
    'RELEASE.md',
    'samza-test/src/main/resources/**',
    'samza-hdfs/src/main/resources/**',
    'samza-hdfs/src/test/resources/**'
  ]
}

allprojects {
  apply plugin: 'idea'

  // For all scala compilation, add extra compiler options, taken from version-specific
  // dependency-versions-scala file applied above.
  tasks.withType(ScalaCompile) {
    scalaCompileOptions.additionalParameters = [ scalaOptions ]
  }
}

subprojects {
  apply plugin: 'eclipse'
  apply plugin: 'project-report'

  tasks.withType(Test) {
    test {
      testLogging {
        showStackTraces = true
        showExceptions = true
        showCauses = true
        displayGranularity = maxGranularity
        exceptionFormat = 'full'
      }
    }
  }
}

project(':samza-api') {
  apply plugin: 'java'
  apply plugin: 'checkstyle'

  dependencies {
    testCompile "junit:junit:$junitVersion"
    testCompile "org.mockito:mockito-all:$mockitoVersion"
  }

  checkstyle {
    configFile = new File(rootDir, "checkstyle/checkstyle.xml")
  }
}

project(":samza-core_$scalaVersion") {
  apply plugin: 'scala'
  apply plugin: 'checkstyle'

  // Force scala joint compilation
  sourceSets.main.scala.srcDir "src/main/java"
  sourceSets.test.scala.srcDir "src/test/java"
  sourceSets.main.java.srcDirs = []

  jar {
    manifest {
      attributes("Implementation-Version": "$version")
    }
  }

  dependencies {
    compile project(':samza-api')
    compile "org.scala-lang:scala-library:$scalaLibVersion"
    compile "org.slf4j:slf4j-api:$slf4jVersion"
    compile "net.sf.jopt-simple:jopt-simple:$joptSimpleVersion"
    compile "org.codehaus.jackson:jackson-mapper-asl:$jacksonVersion"
    compile "org.eclipse.jetty:jetty-webapp:$jettyVersion"
    testCompile "junit:junit:$junitVersion"
    testCompile "org.mockito:mockito-all:$mockitoVersion"
    testCompile "org.scalatest:scalatest_$scalaVersion:$scalaTestVersion"
  }

  checkstyle {
    configFile = new File(rootDir, "checkstyle/checkstyle.xml")
  }
}

project(":samza-autoscaling_$scalaVersion") {
  apply plugin: 'scala'
  apply plugin: 'checkstyle'


  dependencies {
    compile project(':samza-api')
    compile project(":samza-core_$scalaVersion")
    compile "org.scala-lang:scala-library:$scalaLibVersion"
    compile "org.slf4j:slf4j-api:$slf4jVersion"
    compile "net.sf.jopt-simple:jopt-simple:$joptSimpleVersion"
    compile "org.codehaus.jackson:jackson-mapper-asl:$jacksonVersion"
    compile "org.eclipse.jetty:jetty-webapp:$jettyVersion"
    compile("org.apache.hadoop:hadoop-yarn-client:$yarnVersion") {
      exclude module: 'slf4j-log4j12'
      exclude module: 'servlet-api'
    }
    compile("org.apache.hadoop:hadoop-common:$yarnVersion") {
      exclude module: 'slf4j-log4j12'
      exclude module: 'servlet-api'
    }
    compile "org.apache.httpcomponents:httpclient:$httpClientVersion"
    testCompile "junit:junit:$junitVersion"
    testCompile "org.mockito:mockito-all:$mockitoVersion"
    testCompile "org.scalatest:scalatest_$scalaVersion:$scalaTestVersion"
  }

  checkstyle {
    configFile = new File(rootDir, "checkstyle/checkstyle.xml")
  }
}

project(':samza-elasticsearch') {
  apply plugin: 'java'

  dependencies {
    compile project(':samza-api')
    compile project(":samza-core_$scalaVersion")
    compile "org.elasticsearch:elasticsearch:$elasticsearchVersion"
    compile "com.google.guava:guava:$guavaVersion"
    compile "org.slf4j:slf4j-api:$slf4jVersion"
    testCompile "junit:junit:$junitVersion"
    testCompile "org.mockito:mockito-all:$mockitoVersion"

    // Logging in tests is good.
    testRuntime "org.slf4j:slf4j-simple:$slf4jVersion"
  }
}

project(":samza-kafka_$scalaVersion") {
  apply plugin: 'scala'

  sourceSets.test.scala.srcDir "src/test/java"
  sourceSets.test.java.srcDirs = []

  configurations {
    // Remove transitive dependencies from Zookeeper that we don't want.
    compile.exclude group: 'javax.jms', module: 'jms'
    compile.exclude group: 'com.sun.jdmk', module: 'jmxtools'
    compile.exclude group: 'com.sun.jmx', module: 'jmxri'
  }

  dependencies {
    compile project(':samza-api')
    compile project(":samza-core_$scalaVersion")
    compile "org.scala-lang:scala-library:$scalaLibVersion"
    compile "com.101tec:zkclient:$zkClientVersion"
    compile "org.apache.zookeeper:zookeeper:$zookeeperVersion"
    compile "org.codehaus.jackson:jackson-mapper-asl:$jacksonVersion"
    compile "org.apache.kafka:kafka_$scalaVersion:$kafkaVersion"
    compile "org.apache.kafka:kafka-clients:$kafkaVersion"
    testCompile "org.apache.kafka:kafka_$scalaVersion:$kafkaVersion:test"
    testCompile "junit:junit:$junitVersion"
    testCompile "org.mockito:mockito-all:$mockitoVersion"
    testCompile "org.scalatest:scalatest_$scalaVersion:$scalaTestVersion"
    testCompile project(":samza-core_$scalaVersion").sourceSets.test.output

    // Logging in tests is good.
    testRuntime "org.slf4j:slf4j-simple:$slf4jVersion"
  }

  test {
    // Bump up the heap so we can start ZooKeeper and Kafka brokers.
    minHeapSize = "1560m"
    maxHeapSize = "1560m"
    jvmArgs = ["-XX:+UseConcMarkSweepGC", "-server"]
    // There appear to be issues between TestKafkaSystemAdmin and
    // TestKafkaCheckpointManager both running brokeres and ZK. Restarting the
    // gradle worker after every test clears things up. These tests should be
    // moved to the integration test suite.
    forkEvery = 1
  }
}

project(':samza-log4j') {
  apply plugin: 'java'
  apply plugin: 'checkstyle'

  dependencies {
    compile "log4j:log4j:$log4jVersion"
    compile project(':samza-api')
    compile project(":samza-core_$scalaVersion")
    compile "org.codehaus.jackson:jackson-mapper-asl:$jacksonVersion"
    testCompile "junit:junit:$junitVersion"
  }

  checkstyle {
    configFile = new File(rootDir, "checkstyle/checkstyle.xml")
  }
}

project(":samza-yarn_$scalaVersion") {
  apply plugin: 'scala'
  apply plugin: 'lesscss'

  // Force scala joint compilation
  sourceSets.main.scala.srcDir "src/main/java"
  sourceSets.main.java.srcDirs = []

  dependencies {
    compile project(':samza-api')
    compile project(":samza-core_$scalaVersion")
    compile "org.scala-lang:scala-library:$scalaLibVersion"
    compile "org.scala-lang:scala-compiler:$scalaLibVersion"
    compile "org.codehaus.jackson:jackson-mapper-asl:$jacksonVersion"
    compile "commons-httpclient:commons-httpclient:$commonsHttpClientVersion"
    compile("org.apache.hadoop:hadoop-yarn-api:$yarnVersion") {
      exclude module: 'slf4j-log4j12'
    }
    compile("org.apache.hadoop:hadoop-yarn-common:$yarnVersion") {
      exclude module: 'slf4j-log4j12'
      exclude module: 'servlet-api'
    }
    compile("org.apache.hadoop:hadoop-yarn-client:$yarnVersion") {
      exclude module: 'slf4j-log4j12'
      exclude module: 'servlet-api'
    }
    compile("org.apache.hadoop:hadoop-common:$yarnVersion") {
      exclude module: 'slf4j-log4j12'
      exclude module: 'servlet-api'
      // Exclude because YARN's 3.4.5 ZK version is incompatbile with Kafka's 3.3.4.
      exclude module: 'zookeeper'
    }
    compile("org.scalatra:scalatra_$scalaVersion:$scalatraVersion") {
      exclude module: 'scala-compiler'
      exclude module: 'slf4j-api'
    }
    compile("org.scalatra:scalatra-scalate_$scalaVersion:$scalatraVersion") {
      exclude module: 'scala-compiler'
      exclude module: 'slf4j-api'
    }
    compile "joda-time:joda-time:$jodaTimeVersion"
    compile "org.apache.zookeeper:zookeeper:$zookeeperVersion"
    testCompile "junit:junit:$junitVersion"
    testCompile "org.mockito:mockito-all:$mockitoVersion"
    testCompile project(":samza-core_$scalaVersion").sourceSets.test.output
  }

  repositories {
    maven {
      url "http://repo.typesafe.com/typesafe/releases"
    }
  }

  lesscss {
    source = fileTree('src/main/less') {
      include 'main.less'
    }
    dest = "$buildDir/resources/main/scalate/css"
  }

  jar.dependsOn("lesscss")
}

project(":samza-shell") {
  apply plugin: 'java'

  configurations {
    gradleShell
  }

  dependencies {
    gradleShell project(":samza-core_$scalaVersion")
    gradleShell project(":samza-kafka_$scalaVersion")
    gradleShell project(":samza-test_$scalaVersion")
    gradleShell project(":samza-yarn_$scalaVersion")
    gradleShell "org.slf4j:slf4j-log4j12:$slf4jVersion"
    gradleShell "log4j:log4j:1.2.16"
  }

  task shellTarGz(type: Tar) {
    compression = Compression.GZIP
    classifier = 'dist'
    from 'src/main/bash'
    from 'src/main/resources'
  }

  artifacts {
    archives(shellTarGz) {
      name 'samza-shell'
      classifier 'dist'
    }
  }

  // Usage: ./gradlew samza-shell:runJob \
  //    -PconfigPath=file:///path/to/job/config.properties
  task runJob(type:JavaExec) {
    description 'To run a job (defined in a properties file)'
    main = 'org.apache.samza.job.JobRunner'
    classpath = configurations.gradleShell
    if (project.hasProperty('configPath')) args += ['--config-path', configPath]
    jvmArgs = ["-Dlog4j.configuration=file:src/main/resources/log4j-console.xml"]
  }

  // Usage: ./gradlew samza-shell:checkpointTool \
  //    -PconfigPath=file:///path/to/job/config.properties -PnewOffsets=file:///path/to/new/offsets.properties
  task checkpointTool(type:JavaExec) {
    description 'Command-line tool to inspect and manipulate the job’s checkpoint'
    main = 'org.apache.samza.checkpoint.CheckpointTool'
    classpath = configurations.gradleShell
    if (project.hasProperty('configPath')) args += ['--config-path', configPath]
    if (project.hasProperty('newOffsets')) args += ['--new-offsets', newOffsets]
    jvmArgs = ["-Dlog4j.configuration=file:src/main/resources/log4j-console.xml"]
  }

  // Usage: ./gradlew samza-shell:kvPerformanceTest
  //    -PconfigPath=file:///path/to/job/config.properties
  task kvPerformanceTest(type:JavaExec) {
    description 'Command-line tool to run key-value performance tests'
    main = 'org.apache.samza.test.performance.TestKeyValuePerformance'
    classpath = configurations.gradleShell
    if (project.hasProperty('configPath')) args += ['--config-path', configPath]
    jvmArgs = ["-Dlog4j.configuration=file:src/main/resources/log4j-console.xml"]
  }
}

project(":samza-kv_$scalaVersion") {
  apply plugin: 'scala'
  apply plugin: 'checkstyle'

  dependencies {
    compile project(':samza-api')
    compile project(":samza-core_$scalaVersion")
    compile "org.scala-lang:scala-library:$scalaLibVersion"
    testCompile "junit:junit:$junitVersion"
    testCompile "org.mockito:mockito-all:$mockitoVersion"
  }

  checkstyle {
    configFile = new File(rootDir, "checkstyle/checkstyle.xml")
  }
}

project(":samza-kv-inmemory_$scalaVersion") {
  apply plugin: 'scala'

  dependencies {
    compile project(':samza-api')
    compile project(":samza-core_$scalaVersion")
    compile project(":samza-kv_$scalaVersion")
    compile "org.scala-lang:scala-library:$scalaLibVersion"
    compile "com.google.guava:guava:$guavaVersion"
    testCompile "junit:junit:$junitVersion"
  }
}

project(":samza-kv-rocksdb_$scalaVersion") {
  apply plugin: 'scala'

  sourceSets.test.scala.srcDir "src/test/java"
  sourceSets.test.java.srcDirs = []

  dependencies {
    compile project(':samza-api')
    compile project(":samza-core_$scalaVersion")
    compile project(":samza-kv_$scalaVersion")
    compile "org.scala-lang:scala-library:$scalaLibVersion"
    compile "org.rocksdb:rocksdbjni:$rocksdbVersion"
    testCompile "junit:junit:$junitVersion"
    testCompile "org.scalatest:scalatest_$scalaVersion:$scalaTestVersion"
  }
}

project(":samza-hdfs_$scalaVersion") {
    apply plugin: 'scala'

    dependencies {
        compile project(':samza-api')
        compile project(":samza-core_$scalaVersion")
        compile project(":samza-kafka_$scalaVersion")
        compile "org.scala-lang:scala-library:$scalaLibVersion"
        compile("org.apache.hadoop:hadoop-hdfs:$yarnVersion") {
          exclude module: 'slf4j-log4j12'
          exclude module: 'servlet-api'
        }
        compile("org.apache.hadoop:hadoop-common:$yarnVersion") {
          exclude module: 'slf4j-log4j12'
          exclude module: 'servlet-api'
          exclude module: 'zookeeper'
        }

        testCompile "junit:junit:$junitVersion"
        testCompile "org.scalatest:scalatest_$scalaVersion:$scalaTestVersion"
        testCompile "org.apache.hadoop:hadoop-minicluster:$yarnVersion"
    }
}

project(":samza-test_$scalaVersion") {
  apply plugin: 'scala'
  apply plugin: 'checkstyle'

  configurations {
    // Remove transitive dependencies from Zookeeper that we don't want.
    compile.exclude group: 'javax.jms', module: 'jms'
    compile.exclude group: 'com.sun.jdmk', module: 'jmxtools'
    compile.exclude group: 'com.sun.jmx', module: 'jmxri'
  }

  dependencies {
    compile project(':samza-api')
    compile project(":samza-kv-inmemory_$scalaVersion")
    compile project(":samza-kv-rocksdb_$scalaVersion")
    compile project(":samza-core_$scalaVersion")
    runtime project(":samza-log4j")
    runtime project(":samza-yarn_$scalaVersion")
    runtime project(":samza-kafka_$scalaVersion")
    runtime project(":samza-hdfs_$scalaVersion")
    compile "org.scala-lang:scala-library:$scalaLibVersion"
    compile "net.sf.jopt-simple:jopt-simple:$joptSimpleVersion"
    compile "javax.mail:mail:1.4"
    compile "junit:junit:$junitVersion"
    compile "org.apache.kafka:kafka_$scalaVersion:$kafkaVersion"
    testCompile "org.apache.kafka:kafka_$scalaVersion:$kafkaVersion:test"
    testCompile "com.101tec:zkclient:$zkClientVersion"
    testCompile project(":samza-kafka_$scalaVersion")
    testCompile project(":samza-core_$scalaVersion").sourceSets.test.output
    testCompile "org.scalatest:scalatest_$scalaVersion:$scalaTestVersion"
    testRuntime "org.slf4j:slf4j-simple:$slf4jVersion"
  }

  test {
    // Bump up the heap so we can run KV store tests.
    minHeapSize = "1560m"
    maxHeapSize = "1560m"
    jvmArgs = ["-XX:+UseConcMarkSweepGC", "-server"]
  }

  checkstyle {
    configFile = new File(rootDir, "checkstyle/checkstyle.xml")
  }

  tasks.create(name: "releaseTestJobs", dependsOn: configurations.archives.artifacts, type: Tar) {
    description 'Build an integration test tarball'
    compression = Compression.GZIP
    from(file("$projectDir/src/main/config")) { into "config/" }
    from(file("$projectDir/src/main/resources")) { into "lib/" }
    from(project(':samza-shell').file("src/main/bash")) { into "bin/" }
    from(file("$projectDir/src/main/python/ghostface_killah.py")) { into "bin/"}
    from(project(':samza-shell').file("src/main/resources")) { into "lib/" }
    from(project(':samza-shell').file("src/main/resources/log4j-console.xml")) { into "bin/" }
    from '../LICENSE'
    from '../NOTICE'
    from(configurations.runtime) { into("lib/") }
    from(configurations.archives.artifacts.files) { into("lib/") }
  }
}