build.gradle

/*
 * Copyright 2019 GoDataDriven B.V.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

buildscript {
    dependencies {
        classpath group: 'com.github.ben-manes', name: 'gradle-versions-plugin', version: '0.27.0'
    }
}

plugins {
    // Versions 1.6.10 thru 2.0.0 depend on a beta version of slf4j, which we don't resolve.
    // So for now we stay on this version of the plugin.
    id 'com.github.spotbugs' version '1.6.9'
    id 'com.github.johnrengelman.shadow' version '5.2.0'
    id 'pl.allegro.tech.build.axion-release' version '1.12.0'
}

apply plugin: 'groovy'
apply plugin: 'eclipse'
apply plugin: 'pmd'
apply plugin: 'checkstyle'
apply plugin: 'jacoco'
apply plugin: 'application'
apply plugin: 'com.github.ben-manes.versions'
apply plugin: 'pl.allegro.tech.build.axion-release'

scmVersion {
    tag {
        prefix = rootProject.name
    }
    versionIncrementer 'incrementMinorIfNotOnRelease', [releaseBranchPattern: 'support/.*']
}

defaultTasks 'clean', 'build'

group   = 'io.divolte'
// Order matters: the version has to be after the scmVersion extension has determined the version.
project.version = scmVersion.version

sourceCompatibility = 1.8
mainClassName = 'io.divolte.server.Server'
applicationName = rootProject.name
applicationDefaultJvmArgs = [
        "-XX:+UseG1GC",
        "-Djava.awt.headless=true"
]

repositories {
    mavenLocal()
    mavenCentral()
}

dependencies {
    // Define some key versions for components that we use lots of artifacts from.
    // Note: we refer to the Avro documentation in our own documentation.
    def avroVersion = '1.9.1'
    def hadoopVersion = '3.2.1'
    def jacksonVersion = '2.10.0'
    def slf4jVersion = '1.7.28'

    compile group: 'io.divolte', name: 'divolte-schema', version: version
    compile group: 'io.undertow', name: 'undertow-core', version: '2.1.0.Final'
    compile group: 'com.typesafe', name: 'config', version: '1.4.0'
    compile group: 'com.google.guava', name: 'guava', version: '29.0-jre'
    compile group: 'org.apache.avro', name: 'avro', version: avroVersion

    /*
     * We package the Avro Tools to provide an easy way to view Avro files
     * Because the default artifact (without the nodeps classifier) contains
     * some kind of uberjar with a old hadoop dependency, we manually include
     * the deps that we require for the tools to work and depend on the nodeps
     * variant of the tools itself. This is the minimal list to make the tool
     * run and have a working tojson command. The other commands weren't fully
     * tested with these deps.
     */
    runtime (group: 'org.apache.avro', name: 'avro-tools', version: avroVersion, classifier: 'nodeps') {
        exclude group: 'org.apache.avro', module: 'avro-mapred'
    }

    compile (group: 'org.apache.hadoop', name:'hadoop-common', version: hadoopVersion) {
        exclude group: 'jline', module: 'jline'
    }
    compile group: 'org.apache.hadoop', name: 'hadoop-hdfs', version: hadoopVersion
    compile (group: 'com.google.auth', name: 'google-auth-library-oauth2-http', version: '0.21.1') {
        exclude group: 'com.google.guava', module: 'guava-jdk5'
    }
    compile group: 'com.google.cloud', name: 'google-cloud-pubsub', version: '1.101.0'
    compile (group: 'net.sf.uadetector', name: 'uadetector-core', version: '0.9.22') {
        exclude group: 'com.google.code.findbugs', module: 'jsr305'
    }
    compile group: 'net.sf.uadetector', name: 'uadetector-resources', version: '2014.10'
    compile (group: 'com.maxmind.geoip2', name: 'geoip2', version:'2.12.0') {
        // We only use DB mode, and Google's HTTP client pulls in Apache HTTP
        // client which conflicts with the version via Hadoop.
        exclude group: 'com.google.http-client', module: 'google-http-client'
    }
    compile group: 'net.jodah', name: 'failsafe', version: '2.4.0'

    // Compatibility:
    //   - Horton and Cloudera are both on 0.10+ at the time of writing.
    //   - The client from 0.10.2.0 onwards supports brokers from 0.10.0.0 onwards.
    // Note: we refer to versioned producer documentation in our documentation.
    compile group: 'org.apache.kafka', name:'kafka-clients', version: '2.6.0'
    compile group: 'com.google.javascript', name:'closure-compiler-unshaded', version:'v20200920'
    compile group: 'org.codehaus.groovy', name:'groovy', version: '3.0.5', classifier: 'indy'
    compile group: 'net.sf.jopt-simple', name:'jopt-simple', version: '5.0.4'
    compile group: 'com.jayway.jsonpath', name: 'json-path', version: '2.4.0'
    compile group: 'com.fasterxml.jackson.core', name:'jackson-databind', version: jacksonVersion
    compile group: 'com.fasterxml.jackson.datatype', name:'jackson-datatype-jdk8', version: jacksonVersion
    compile group: 'com.fasterxml.jackson.datatype', name:'jackson-datatype-guava', version: jacksonVersion
    compile group: 'com.fasterxml.jackson.module', name:'jackson-module-parameter-names', version: jacksonVersion
    compile group: 'com.jasonclawson', name: 'jackson-dataformat-hocon', version: '1.1.0'

    // Used for configuration validation
    compile group: 'javax.validation', name:'validation-api', version: '2.0.1.Final'
    compile group: 'org.hibernate', name:'hibernate-validator', version: '6.1.5.Final'
    compile group: 'org.glassfish', name:'javax.el', version: '3.0.0'

    // We use the SLF4J API. At runtime, this is LogBack.
    // (We also force any dependencies that use Log4J to go via SLF4J.)
    compile group: 'org.slf4j', name: 'slf4j-api', version: slf4jVersion
    runtime group: 'ch.qos.logback', name: 'logback-classic', version: '1.2.3'
    runtime group: 'org.slf4j', name: 'log4j-over-slf4j', version: slf4jVersion
    runtime group: 'org.slf4j', name: 'jul-to-slf4j', version: slf4jVersion

    testCompile group: 'junit', name: 'junit', version: '4.13'
    testCompile group: 'org.hamcrest', name: 'hamcrest-all', version: '1.3'
    testCompile group: 'org.mockito', name: 'mockito-all', version: '1.10.19'
    testCompile group: 'com.saucelabs', name:'sauce_junit', version: '2.1.25'

    testCompile group: 'org.seleniumhq.selenium', name:'selenium-java', version: '3.141.59'
    testCompile group: 'com.saucelabs', name:'saucerest', version: '1.0.44'
    testCompile group: 'com.codeborne', name: 'phantomjsdriver', version: '1.4.4'
}

configurations {
    // Exclude all traces of Log4J via transitive dependencies.
    // (At runtime these are redirected over SLF4J.)
    all*.exclude group: 'org.slf4j', module: 'slf4j-log4j12'
    all*.exclude group: 'log4j', module: 'log4j'

    // For dependency update tracking, reject non-releases that are in release repositories.
    all {
        resolutionStrategy {
            final List<String> whitelist = [
                // These never left beta.
                "xml-apis:xml-apis",
            ]
            componentSelection {
                all { final ComponentSelection selection ->
                    final String selector = "${selection.candidate.group}:${selection.candidate.module}"
                    final boolean rejected = !(selector in whitelist) &&
                        ['alpha', 'beta', 'cr', 'pr', 'b'].any { qualifier ->
                            selection.candidate.version ==~ /(?i).*[.-]${qualifier}[.\d-]*/
                        }
                    if (rejected) {
                        selection.reject('Non-release version')
                    }
                }
            }
            force 'io.netty:netty-all:4.1.52.Final'
        }
    }
}

shadowJar {
    mergeServiceFiles()
}

// This applies to all JavaCompile tasks, not just :compileJava
// (which is the delegate for compileJava {}).
tasks.withType(JavaCompile) {
    // All our files are UTF-8 encoded.
    options.encoding = 'UTF-8'

    // Log details about usage of deprecated symbols.
    options.deprecation = true

    // Java 8 feature: parameter names via reflection, which makes
    // stuff like Jackson mapping easier.
    options.compilerArgs = [
        "-parameters",
        "-Xlint:unchecked"
    ]
}

spotbugs() {
    excludeFilter = file("config/findbugs/findbugs-excludes.xml")
    // Versions 3.1.9 thru 3.1.12 depend on a beta version of slf4j, which our component selection rules block.
    toolVersion = "3.1.8"
}

pmd {
    toolVersion = "5.8.1"
    ruleSetFiles = files("config/pmd/ruleset.xml")
    // The default set includes 'basic', so empty it. (We include basic via our custom ruleset.)
    ruleSets = []
}

checkstyle {
    toolVersion = "8.25"
}

jacoco {
    toolVersion = "0.8.5"
}

spotbugsMain.reports {
    xml.enabled = true
    html.enabled = false
}
spotbugsTest.reports {
    xml.enabled = true
    html.enabled = false
}

wrapper {
    gradleVersion = '5.6.3'
    distributionType = Wrapper.DistributionType.ALL
}

/*
 * This only applies to running with gradle run. We use a different run
 * script when creating a distribution.
 */
run {
    if (System.getenv().containsKey("HADOOP_CONF_DIR")) {
        classpath += files(System.getenv("HADOOP_CONF_DIR"))
    }
    if (System.getenv().containsKey("YARN_CONF_DIR")) {
        classpath += files(System.getenv("YARN_CONF_DIR"))
    }
}

// Project property for controlling how the server binds during tests.
if (!hasProperty("bindTestServerExternal")) {
    ext.bindTestServerExternal = false
}

test {
    maxHeapSize = "1g"

    /* Test results depend on the value of this project property. */
    systemProperty "io.divolte.test.bindExternal", bindTestServerExternal

    /*
     * Some tests do low-level manipulation with HttpUrlConnection. This
     * property gives them more freedom.
     */
    systemProperty "sun.net.http.allowRestrictedHeaders", "true"

    // This is a hack to make gradle realise that the tests depend on the
    // SELENIUM_DRIVER environment variable, and re-run tests if its value
    // changes.
    if (System.getenv().containsKey("SELENIUM_DRIVER")) {
        systemProperty "io.divolte.test.selenium.driver", System.getenv("SELENIUM_DRIVER")
    }
}

/*
 * Build distribution .tar.gz or .zip. We don't use the distribution
 * plugin as that allows for less flexibility in laying out the
 * archive. It also conflicts in non-trivial ways with the application
 * plugin (or the other way around).
 */
def distributionLayout = {
    def baseDir = archiveName - (".${extension}")

    into("${baseDir}/lib/") {
        from project.configurations.runtime
    }
    into("${baseDir}/lib/") {
        from(project.buildDir.path + "/libs") {
            include "**/*.jar"
        }
    }

    into("${baseDir}/conf/") {
        from("src/dist/conf") {
            include "**/*"
        }
        // Use the reference.conf as a example application.conf
        from("src/main/resources/reference.conf") {
            rename("reference.conf", "divolte-collector.conf.example")
        }
    }

    into("${baseDir}/bin/") {
        from("src/scripts") {
            include "**/*"
        }
    }
}

task tarball(dependsOn: build, type: Tar) {
    compression = Compression.GZIP
    extension = "tar.gz"

    configure distributionLayout
}

task zip(dependsOn: build, type: Zip) {
    configure distributionLayout
}


/*
 * Generate Sphinx documentation
 */
import org.apache.tools.ant.filters.ReplaceTokens
task processUserDoc {
    ext.source = file("docs")
    ext.sphinxDir = new File(buildDir, "sphinx")
    inputs.dir source
    outputs.dir sphinxDir

    doLast {

        // Copy/replace @version@
        copy {
            from(source) {
                filter(ReplaceTokens, tokens: ["version": version])
            }
            into sphinxDir
            exclude "**/*.png"
        }
        // Copy images (no replacement)
        copy {
            from(source)
            into sphinxDir
            include "**/*.png"
        }
    }
}

task userDoc(dependsOn: processUserDoc, type: Exec) {
    ext.sphinxDir = processUserDoc.sphinxDir
    ext.htmlDocDir = new File(buildDir, "userdoc/html")
    inputs.dir sphinxDir
    outputs.dir htmlDocDir

    workingDir sphinxDir

    executable "sphinx-build"
    args '-b', 'html', sphinxDir, htmlDocDir
}

def sourceFiles = copySpec {
    from('.') {
        exclude 'build'
        exclude 'docs'
        exclude 'rpm'
        exclude 'target'
        exclude '.idea'
        exclude '.gradle'
        exclude '**/.git*'
        exclude '.git*'
        exclude '**/*.ipr'
        exclude '**/*.iws'
        exclude '**/*.iml'
    }
}

task tarSource(type: Tar) {
    description = "Tars the source distribution."
    compression = Compression.GZIP
    extension = "tar.gz"

    into("divolte-collector-" + version) {
        with sourceFiles
    }
}

task copySourceTar(type: Copy) {
    dependsOn = [tarSource]
    from('build/distributions/') {
        include '*.tar.gz'
    }
    into 'rpm/SOURCES'
}

import java.text.SimpleDateFormat

static def buildTime() {
    final SimpleDateFormat df = new SimpleDateFormat("yyyyMMddHHmmss")
    df.setTimeZone(TimeZone.getTimeZone("GMT"))
    return df.format(new Date())
}

task buildRpm(type:Exec) {
    dependsOn = [copySourceTar]

    workingDir 'rpm'

    if (version.contains('SNAPSHOT')) {
        commandLine 'rpmbuild', '-ba',
            '--target', 'noarch-redhat-linux',
            '--define', 'snapshotVersion -SNAPSHOT',
            '--define', 'dist _' + buildTime(),
            'divolte-collector.spec'
    } else {
        commandLine 'rpmbuild', '-ba',
            '--target', 'noarch-redhat-linux',
            'divolte-collector.spec'
    }
}