METRON-2265: Update Kerberos settings

metron-deployment/Kerberos-manual-setup.md

@@ -70,7 +70,6 @@ Setup
 1. Create the `metron` user's home directory in HDFS.
 
     ```
-    sudo -u hdfs hdfs dfs -mkdir /user/metron
     sudo -u hdfs hdfs dfs -chown metron:hdfs /user/metron
     sudo -u hdfs hdfs dfs -chmod 770 /user/metron
     ```
@@ -84,6 +83,38 @@ Setup a KDC
     yum -y install krb5-server krb5-libs krb5-workstation
     ```
 
+1. Uncomment all commented lines in `/etc/krb5.conf`.  The contents of `/etc/krb5.conf` should look like:
+
+   ```
+   # Configuration snippets may be placed in this directory as well
+   includedir /etc/krb5.conf.d/
+
+   [logging]
+    default = FILE:/var/log/krb5libs.log
+    kdc = FILE:/var/log/krb5kdc.log
+    admin_server = FILE:/var/log/kadmind.log
+
+   [libdefaults]
+    dns_lookup_realm = false
+    ticket_lifetime = 24h
+    renew_lifetime = 7d
+    forwardable = true
+    rdns = false
+    pkinit_anchors = /etc/pki/tls/certs/ca-bundle.crt
+    default_realm = EXAMPLE.COM
+    default_ccache_name = KEYRING:persistent:%{uid}
+
+   [realms]
+    EXAMPLE.COM = {
+     kdc = node1
+     admin_server = node1
+    }
+
+   [domain_realm]
+    .example.com = EXAMPLE.COM
+    example.com = EXAMPLE.COM
+   ```
+
 1. Define the current host as the KDC.
 
     ```
@@ -92,7 +123,7 @@ Setup a KDC
     cp -f /etc/krb5.conf /var/lib/ambari-server/resources/scripts
     ```
 
-1. Ensure that the KDC can issue renewable tickets. This may be necessary on a real cluster, but should not be on a [single VM](development/centos6/README.md).
+1. Ensure that the KDC can issue renewable tickets.
 
     Edit `/var/kerberos/krb5kdc/kdc.conf` and ensure the following is added to the `realm` section
 
@@ -109,14 +140,14 @@ Setup a KDC
 1. Start the KDC and ensure that it starts on boot.
 
     ```
-    /etc/rc.d/init.d/krb5kdc start
+    krb5kdc
     chkconfig krb5kdc on    
     ```
 
 1. Start the Kerberos Admin service and ensure that it starts on boot.    
 
     ```
-    /etc/rc.d/init.d/kadmin start
+    kadmind
     chkconfig kadmin on
     ```
 
@@ -203,6 +234,7 @@ Enable Kerberos
 
     ```
     kadmin.local -q "ktadd -k metron.headless.keytab [email protected]"
+    mkdir /etc/security/keytabs
     cp metron.headless.keytab /etc/security/keytabs
     chown metron:hadoop /etc/security/keytabs/metron.headless.keytab
     chmod 440 /etc/security/keytabs/metron.headless.keytab
@@ -417,9 +449,12 @@ Push Data
 1. Push some sample data to one of the parser topics. E.g for Bro we took raw data from [metron/metron-platform/metron-integration-test/src/main/sample/data/bro/raw/BroExampleOutput](../metron-platform/metron-integration-test/src/main/sample/data/bro/raw/BroExampleOutput)
 
     ```
-    cat sample-bro.txt | ${KAFKA_HOME}/kafka-broker/bin/kafka-console-producer.sh \
+    source /etc/default/metron
+    kinit -kt $METRON_SERVICE_KEYTAB $METRON_PRINCIPAL_NAME
+    export KAFKA_OPTS=$CLIENT_JAAS_ARG
+    cat sample-bro.txt | ${KAFKA_HOME}/bin/kafka-console-producer.sh \
             --broker-list ${BROKERLIST} \
-            --security-protocol SASL_PLAINTEXT \
+            --producer-property security.protocol=$KAFKA_SECURITY_PROTOCOL \
             --topic bro
     ```
 
@@ -494,26 +529,18 @@ More notes can be found in [metron/metron-sensors/README.md](../metron-sensors/R
 ```
 cat sample-yaf.txt | ${KAFKA_HOME}/bin/kafka-console-producer.sh \
         --broker-list ${BROKERLIST} \
-        --security-protocol PLAINTEXTSASL \
+        --producer-property security.protocol=$KAFKA_SECURITY_PROTOCOL \
         --topic yaf
 ```
 
 #### View topic data from latest offset with SASL
 
 ```
 ${KAFKA_HOME}/bin/kafka-console-consumer.sh \
-        --zookeeper ${ZOOKEEPER} \
-        --security-protocol PLAINTEXTSASL \
-        --topic yaf
-```
-
-#### Modify the sensor-stubs to send logs via SASL
-```
-sed -i 's/node1:6667 --topic/node1:6667 --security-protocol PLAINTEXTSASL --topic/' /opt/sensor-stubs/bin/start-*-stub
-for sensorstub in bro snort; do
-    service sensor-stubs stop ${sensorstub};
-    service sensor-stubs start ${sensorstub};
-done
+        --bootstrap-server ${BROKERLIST} \
+        --consumer-property security.protocol=$KAFKA_SECURITY_PROTOCOL \
+        --topic yaf \
+        --consumer-property group.id=<authorized group>
 ```
 
 #### Model as a Service on Kerberos

metron-deployment/ansible/roles/sensor-stubs/templates/start-bro-stub

@@ -55,6 +55,7 @@ KAFKA_SECURITY_PROTOCOL=${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT}
 if [ ${SECURITY_ENABLED,,} == 'true' ]; then
   echo "Security enabled"
   kinit -kt $METRON_SERVICE_KEYTAB $METRON_PRINCIPAL_NAME
+  export KAFKA_OPTS=$CLIENT_JAAS_ARG
 fi
 
 while true; do

metron-deployment/ansible/roles/sensor-stubs/templates/start-snort-stub

@@ -55,6 +55,7 @@ KAFKA_SECURITY_PROTOCOL=${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT}
 if [ ${SECURITY_ENABLED,,} == 'true' ]; then
   echo "Security enabled"
   kinit -kt $METRON_SERVICE_KEYTAB $METRON_PRINCIPAL_NAME
+  export KAFKA_OPTS=$CLIENT_JAAS_ARG
 fi
 
 while true; do

metron-deployment/ansible/roles/sensor-stubs/templates/start-yaf-stub

@@ -55,6 +55,7 @@ KAFKA_SECURITY_PROTOCOL=${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT}
 if [ ${SECURITY_ENABLED,,} == 'true' ]; then
   echo "Security enabled"
   kinit -kt $METRON_SERVICE_KEYTAB $METRON_PRINCIPAL_NAME
+  export KAFKA_OPTS=$CLIENT_JAAS_ARG
 fi
 
 while true; do

metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py

@@ -263,7 +263,7 @@
 topology_auto_credentials_double_quotes = str(topology_auto_credentials).replace("'", '"')
 
 if security_enabled:
-    hostname_lowercase = config['hostname'].lower()
+    hostname_lowercase = hostname.lower()
     metron_principal_name = status_params.metron_principal_name
     metron_keytab_path = status_params.metron_keytab_path
     kinit_path_local = status_params.kinit_path_local

metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2

@@ -48,16 +48,19 @@ METRON_LDAP_SSL_TRUSTSTORE="{{metron_ldap_ssl_truststore}}"
 METRON_USER_ROLE="{{metron_user_role}}"
 METRON_ADMIN_ROLE="{{metron_admin_role}}"
 
-ZOOKEEPER="{{zookeeper_quorum}}"
-BROKERLIST="{{kafka_brokers}}"
-HADOOP_CONF_DIR="/etc/hadoop/conf/"
-STORM_REST_URL="{{storm_rest_addr}}"
+#Security
 SECURITY_ENABLED={{security_enabled|lower}}
 {% if metron_principal_name is defined %}METRON_PRINCIPAL_NAME="{{metron_principal_name}}"
 {% endif %}
 {% if metron_keytab_path is defined %}METRON_SERVICE_KEYTAB="{{metron_keytab_path}}"
 {% endif %}
 KAFKA_SECURITY_PROTOCOL="{{kafka_security_protocol}}"
+CLIENT_JAAS_ARG="{{client_jaas_arg}}"
+
+ZOOKEEPER="{{zookeeper_quorum}}"
+BROKERLIST="{{kafka_brokers}}"
+HADOOP_CONF_DIR="/etc/hadoop/conf/"
+STORM_REST_URL="{{storm_rest_addr}}"
 PARSER_TOPOLOGY_OPTIONS="/home/{{metron_user}}/.storm/storm.config"
 METRON_ESCALATION_TOPIC="{{metron_escalation_topic}}"
 PCAP_BASE_PATH="{{pcap_base_path}}"
@@ -68,4 +71,3 @@ PCAP_YARN_QUEUE="{{pcap_yarn_queue}}"
 PCAP_FINALIZER_THREADPOOL_SIZE="{{pcap_finalizer_threadpool_size}}"
 STORM_STATUS_CACHE_MAX_SIZE="{{storm_status_cache_max_size}}"
 STORM_STATUS_CACHE_TIMEOUT_SECONDS="{{storm_status_cache_timeout_seconds}}"
-

metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/KafkaConfig.java

@@ -99,7 +99,7 @@ public Map<String, Object> producerProperties() {
     producerConfig.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer");
     producerConfig.put(ProducerConfig.ACKS_CONFIG, "1");
     if (environment.getProperty(MetronRestConstants.KERBEROS_ENABLED_SPRING_PROPERTY, Boolean.class, false)) {
-      producerConfig.put("security.protocol", SecurityProtocol.SASL_PLAINTEXT);
+      producerConfig.put("security.protocol", KafkaUtils.INSTANCE.normalizeProtocol(environment.getProperty(MetronRestConstants.KAFKA_SECURITY_PROTOCOL_SPRING_PROPERTY)));
     }
     return producerConfig;
   }

metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/KafkaConfigTest.java

@@ -67,7 +67,7 @@ public void kafkaConfigShouldProperlyReturnProducerProperties() throws Exception
     when(environment.getProperty(MetronRestConstants.KERBEROS_ENABLED_SPRING_PROPERTY, Boolean.class, false)).thenReturn(true);
     when(environment.getProperty(MetronRestConstants.KAFKA_SECURITY_PROTOCOL_SPRING_PROPERTY)).thenReturn("kafka security protocol");
 
-    producerProperties = kafkaConfig.consumerProperties();
+    producerProperties = kafkaConfig.producerProperties();
     assertEquals("kafka security protocol", producerProperties.get("security.protocol"));
   }
 

metron-platform/elasticsearch-shaded/pom.xml

@@ -135,15 +135,19 @@
                             </filter>
                           </filters>
                           <relocations>
-                                <!-- The REST API has conflicts with these packages -->
-                                <relocation>
-                                    <pattern>io.netty</pattern>
-                                    <shadedPattern>org.apache.metron.io.netty</shadedPattern>
-                                </relocation>
-                                <relocation>
-                                    <pattern>com.google.common</pattern>
-                                    <shadedPattern>org.apache.metron.guava.${guava_version}</shadedPattern>
-                                </relocation>
+                             <!-- The REST API has conflicts with these packages -->
+                              <relocation>
+                                  <pattern>io.netty</pattern>
+                                  <shadedPattern>org.apache.metron.io.netty</shadedPattern>
+                              </relocation>
+                              <relocation>
+                                  <pattern>com.google.common</pattern>
+                                  <shadedPattern>org.apache.metron.guava.${guava_version}</shadedPattern>
+                              </relocation>
+                              <relocation>
+                                  <pattern>org.apache.http</pattern>
+                                  <shadedPattern>org.apache.metron.http.elasticsearch</shadedPattern>
+                              </relocation>
                             </relocations>
                             <artifactSet>
                                 <excludes>

metron-platform/metron-common/src/main/scripts/stellar

@@ -36,6 +36,15 @@ fi
 # treat unset vars as an error; METRON_HOME
 set -u
 
+SECURITY_ENABLED=${SECURITY_ENABLED:-false}
+KAFKA_SECURITY_PROTOCOL=${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT}
+
+if [ ${SECURITY_ENABLED,,} == 'true' ]; then
+  echo "Security enabled"
+  kinit -kt $METRON_SERVICE_KEYTAB $METRON_PRINCIPAL_NAME
+  export KAFKA_OPTS=$CLIENT_JAAS_ARG
+fi
+
 export METRON_PARSERS_PLATFORM="${METRON_PARSERS_PLATFORM:-storm}"
 export HBASE_CONFIGS=$(hbase classpath)
 export STELLAR_LIB=$(find $METRON_HOME/lib/ -name stellar-common*.jar)

metron-platform/metron-data-management/pom.xml

@@ -416,6 +416,13 @@
                                     <pattern>com.google.common</pattern>
                                     <shadedPattern>org.apache.metron.guava.${guava_version}</shadedPattern>
                                 </relocation>
+                                <relocation>
+                                    <!-- need to also relocate Guava's `thirdparty` packages. otherwise, the class
+                                         `thirdparty.publicsuffix.PublicSuffixPatterns` cannot be found after
+                                         relocation of Guava's `com.google.commmon` packages. -->
+                                    <pattern>com.google.thirdparty</pattern>
+                                    <shadedPattern>org.apache.metron.guava.thirdparty.${guava_version}</shadedPattern>
+                                </relocation>
                                 <relocation>
                                     <pattern>org.apache.http</pattern>
                                     <shadedPattern>org.apache.metron.httpcore.dataload</shadedPattern>

metron-platform/metron-elasticsearch/metron-elasticsearch-storm/pom.xml

@@ -167,6 +167,10 @@
                                     <pattern>com.fasterxml.jackson</pattern>
                                     <shadedPattern>org.apache.metron.jackson</shadedPattern>
                                 </relocation>
+                                <relocation>
+                                    <pattern>org.apache.http</pattern>
+                                    <shadedPattern>org.apache.metron.http.elasticsearch</shadedPattern>
+                                </relocation>
                             </relocations>
                             <artifactSet>
                                 <excludes>

metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDaoTest.java

@@ -46,7 +46,6 @@
 
 import com.google.common.collect.ImmutableMap;
 import org.adrianwalker.multilinestring.Multiline;
-import org.apache.commons.math.util.MathUtils;
 import org.apache.metron.common.Constants;
 import org.apache.metron.common.Constants.Fields;
 import org.apache.metron.indexing.dao.IndexDao;
@@ -846,11 +845,11 @@ private boolean metaAlertDocumentEquals(Document expected, Document actual) {
       Object value = entry.getValue();
       Object actualValue = actual.getDocument().get(entry.getKey());
       if (value instanceof Float) {
-        if (!MathUtils.equals((Float) value, (Float) actualValue, EPS)) {
+        if (!(Math.abs((Float) value - (Float) actualValue) < EPS)) {
           return false;
         }
       } else if (value instanceof Double) {
-        if (!MathUtils.equals((Double) value, (Double) actualValue, EPS)) {
+        if (!(Math.abs((Double) value - (Double) actualValue) < EPS)) {
           return false;
         }
       } else {

metron-platform/metron-management/src/main/java/org/apache/metron/management/KafkaFunctions.java

@@ -175,7 +175,9 @@ public class KafkaFunctions {
           namespace = "KAFKA",
           name = "GET",
           description = "Retrieves messages from a Kafka topic.  Subsequent calls will" +
-                  "continue retrieving messages sequentially from the original offset.",
+                  "continue retrieving messages sequentially from the original offset.  When Kerberos is enabled, the " +
+                  "'security.protocol' config key must be set to 'SASL_PLAINTEXT' and the 'group.id' config key must be " +
+                  "set to an authorized group.",
           params = {
                   "topic - The name of the Kafka topic",
                   "count - The number of Kafka messages to retrieve",
@@ -291,7 +293,9 @@ public boolean isInitialized() {
   @Stellar(
           namespace = "KAFKA",
           name = "TAIL",
-          description = "Tails messages from a Kafka topic always starting with the most recently received message.",
+          description = "Tails messages from a Kafka topic always starting with the most recently received message.  When " +
+                  "Kerberos is enabled, the 'security.protocol' config key must be set to 'SASL_PLAINTEXT' and the " +
+                  "'group.id' config key must be set to an authorized group.",
           params = {
                   "topic - The name of the Kafka topic",
                   "count - The number of Kafka messages to retrieve",
@@ -403,7 +407,8 @@ public boolean isInitialized() {
   @Stellar(
           namespace = "KAFKA",
           name = "PUT",
-          description = "Sends messages to a Kafka topic. ",
+          description = "Sends messages to a Kafka topic. When Kerberos is enabled, the 'security.protocol' config key " +
+                  "must be set to 'SASL_PLAINTEXT' and the 'group.id' config key must be set to an authorized group.",
           params = {
                   "topic - The name of the Kafka topic.",
                   "messages - A list of messages to write.",

metron-platform/metron-parsing/metron-parsing-storm/pom.xml

@@ -28,6 +28,7 @@
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+    <guava_version>${global_hbase_guava_version}</guava_version>
   </properties>
   <dependencies>
     <!-- Dependencies needed preferentially -->
@@ -218,6 +219,17 @@
                   <pattern>com.fasterxml.jackson</pattern>
                   <shadedPattern>org.apache.metron.jackson</shadedPattern>
                 </relocation>
+                <relocation>
+                  <pattern>com.google.common</pattern>
+                  <shadedPattern>org.apache.metron.guava.${guava_version}</shadedPattern>
+                </relocation>
+                <relocation>
+                  <!-- need to also relocate Guava's `thirdparty` packages. otherwise, the class
+                       `thirdparty.publicsuffix.PublicSuffixPatterns` cannot be found after
+                       relocation of Guava's `com.google.commmon` packages. -->
+                  <pattern>com.google.thirdparty</pattern>
+                  <shadedPattern>org.apache.metron.guava.thirdparty.${guava_version}</shadedPattern>
+                </relocation>
               </relocations>
               <artifactSet>
                 <excludes>