Merge branch 'hotfix/4.1.12'

CHANGELOG.md

@@ -1,5 +1,15 @@
 # Change Log
 
+## [4.1.12](https://github.com/TheHive-Project/TheHive/milestone/82) (2021-10-29)
+
+**Fixed bugs:**
+
+- [Bug] Upon case merge: missing webhook events about the operation performed on merged cases [\#1969](https://github.com/TheHive-Project/TheHive/issues/1969)
+- [Bug] Uploading zipped observables raises a ClassCastException [\#2224](https://github.com/TheHive-Project/TheHive/issues/2224)
+- [Bug] The search for an observable data is slow [\#2225](https://github.com/TheHive-Project/TheHive/issues/2225)
+- [Bug] Uploading files could cause TheHive crash with "too many open files" errror [\#2226](https://github.com/TheHive-Project/TheHive/issues/2226)
+- [Bug] Configuration containing endpoint with "authentication: none" cannot be written [\#2231](https://github.com/TheHive-Project/TheHive/issues/2231)
+
 ## [4.1.11](https://github.com/TheHive-Project/TheHive/milestone/81) (2021-10-06)
 
 **Implemented enhancements:**

build.sbt

@@ -2,7 +2,7 @@ import Dependencies._
 import com.typesafe.sbt.packager.Keys.bashScriptDefines
 import org.thp.ghcl.Milestone
 
-val thehiveVersion         = "4.1.11-1"
+val thehiveVersion         = "4.1.12-1"
 val scala212               = "2.12.13"
 val scala213               = "2.13.1"
 val supportedScalaVersions = List(scala212, scala213)

client-common/src/main/scala/org/thp/client/Authentication.scala

@@ -28,6 +28,8 @@ object Authentication {
     case PasswordAuthentication(username, password) => Json.obj("type" -> "basic", "username" -> username, "password" -> password)
     case KeyAuthentication(key, "")                 => Json.obj("type" -> "key", "key" -> key)
     case KeyAuthentication(key, "Bearer ")          => Json.obj("type" -> "bearer", "key" -> key)
+    case NoAuthentication                           => Json.obj("type" -> "none")
+    case KeyAuthentication(key, other)              => Json.obj("type" -> other, "key" -> key)
   }
   implicit val format: Format[Authentication] = Format(reads, writes)
 }

cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/ActionCtrl.scala

@@ -93,7 +93,8 @@ class PublicAction @Inject() (actionSrv: ActionSrv, organisationSrv: Organisatio
   val actionsQuery: Query = new Query {
     override val name: String = "actions"
     override def checkFrom(t: ru.Type): Boolean =
-      SubType(t, ru.typeOf[Traversal.V[Case]]) || SubType(t, ru.typeOf[Traversal.V[Observable]]) ||
+      SubType(t, ru.typeOf[Traversal.V[Case]]) ||
+        SubType(t, ru.typeOf[Traversal.V[Observable]]) ||
         SubType(t, ru.typeOf[Traversal.V[Task]]) ||
         SubType(t, ru.typeOf[Traversal.V[Log]]) ||
         SubType(t, ru.typeOf[Traversal.V[Alert]])

frontend/bower.json

@@ -1,6 +1,6 @@
 {
   "name": "thehive",
-  "version": "4.1.11-1",
+  "version": "4.1.12-1",
   "license": "AGPL-3.0",
   "dependencies": {
     "jquery": "^3.4.1",

frontend/package.json

@@ -1,6 +1,6 @@
 {
     "name": "thehive",
-    "version": "4.1.11-1",
+    "version": "4.1.12-1",
     "license": "AGPL-3.0",
     "repository": {
         "type": "git",

thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala

@@ -1,7 +1,7 @@
 package org.thp.thehive.controllers.v0
 
 import io.scalaland.chimney.dsl._
-import org.apache.tinkerpop.gremlin.process.traversal.{Compare, Contains}
+import org.apache.tinkerpop.gremlin.process.traversal.{Compare, Contains, P}
 import org.thp.scalligraph.auth.AuthContext
 import org.thp.scalligraph.controllers._
 import org.thp.scalligraph.models.{Database, Entity, UMapping}
@@ -367,6 +367,7 @@ class PublicAlert @Inject() (
     alertSrv: AlertSrv,
     organisationSrv: OrganisationSrv,
     customFieldSrv: CustomFieldSrv,
+    observableSrv: ObservableSrv,
     db: Database
 ) extends PublicData {
   override val entityName: String = "alert"
@@ -392,7 +393,11 @@ class PublicAlert @Inject() (
   override val outputQuery: Query = Query.output[RichAlert, Traversal.V[Alert]](_.richAlert)
   override val extraQueries: Seq[ParamQuery[_]] = Seq(
     Query[Traversal.V[Alert], Traversal.V[Case]]("cases", (alertSteps, _) => alertSteps.`case`),
-    Query[Traversal.V[Alert], Traversal.V[Observable]]("observables", (alertSteps, _) => alertSteps.observables),
+    Query[Traversal.V[Alert], Traversal.V[Observable]](
+      "observables",
+      (alertSteps, authContext) =>
+        observableSrv.startTraversal(alertSteps.graph).has(_.relatedId, P.within(alertSteps._id.toSeq: _*)).visible(organisationSrv)(authContext)
+    ),
     Query[
       Traversal.V[Alert],
       Traversal[(RichAlert, Seq[RichObservable]), JMap[String, Any], Converter[(RichAlert, Seq[RichObservable]), JMap[String, Any]]]

thehive/app/org/thp/thehive/controllers/v0/DashboardCtrl.scala

@@ -23,7 +23,6 @@ import scala.util.{Failure, Success}
 class DashboardCtrl @Inject() (
     override val entrypoint: Entrypoint,
     dashboardSrv: DashboardSrv,
-    userSrv: UserSrv,
     implicit val db: Database,
     override val publicData: PublicDashboard,
     @Named("v0") override val queryExecutor: QueryExecutor

thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala

@@ -343,8 +343,8 @@ class ObservableCtrl @Inject() (
 
   private def getZipFiles(observable: InputObservable, zipPassword: Option[String]): Seq[InputObservable] =
     observable.attachment.flatMap(_.swap.toSeq).flatMap { attachment =>
-      val zipFile                = new ZipFile(attachment.filepath.toFile)
-      val files: Seq[FileHeader] = zipFile.getFileHeaders.asScala.asInstanceOf[Seq[FileHeader]]
+      val zipFile = new ZipFile(attachment.filepath.toFile)
+      val files   = zipFile.getFileHeaders.asScala
 
       if (zipFile.isEncrypted)
         zipFile.setPassword(zipPassword.getOrElse(configuration.get[String]("datastore.attachment.password")).toCharArray)
@@ -367,7 +367,7 @@ class PublicObservable @Inject() (
   override val initialQuery: Query =
     Query.init[Traversal.V[Observable]](
       "listObservable",
-      (graph, authContext) => organisationSrv.get(authContext.organisation)(graph).shares.observables
+      (graph, authContext) => observableSrv.startTraversal(graph).visible(organisationSrv)(authContext)
     )
   override val getQuery: ParamQuery[EntityIdOrName] = Query.initWithParam[EntityIdOrName, Traversal.V[Observable]](
     "getObservable",

thehive/app/org/thp/thehive/controllers/v0/TaskCtrl.scala

@@ -99,7 +99,7 @@ class PublicTask @Inject() (taskSrv: TaskSrv, organisationSrv: OrganisationSrv,
   override val initialQuery: Query =
     Query.init[Traversal.V[Task]](
       "listTask",
-      (graph, authContext) => taskSrv.startTraversal(graph).inOrganisation(organisationSrv.currentId(graph, authContext))
+      (graph, authContext) => taskSrv.startTraversal(graph).visible(organisationSrv)(authContext)
     )
   //organisationSrv.get(authContext.organisation)(graph).shares.tasks)
   override val pageQuery: ParamQuery[OutputParam] = Query.withParam[OutputParam, Traversal.V[Task], IteratorOutput](

thehive/app/org/thp/thehive/controllers/v0/TheHiveQueryExecutor.scala

@@ -9,7 +9,7 @@ import org.thp.scalligraph.services.config.{ApplicationConfig, ConfigItem}
 import org.thp.scalligraph.traversal.Traversal
 import org.thp.scalligraph.traversal.TraversalOps._
 import org.thp.scalligraph.utils.RichType
-import org.thp.scalligraph.{BadRequestError, EntityId, EntityIdOrName, GlobalQueryExecutor}
+import org.thp.scalligraph.{BadRequestError, EntityId, GlobalQueryExecutor}
 import org.thp.thehive.models._
 import org.thp.thehive.services.AlertOps._
 import org.thp.thehive.services.CaseOps._
@@ -90,7 +90,7 @@ class TheHiveQueryExecutor @Inject() (
   override val customFilterQuery: FilterQuery = FilterQuery(publicProperties) { (tpe, globalParser) =>
     FieldsParser("parentChildFilter") {
       case (_, FObjOne("_parent", ParentIdFilter(parentType, parentId))) if parentTypes.isDefinedAt((tpe, parentType)) =>
-        Good(new ParentIdInputFilter(parentType, parentId))
+        Good(new ParentIdInputFilter(parentId))
       case (path, FObjOne("_parent", ParentQueryFilter(parentType, parentFilterField))) if parentTypes.isDefinedAt((tpe, parentType)) =>
         globalParser(parentTypes((tpe, parentType))).apply(path, parentFilterField).map(query => new ParentQueryInputFilter(parentType, query))
       case (path, FObjOne("_child", ChildQueryFilter(childType, childQueryField))) if childTypes.isDefinedAt((tpe, childType)) =>
@@ -118,7 +118,7 @@ object ParentIdFilter {
       .fold(Some(_), _ => None)
 }
 
-class ParentIdInputFilter(parentType: String, parentId: String) extends InputQuery[Traversal.Unk, Traversal.Unk] {
+class ParentIdInputFilter(parentId: String) extends InputQuery[Traversal.Unk, Traversal.Unk] {
   override def apply(
       publicProperties: PublicProperties,
       traversalType: ru.Type,
@@ -129,35 +129,20 @@ class ParentIdInputFilter(parentType: String, parentId: String) extends InputQue
       .getTypeArgs(traversalType, ru.typeOf[Traversal[_, _, _]])
       .headOption
       .collect {
-        case t if t <:< ru.typeOf[Task] && parentType == "caseTemplate" =>
-          traversal
-            .asInstanceOf[Traversal.V[Task]]
-            .filter(_.caseTemplate.get(EntityIdOrName(parentId)))
-            .asInstanceOf[Traversal.Unk]
         case t if t <:< ru.typeOf[Task] =>
           traversal
             .asInstanceOf[Traversal.V[Task]]
-            .filter(_.`case`.get(EntityIdOrName(parentId)))
+            .has(_.relatedId, EntityId(parentId))
             .asInstanceOf[Traversal.Unk]
         case t if t <:< ru.typeOf[Observable] =>
           traversal
             .asInstanceOf[Traversal.V[Observable]]
             .has(_.relatedId, EntityId(parentId))
             .asInstanceOf[Traversal.Unk]
-//          && parentType == "alert" =>
-//          traversal
-//            .asInstanceOf[Traversal.V[Observable]]
-//            .filter(_.alert.get(EntityIdOrName(parentId)))
-//            .asInstanceOf[Traversal.Unk]
-//        case t if t <:< ru.typeOf[Observable] =>
-//          traversal
-//            .asInstanceOf[Traversal.V[Observable]]
-//            .filter(_.`case`.get(EntityIdOrName(parentId)))
-//            .asInstanceOf[Traversal.Unk]
         case t if t <:< ru.typeOf[Log] =>
           traversal
             .asInstanceOf[Traversal.V[Log]]
-            .filter(_.task.get(EntityIdOrName(parentId)))
+            .has(_.taskId, EntityId(parentId))
             .asInstanceOf[Traversal.Unk]
       }
       .getOrElse(throw BadRequestError(s"$traversalType hasn't parent"))

thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala

@@ -56,7 +56,7 @@ class ObservableCtrl @Inject() (
   override val initialQuery: Query =
     Query.init[Traversal.V[Observable]](
       "listObservable",
-      (graph, authContext) => organisationSrv.get(authContext.organisation)(graph).shares.observables
+      (graph, authContext) => observableSrv.startTraversal(graph).visible(organisationSrv)(authContext)
     )
   override val getQuery: ParamQuery[EntityIdOrName] = Query.initWithParam[EntityIdOrName, Traversal.V[Observable]](
     "getObservable",
@@ -372,8 +372,8 @@ class ObservableCtrl @Inject() (
 
   private def getZipFiles(observable: InputObservable, zipPassword: Option[String]): Seq[InputObservable] =
     observable.attachment.flatMap(_.swap.toSeq).flatMap { attachment =>
-      val zipFile                = new ZipFile(attachment.filepath.toFile)
-      val files: Seq[FileHeader] = zipFile.getFileHeaders.asScala.asInstanceOf[Seq[FileHeader]]
+      val zipFile = new ZipFile(attachment.filepath.toFile)
+      val files   = zipFile.getFileHeaders.asScala
 
       if (zipFile.isEncrypted)
         zipFile.setPassword(zipPassword.getOrElse(configuration.get[String]("datastore.attachment.password")).toCharArray)

thehive/app/org/thp/thehive/controllers/v1/TaskCtrl.scala

@@ -36,7 +36,7 @@ class TaskCtrl @Inject() (
   override val initialQuery: Query =
     Query.init[Traversal.V[Task]](
       "listTask",
-      (graph, authContext) => taskSrv.startTraversal(graph).inOrganisation(organisationSrv.currentId(graph, authContext))
+      (graph, authContext) => taskSrv.startTraversal(graph).visible(organisationSrv)(authContext)
 //        organisationSrv.get(authContext.organisation)(graph).shares.tasks)
     )
   override val pageQuery: ParamQuery[OutputParam] = Query.withParam[OutputParam, Traversal.V[Task], IteratorOutput](

thehive/app/org/thp/thehive/services/AttachmentSrv.scala

@@ -33,12 +33,10 @@ class AttachmentSrv @Inject() (configuration: Configuration, storageSrv: Storage
     val hs = hashers.fromPath(file.filepath)
     val id = hs.head.toString
     val is = Files.newInputStream(file.filepath)
-    val result =
-      storageSrv
-        .saveBinary("attachment", id, is)
-        .flatMap(_ => createEntity(Attachment(file.filename, Files.size(file.filepath), file.contentType, hs, id)))
-    is.close()
-    result
+    try storageSrv
+      .saveBinary("attachment", id, is)
+      .flatMap(_ => createEntity(Attachment(file.filename, Files.size(file.filepath), file.contentType, hs, id)))
+    finally is.close()
   }
 
   def create(filename: String, contentType: String, data: Array[Byte])(implicit

thehive/app/org/thp/thehive/services/AuditSrv.scala

@@ -176,11 +176,6 @@ class AuditSrv @Inject() (
     def delete(entity: E with Entity, context: C with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] =
       auditSrv.create(Audit(Audit.delete, entity, None), context, None)
 
-    def merge(entity: E with Entity, destination: C with Entity, details: Option[JsObject] = None)(implicit
-        graph: Graph,
-        authContext: AuthContext
-    ): Try[Unit] =
-      auditSrv.create(Audit(Audit.merge, destination, details.map(_.toString())), destination, Some(destination))
   }
 
   class SelfContextObjectAudit[E <: Product] {
@@ -197,6 +192,12 @@ class AuditSrv @Inject() (
         authContext: AuthContext
     ): Try[Unit] =
       auditSrv.create(Audit(Audit.delete, entity, details.map(_.toString())), context, None)
+
+    def merge(entity: E with Entity, details: Option[JsObject] = None)(implicit
+        graph: Graph,
+        authContext: AuthContext
+    ): Try[Unit] =
+      auditSrv.create(Audit(Audit.merge, entity, details.map(_.toString())), entity, Some(entity))
   }
 
   class UserAudit extends SelfContextObjectAudit[User] {

thehive/app/org/thp/thehive/services/CaseSrv.scala

@@ -352,61 +352,67 @@ class CaseSrv @Inject() (
   }
 
   def merge(cases: Seq[Case with Entity])(implicit graph: Graph, authContext: AuthContext): Try[RichCase] =
-    if (cases.size > 1 && canMerge(cases)) {
-      val mergedCase = Case(
-        cases.map(_.title).mkString(" / "),
-        cases.map(_.description).mkString("\n\n"),
-        cases.map(_.severity).max,
-        cases.map(_.startDate).min,
-        None,
-        cases.exists(_.flag),
-        cases.map(_.tlp).max,
-        cases.map(_.pap).max,
-        CaseStatus.Open,
-        cases.map(_.summary).fold(None)((s1, s2) => (s1 ++ s2).reduceOption(_ + "\n\n" + _)),
-        cases.flatMap(_.tags).distinct
-      )
+    if (cases.size > 1 && canMerge(cases))
+      auditSrv.mergeAudits {
+        val mergedCase = Case(
+          cases.map(_.title).mkString(" / "),
+          cases.map(_.description).mkString("\n\n"),
+          cases.map(_.severity).max,
+          cases.map(_.startDate).min,
+          None,
+          cases.exists(_.flag),
+          cases.map(_.tlp).max,
+          cases.map(_.pap).max,
+          CaseStatus.Open,
+          cases.map(_.summary).fold(None)((s1, s2) => (s1 ++ s2).reduceOption(_ + "\n\n" + _)),
+          cases.flatMap(_.tags).distinct
+        )
 
-      val allProfilesOrgas: Seq[(Profile with Entity, Organisation with Entity)] = get(cases.head)
-        .shares
-        .project(_.by(_.profile).by(_.organisation))
-        .toSeq
-
-      for {
-        user        <- userSrv.current.getOrFail("User")
-        currentOrga <- organisationSrv.current.getOrFail("Organisation")
-        richCase    <- create(mergedCase, Some(user), currentOrga, Seq(), None, Seq())
-        // Share case with all organisations except the one who created the merged case
-        _ <-
-          allProfilesOrgas
-            .filterNot(_._2._id == currentOrga._id)
-            .toTry(profileOrg => shareSrv.shareCase(owner = false, richCase.`case`, profileOrg._2, profileOrg._1))
-        _ <- cases.toTry { c =>
-          for {
-
-            _ <- shareMergedCaseTasks(allProfilesOrgas.map(_._2), c, richCase.`case`)
-            _ <- shareMergedCaseObservables(allProfilesOrgas.map(_._2), c, richCase.`case`)
-            _ <-
-              get(c)
-                .alert
-                .update(_.caseId, richCase._id)
-                .toSeq
-                .toTry(alertSrv.alertCaseSrv.create(AlertCase(), _, richCase.`case`))
-            _ <-
-              get(c)
-                .procedure
-                .toSeq
-                .toTry(caseProcedureSrv.create(CaseProcedure(), richCase.`case`, _))
-            _ <-
-              get(c)
-                .richCustomFields
-                .toSeq
-                .toTry(c => createCustomField(richCase.`case`, EntityIdOrName(c.customField.name), c.value, c.order))
-          } yield Success(())
-        }
-        _ <- cases.toTry(super.delete(_))
-      } yield richCase
-    } else
+        val allProfilesOrgas: Seq[(Profile with Entity, Organisation with Entity)] = get(cases.head)
+          .shares
+          .project(_.by(_.profile).by(_.organisation))
+          .toSeq
+
+        for {
+          user        <- userSrv.current.getOrFail("User")
+          currentOrga <- organisationSrv.current.getOrFail("Organisation")
+          richCase    <- create(mergedCase, Some(user), currentOrga, Seq(), None, Seq())
+          // Share case with all organisations except the one who created the merged case
+          _ <-
+            allProfilesOrgas
+              .filterNot(_._2._id == currentOrga._id)
+              .toTry(profileOrg => shareSrv.shareCase(owner = false, richCase.`case`, profileOrg._2, profileOrg._1))
+          _ <- cases.toTry { c =>
+            for {
+
+              _ <- shareMergedCaseTasks(allProfilesOrgas.map(_._2), c, richCase.`case`)
+              _ <- shareMergedCaseObservables(allProfilesOrgas.map(_._2), c, richCase.`case`)
+              _ <-
+                get(c)
+                  .alert
+                  .update(_.caseId, richCase._id)
+                  .toSeq
+                  .toTry(alertSrv.alertCaseSrv.create(AlertCase(), _, richCase.`case`))
+              _ <-
+                get(c)
+                  .procedure
+                  .toSeq
+                  .toTry(caseProcedureSrv.create(CaseProcedure(), richCase.`case`, _))
+              _ <-
+                get(c)
+                  .richCustomFields
+                  .toSeq
+                  .toTry(c => createCustomField(richCase.`case`, EntityIdOrName(c.customField.name), c.value, c.order))
+            } yield Success(())
+          }
+          _ <- cases.toTry(super.delete(_))
+        } yield richCase
+      }(mergedCase =>
+        auditSrv
+          .`case`
+          .merge(mergedCase.`case`, Some(Json.obj("cases" -> cases.map(c => Json.obj("_id" -> c._id, "number" -> c.number, "title" -> c.title)))))
+      )
+    else
       Failure(BadRequestError("To be able to merge, cases must have same organisation / profile pair and user must be org-admin"))
 
   private def canMerge(cases: Seq[Case with Entity])(implicit graph: Graph, authContext: AuthContext): Boolean = {

thehive/app/org/thp/thehive/services/ObservableSrv.scala

@@ -253,7 +253,7 @@ object ObservableOps {
 
     def organisations: Traversal.V[Organisation] =
       traversal
-        .unionFlat(identity, _.in("ReportObservable").in("ObservableJob").v[Observable])
+        .optional(_.in("ReportObservable").in("ObservableJob").v[Observable])
         .unionFlat(_.shares.organisation, _.alert.organisation)
 //      traversal.coalesceIdent(_.in[ShareObservable].in[OrganisationShare], _.in[AlertObservable].out[AlertOrganisation]).v[Organisation]