From 83b8bdbe892eb24db10272c841768ce3996cc989 Mon Sep 17 00:00:00 2001 From: To-om Date: Wed, 14 Apr 2021 15:33:53 +0200 Subject: [PATCH 01/17] #1964 Prevent failure if index is not found --- .../app/org/thp/thehive/controllers/v1/AdminCtrl.scala | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/thehive/app/org/thp/thehive/controllers/v1/AdminCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/AdminCtrl.scala index 77ca67817b..21637a1986 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/AdminCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/AdminCtrl.scala @@ -117,7 +117,15 @@ class AdminCtrl @Inject() ( entrypoint("Get index status") .authPermittedRoTransaction(db, Permissions.managePlatform) { _ => graph => val indices = labels.map { label => - Json.obj("name" -> label, "count" -> graph.indexCountQuery(s"""v."_label":$label""")) + val count = + try graph.indexCountQuery(s"""v."_label":$label""") + catch { + case error: Throwable => + logger.error("Index fetch error", error) + 0 + } + Json.obj("name" -> label, "count" -> count) + } val indexCount = Json.obj("name" -> "global", "indices" -> indices) Success(Results.Ok(Json.obj("index" -> Seq(indexCount)))) From 852f8290137297ac65387421003e3646e7abef35 Mon Sep 17 00:00:00 2001 From: To-om Date: Wed, 14 Apr 2021 15:35:17 +0200 Subject: [PATCH 02/17] #1964 Hide not-real-errors --- conf/logback.xml | 1 + package/logback.xml | 1 + 2 files changed, 2 insertions(+) diff --git a/conf/logback.xml b/conf/logback.xml index d8072baa3e..81ebc5d266 100644 --- a/conf/logback.xml +++ b/conf/logback.xml @@ -37,6 +37,7 @@ + diff --git a/package/logback.xml b/package/logback.xml index ae905cd3d9..bf16f2bb5e 100644 --- a/package/logback.xml +++ b/package/logback.xml @@ -37,6 +37,7 @@ + From 0550a0823ecc25bb60d2b2332c2e904c9316f184 Mon Sep 17 00:00:00 2001 From: To-om Date: Wed, 14 Apr 2021 15:35:50 +0200 Subject: [PATCH 03/17] #1964 Set fielddata parameter on full-text fields --- ScalliGraph | 2 +- .../app/org/thp/thehive/models/TheHiveSchemaDefinition.scala | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ScalliGraph b/ScalliGraph index 78de3a32ac..97ecfaf7e9 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit 78de3a32ac7d9d09142eff2dd5b4c0d44147067b +Subproject commit 97ecfaf7e9d0f29afa5b12d2f831ca85b7504d8f diff --git a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala index 0763d4397e..83e6ac6acf 100644 --- a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala +++ b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala @@ -469,7 +469,8 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { .removeIndex("Observable", IndexType.standard) .removeIndex("Tag", IndexType.standard) .removeIndex("Task", IndexType.standard) - //=====[release 4.1.3]===== + //=====[release 4.1.3]===== + .removeIndex("global", IndexType.fulltext) val reflectionClasses = new Reflections( new ConfigurationBuilder() From 7335da976d6abe6613c4fc424186535bfb9f0c2a Mon Sep 17 00:00:00 2001 From: To-om Date: Wed, 14 Apr 2021 16:39:22 +0200 Subject: [PATCH 04/17] #1965 Fix customField selection query for alerts --- thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala | 4 +--- thehive/app/org/thp/thehive/services/CustomFieldSrv.scala | 7 ------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala index 33e6294d49..076d7746fe 100644 --- a/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala +++ b/thehive/app/org/thp/thehive/controllers/v0/AlertCtrl.scala @@ -468,9 +468,7 @@ class PublicAlert @Inject() ( .property("user", UMapping.string)(_.field.updatable) .property("customFields", UMapping.jsonNative)(_.subSelect { case (FPathElem(_, FPathElem(idOrName, _)), alerts) => - alerts - .customFields(EntityIdOrName(idOrName)) - .jsonValue + alerts.customFieldJsonValue(customFieldSrv, EntityIdOrName(idOrName)) case (_, alerts) => alerts.customFields.nameJsonValue.fold.domainMap(JsObject(_)) } .filter[JsValue] { diff --git a/thehive/app/org/thp/thehive/services/CustomFieldSrv.scala b/thehive/app/org/thp/thehive/services/CustomFieldSrv.scala index 5b0c9af5be..9f379b5400 100644 --- a/thehive/app/org/thp/thehive/services/CustomFieldSrv.scala +++ b/thehive/app/org/thp/thehive/services/CustomFieldSrv.scala @@ -131,13 +131,6 @@ object CustomFieldOps { name -> CustomFieldType.map(tpe).getJsonValue(new CustomFieldValueEdge(edge)) } - def jsonValue: Traversal[JsValue, JMap[String, Any], Converter[JsValue, JMap[String, Any]]] = - edgeNameType - .domainMap { - case (edge, _, tpe) => - CustomFieldType.map(tpe).getJsonValue(new CustomFieldValueEdge(edge)) - } - def nameValue: Traversal[(String, Option[_]), JMap[String, Any], Converter[(String, Option[_]), JMap[String, Any]]] = edgeNameType .domainMap { From 67dfae203b2055aa9c183f58db30b282ea86719e Mon Sep 17 00:00:00 2001 From: To-om Date: Wed, 14 Apr 2021 16:54:12 +0200 Subject: [PATCH 05/17] #1966 Change database initialisation timeout --- thehive/conf/reference.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/thehive/conf/reference.conf b/thehive/conf/reference.conf index 0bcf7561f6..c2d37f6e68 100644 --- a/thehive/conf/reference.conf +++ b/thehive/conf/reference.conf @@ -9,7 +9,7 @@ db { directory: /opt/thp/thehive/database } } - initialisationTimeout: 1 hour + initialisationTimeout: 14 days } query.limitedCountThreshold: 1000 From 23e407ced50d59574157e35c3c78eb67551492c1 Mon Sep 17 00:00:00 2001 From: To-om Date: Wed, 14 Apr 2021 17:26:56 +0200 Subject: [PATCH 06/17] #1967 Make alert deletion more robust --- .../org/thp/thehive/services/AlertSrv.scala | 15 ++++--- .../thp/thehive/services/ObservableSrv.scala | 40 ++++++++++--------- 2 files changed, 31 insertions(+), 24 deletions(-) diff --git a/thehive/app/org/thp/thehive/services/AlertSrv.scala b/thehive/app/org/thp/thehive/services/AlertSrv.scala index 9783100087..44924b27a3 100644 --- a/thehive/app/org/thp/thehive/services/AlertSrv.scala +++ b/thehive/app/org/thp/thehive/services/AlertSrv.scala @@ -354,12 +354,15 @@ class AlertSrv @Inject() ( .map(_ => ()) def remove(alert: Alert with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = - for { - organisation <- organisationSrv.getOrFail(authContext.organisation) - _ <- get(alert).observables.toIterator.toTry(observableSrv.delete(_)) - _ = get(alert).remove() - _ <- auditSrv.alert.delete(alert, organisation) - } yield () + auditSrv.mergeAudits { + get(alert).observables.toIterator.foreach(observableSrv.delete(_)) + Success(()) + } { _ => + for { + organisation <- organisationSrv.getOrFail(authContext.organisation) + _ <- auditSrv.alert.delete(alert, organisation) + } yield get(alert).remove() + } } object AlertOps { diff --git a/thehive/app/org/thp/thehive/services/ObservableSrv.scala b/thehive/app/org/thp/thehive/services/ObservableSrv.scala index cbea7a27e1..a2e9f0d8cd 100644 --- a/thehive/app/org/thp/thehive/services/ObservableSrv.scala +++ b/thehive/app/org/thp/thehive/services/ObservableSrv.scala @@ -137,24 +137,28 @@ class ObservableSrv @Inject() ( override def delete(observable: Observable with Entity)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = get(observable).alert.headOption match { case None => - get(observable).share.getOrFail("Share").flatMap { - case share if share.owner => - get(observable) - .shares - .toIterator - .toTry { share => - auditSrv - .observable - .delete(observable, share) - } - .map(_ => get(observable).remove()) - case share => - for { - organisation <- organisationSrv.current.getOrFail("Organisation") - _ <- shareSrv.unshareObservable(observable, organisation) - _ <- auditSrv.observable.delete(observable, share) - } yield () - } + get(observable) + .share + .toIterator + .toTry { + case share if share.owner => + get(observable) + .shares + .toIterator + .toTry { share => + auditSrv + .observable + .delete(observable, share) + } + .map(_ => get(observable).remove()) + case share => + for { + organisation <- organisationSrv.current.getOrFail("Organisation") + _ <- shareSrv.unshareObservable(observable, organisation) + _ <- auditSrv.observable.delete(observable, share) + } yield () + } + .map(_ => ()) case Some(alert) => get(observable).remove() auditSrv.observableInAlert.delete(observable, alert) From c8b24f9b650de831ae66e616da14f99617dc8e71 Mon Sep 17 00:00:00 2001 From: To-om Date: Wed, 14 Apr 2021 19:25:11 +0200 Subject: [PATCH 07/17] #1968 Add manageProcedure to analyst role --- thehive/app/org/thp/thehive/models/Role.scala | 3 ++- .../models/TheHiveSchemaDefinition.scala | 27 ++++++++++++------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/thehive/app/org/thp/thehive/models/Role.scala b/thehive/app/org/thp/thehive/models/Role.scala index 996b9709bd..953974dce6 100644 --- a/thehive/app/org/thp/thehive/models/Role.scala +++ b/thehive/app/org/thp/thehive/models/Role.scala @@ -27,7 +27,8 @@ object Profile { Permissions.manageShare, Permissions.manageAnalyse, Permissions.managePage, - Permissions.accessTheHiveFS + Permissions.accessTheHiveFS, + Permissions.manageProcedure ) ) val readonly: Profile = Profile("read-only", Set.empty) diff --git a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala index 83e6ac6acf..492ceca462 100644 --- a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala +++ b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala @@ -184,14 +184,15 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { traversal.unsafeHas("name", "admin").raw.property("permissions", "managePattern").iterate() Success(()) } - .updateGraph("Add manageProcedure permission to org-admin and analyst profiles", "Profile") { traversal => - traversal - .unsafeHas("name", P.within("org-admin", "analyst")) - .raw - .property("permissions", "manageProcedure") - .iterate() - Success(()) - } + .noop +// .updateGraph("Add manageProcedure permission to org-admin and analyst profiles", "Profile") { traversal => +// traversal +// .unsafeHas("name", P.within("org-admin", "analyst")) +// .raw +// .property("permissions", "manageProcedure") +// .iterate() +// Success(()) +// } // Index backend /* Alert index */ .addProperty[Seq[String]]("Alert", "tags") @@ -266,7 +267,7 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { } /* CaseTemplate index */ .addProperty[Seq[String]]("CaseTemplate", "tags") - .updateGraph("Add tags in caseTempates", "CaseTemplate") { traversal => + .updateGraph("Add tags in caseTemplates", "CaseTemplate") { traversal => traversal .project( _.by @@ -471,6 +472,14 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { .removeIndex("Task", IndexType.standard) //=====[release 4.1.3]===== .removeIndex("global", IndexType.fulltext) + .updateGraph("Add manageProcedure permission to org-admin and analyst profiles", "Profile") { traversal => + traversal + .unsafeHas("name", P.within("org-admin", "analyst")) + .raw + .property("permissions", "manageProcedure") + .iterate() + Success(()) + } val reflectionClasses = new Reflections( new ConfigurationBuilder() From 6475b8657d53678c821eaef75d45df04c0159b59 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Thu, 15 Apr 2021 06:25:50 +0200 Subject: [PATCH 08/17] #1968 Add manageProcedure permission to profile dialog. --- .../app/scripts/services/api/ProfileSrv.js | 28 ++++---- .../views/partials/case/case.procedures.html | 67 +++++++++++++------ .../partials/case/procedures/toolbar.html | 6 +- 3 files changed, 65 insertions(+), 36 deletions(-) diff --git a/frontend/app/scripts/services/api/ProfileSrv.js b/frontend/app/scripts/services/api/ProfileSrv.js index 3a1becdfac..1523daa2ac 100644 --- a/frontend/app/scripts/services/api/ProfileSrv.js +++ b/frontend/app/scripts/services/api/ProfileSrv.js @@ -1,7 +1,7 @@ -(function() { +(function () { 'use strict'; angular.module('theHiveServices') - .service('ProfileSrv', function($http) { + .service('ProfileSrv', function ($http) { var self = this; var baseUrl = './api/profile'; @@ -46,6 +46,7 @@ 'manageShare', 'manageObservable', 'manageTask', + 'manageProcedure', 'manageAction', 'manageAnalyse', 'accessTheHiveFS' @@ -59,6 +60,7 @@ manageShare: 'Manage sharing', manageObservable: 'Manage observables', manageTask: 'Manage tasks', + manageProcedure: 'Manage TTPs', manageAction: 'Run Cortex responders', manageAnalyse: 'Run Cortex analyzer', accessTheHiveFS: 'Access to TheHiveFS service' @@ -66,32 +68,34 @@ } }; - this.list = function() { - return $http.get(baseUrl, {params: { - range: 'all' - }}); + this.list = function () { + return $http.get(baseUrl, { + params: { + range: 'all' + } + }); }; - this.get = function(name) { + this.get = function (name) { return $http.get(baseUrl + '/' + name); }; - this.map = function() { + this.map = function () { return self.list() - .then(function(response) { + .then(function (response) { return _.indexBy(response.data, 'name'); }); }; - this.create = function(profile) { + this.create = function (profile) { return $http.post(baseUrl, profile); }; - this.update = function(id, profile) { + this.update = function (id, profile) { return $http.patch(baseUrl + '/' + id, profile); }; - this.remove = function(id) { + this.remove = function (id) { return $http.delete(baseUrl + '/' + id); }; }); diff --git a/frontend/app/views/partials/case/case.procedures.html b/frontend/app/views/partials/case/case.procedures.html index 27a86a1415..37198179e4 100644 --- a/frontend/app/views/partials/case/case.procedures.html +++ b/frontend/app/views/partials/case/case.procedures.html @@ -1,6 +1,7 @@
-
+
@@ -11,7 +12,8 @@

- + @@ -28,7 +30,8 @@

Tactic - + @@ -36,7 +39,8 @@

Technique - + @@ -45,7 +49,8 @@

Occur Date - + @@ -55,29 +60,36 @@

-
+
- + {{$vm.tactics[proc.tactic].label}} - +
- {{proc.patternId}} - {{proc.extraData.patternParent.name}}:{{proc.extraData.pattern.name}} + {{proc.patternId}} - + {{proc.extraData.patternParent.name}}:{{proc.extraData.pattern.name}} - +
- {{proc.patternId}} - {{proc.extraData.pattern.name}} + {{proc.patternId}} - + {{proc.extraData.pattern.name}} - +
@@ -86,11 +98,13 @@

-->
- {{proc.occurDate | shortDate}} + {{proc.occurDate | shortDate}}
- +
@@ -100,35 +114,44 @@

- / + /
- {{proc._createdAt | shortDate}} + {{proc._createdAt | shortDate}}
- {{(proc._updatedAt | shortDate) || '-'}} + {{(proc._updatedAt | shortDate) || '-'}}
-
- +
+ +
+
+ {{(proc.occurDate | shortDate) || '-'}}
-
- +
+ +
-
+
+
+ Not Specified +
diff --git a/frontend/app/views/partials/case/procedures/toolbar.html b/frontend/app/views/partials/case/procedures/toolbar.html index 5cf4cdb69c..763089d138 100644 --- a/frontend/app/views/partials/case/procedures/toolbar.html +++ b/frontend/app/views/partials/case/procedures/toolbar.html @@ -2,7 +2,7 @@

- List of taxonomies

}[!!taxonomy.extraData.enabled]"> - + - +
From b9cd5bee4a44548d87ea24ee766a59882f399079 Mon Sep 17 00:00:00 2001 From: To-om Date: Thu, 15 Apr 2021 08:07:46 +0200 Subject: [PATCH 10/17] #1964 Renew global index only if ElasticSearch is used --- .../org/thp/thehive/models/TheHiveSchemaDefinition.scala | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala index 492ceca462..a910840f40 100644 --- a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala +++ b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala @@ -471,7 +471,12 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { .removeIndex("Tag", IndexType.standard) .removeIndex("Task", IndexType.standard) //=====[release 4.1.3]===== - .removeIndex("global", IndexType.fulltext) + .dbOperation[JanusDatabase]("Remove global index if ElasticSearch is used") { db => + db.managementTransaction(mgmt => Try(mgmt.get("index.search.backend"))).flatMap { + case "elasticsearch" => db.removeIndex("global", IndexType.fulltext, Nil) + case _ => Success(()) + } + } .updateGraph("Add manageProcedure permission to org-admin and analyst profiles", "Profile") { traversal => traversal .unsafeHas("name", P.within("org-admin", "analyst")) From 707b40e779e01c63ffbd8439636f670b928128a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Leonard?= Date: Thu, 15 Apr 2021 11:48:11 +0200 Subject: [PATCH 11/17] #1971 give more context info of the running instance --- .github/ISSUE_TEMPLATE/thehive3_bug_report.md | 8 +++++++- .github/ISSUE_TEMPLATE/thehive4_bug_report.md | 6 ++++++ .github/ISSUE_TEMPLATE/thehive4_question.md | 20 ++++++++++++------- ScalliGraph | 2 +- 4 files changed, 27 insertions(+), 9 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/thehive3_bug_report.md b/.github/ISSUE_TEMPLATE/thehive3_bug_report.md index f73685e1cc..f49a423737 100644 --- a/.github/ISSUE_TEMPLATE/thehive3_bug_report.md +++ b/.github/ISSUE_TEMPLATE/thehive3_bug_report.md @@ -16,8 +16,14 @@ Bug | -------------------------- | ------------------------------------- | | OS version (server) | Debian, Ubuntu, CentOS, RedHat, ... | | OS version (client) | XP, Seven, 10, Ubuntu, ... | -| TheHive version / git hash | 3.x, hash of the commit | +| Virtualized Env. | True / False | +| Dedicated RAM | XX GB | +| vCPU | 4 / 8 / 16 / 32 | +| TheHive version / git hash | 4.x, hash of the commit | | Package Type | RPM, DEB, Docker, Binary, From source | +| Database | Cassandra / BerlkelyDB | +| Index type | Lucene / Elasticsearch | +| Attachments storage | Local, NFS, S3, HDFS | | Browser type & version | If applicable | diff --git a/.github/ISSUE_TEMPLATE/thehive4_bug_report.md b/.github/ISSUE_TEMPLATE/thehive4_bug_report.md index 3e158e3a33..19a0214625 100644 --- a/.github/ISSUE_TEMPLATE/thehive4_bug_report.md +++ b/.github/ISSUE_TEMPLATE/thehive4_bug_report.md @@ -16,8 +16,14 @@ Bug | -------------------------- | ------------------------------------- | | OS version (server) | Debian, Ubuntu, CentOS, RedHat, ... | | OS version (client) | XP, Seven, 10, Ubuntu, ... | +| Virtualized Env. | True / False | +| Dedicated RAM | XX GB | +| vCPU | 4 / 8 / 16 / 32 | | TheHive version / git hash | 4.x, hash of the commit | | Package Type | RPM, DEB, Docker, Binary, From source | +| Database | Cassandra / BerlkelyDB | +| Index type | Lucene / Elasticsearch | +| Attachments storage | Local, NFS, S3, HDFS | | Browser type & version | If applicable | diff --git a/.github/ISSUE_TEMPLATE/thehive4_question.md b/.github/ISSUE_TEMPLATE/thehive4_question.md index a551433c07..3cab78c3e0 100644 --- a/.github/ISSUE_TEMPLATE/thehive4_question.md +++ b/.github/ISSUE_TEMPLATE/thehive4_question.md @@ -13,13 +13,19 @@ Question ### Work Environment -| Question | Answer -|---------------------------|-------------------- -| OS version (server) | Debian, Ubuntu, CentOS, RedHat, ... -| OS version (client) | XP, Seven, 10, Ubuntu, ... -| TheHive version / git hash | 4.x, hash of the commit -| Package Type | RPM, DEB, Docker, Binary, From source -| Browser type & version | If applicable +| Question | Answer | +| -------------------------- | ------------------------------------- | +| OS version (server) | Debian, Ubuntu, CentOS, RedHat, ... | +| OS version (client) | XP, Seven, 10, Ubuntu, ... | +| Virtualized Env. | True / False | +| Dedicated RAM | XX GB | +| vCPU | 4 / 8 / 16 / 32 | +| TheHive version / git hash | 4.x, hash of the commit | +| Package Type | RPM, DEB, Docker, Binary, From source | +| Database | Cassandra / BerlkelyDB | +| Index type | Lucene / Elasticsearch | +| Attachments storage | Local, NFS, S3, HDFS | +| Browser type & version | If applicable | ### Question diff --git a/ScalliGraph b/ScalliGraph index 97ecfaf7e9..ae7ceb013c 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit 97ecfaf7e9d0f29afa5b12d2f831ca85b7504d8f +Subproject commit ae7ceb013c923295f3ac95f7e2514b8e19218355 From 907e2d6425d41c15c10c148cf044d9257f32a4c7 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Thu, 15 Apr 2021 12:06:05 +0200 Subject: [PATCH 12/17] #1911 Update alert counter on alert create/update/delete --- frontend/app/scripts/controllers/RootCtrl.js | 1 - .../controllers/alert/AlertListCtrl.js | 7 +- .../app/views/partials/case/case.alerts.html | 96 ++++++++++--------- .../org/thp/thehive/services/AuditSrv.scala | 2 + 4 files changed, 54 insertions(+), 52 deletions(-) diff --git a/frontend/app/scripts/controllers/RootCtrl.js b/frontend/app/scripts/controllers/RootCtrl.js index 6abb174e2a..f7a82c3edc 100644 --- a/frontend/app/scripts/controllers/RootCtrl.js +++ b/frontend/app/scripts/controllers/RootCtrl.js @@ -106,7 +106,6 @@ angular.module('theHiveControllers').controller('RootCtrl', name: 'unread-alert-count' } }, - guard: UtilsSrv.hasAddDeleteEvents, onUpdate: function (data) { $scope.unreadAlertCount = data; } diff --git a/frontend/app/scripts/controllers/alert/AlertListCtrl.js b/frontend/app/scripts/controllers/alert/AlertListCtrl.js index c53e33f3da..ef698f5404 100755 --- a/frontend/app/scripts/controllers/alert/AlertListCtrl.js +++ b/frontend/app/scripts/controllers/alert/AlertListCtrl.js @@ -262,11 +262,10 @@ self.menu.markAsUnread = temp.length === 1 && temp[0] === true; // TODO nadouani: don't rely on alert status - self.menu.createNewCase = temp.indexOf('Imported') === -1; - self.menu.mergeInCase = temp.indexOf('Imported') === -1; - - temp = _.without(_.uniq(_.pluck(self.selection, 'case')), null, undefined); + temp = _.without(_.uniq(_.pluck(self.selection, 'caseId')), null, undefined); + self.menu.createNewCase = temp.length === 0; + self.menu.mergeInCase = temp.length === 0; self.menu.delete = temp.length === 0; }; diff --git a/frontend/app/views/partials/case/case.alerts.html b/frontend/app/views/partials/case/case.alerts.html index 590976743b..2da03ea95b 100644 --- a/frontend/app/views/partials/case/case.alerts.html +++ b/frontend/app/views/partials/case/case.alerts.html @@ -4,21 +4,19 @@

-
+
- All ({{alerts.length || 0}}) Type: - {{statsItem.key}} ({{statsItem.value}}) Source: - {{statsItem.key}} ({{statsItem.value}})
@@ -33,55 +31,57 @@ - - Reference - - - - + + Reference + + + + - - Type - - - - + + Type + + + + - - Title - - - - + + Title + + + + - - Source - - - - + + Source + + + + - - Severity - - - - + + Severity + + + + - Attributes + Attributes - - Date - - - - + + Date + + + + @@ -112,7 +112,7 @@
- {{::event.artifacts.length || 0}} + {{::event.observableCount || 0}} {{event.date | shortDate}} @@ -125,14 +125,16 @@
None - {{tag}} + {{tag}}
- + + diff --git a/thehive/app/org/thp/thehive/services/AuditSrv.scala b/thehive/app/org/thp/thehive/services/AuditSrv.scala index 39d89adc80..3508249bd7 100644 --- a/thehive/app/org/thp/thehive/services/AuditSrv.scala +++ b/thehive/app/org/thp/thehive/services/AuditSrv.scala @@ -286,6 +286,7 @@ class AuditSrv @Inject() ( graph: Graph, authContext: AuthContext ): Try[Unit] = { + auditSrv.create(Audit(Audit.update, alert, Some(Json.obj("caseId" -> `case`._id).toString)), `case`, Some(`case`)) val detailsWithAlert = details + ("fromAlert" -> Json.obj( "_id" -> alert._id.toString, "type" -> alert.`type`, @@ -299,6 +300,7 @@ class AuditSrv @Inject() ( graph: Graph, authContext: AuthContext ): Try[Unit] = { + auditSrv.create(Audit(Audit.update, alert, Some(Json.obj("caseId" -> `case`._id).toString)), `case`, Some(`case`)) val detailsWithAlert = details + ("fromAlert" -> Json.obj( "_id" -> alert._id.toString, "type" -> alert.`type`, From 95dbdfed250c11e27e8b9a1ab1f6e914bf00cfde Mon Sep 17 00:00:00 2001 From: To-om Date: Thu, 15 Apr 2021 14:58:07 +0200 Subject: [PATCH 13/17] #1972 Fix data duplication check --- ScalliGraph | 2 +- .../org/thp/thehive/models/Observable.scala | 2 +- .../models/TheHiveSchemaDefinition.scala | 1 + .../org/thp/thehive/services/DataSrv.scala | 19 +++++++++++++++++++ .../services/IntegrityCheckActor.scala | 6 +++--- 5 files changed, 25 insertions(+), 5 deletions(-) diff --git a/ScalliGraph b/ScalliGraph index ae7ceb013c..1a8a32c679 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit ae7ceb013c923295f3ac95f7e2514b8e19218355 +Subproject commit 1a8a32c679a02dc57b59a46cf41c88525f1546c6 diff --git a/thehive/app/org/thp/thehive/models/Observable.scala b/thehive/app/org/thp/thehive/models/Observable.scala index 158c1de33e..b0a92a85e8 100644 --- a/thehive/app/org/thp/thehive/models/Observable.scala +++ b/thehive/app/org/thp/thehive/models/Observable.scala @@ -66,6 +66,6 @@ case class RichObservable( def tags: Seq[String] = observable.tags } -@DefineIndex(IndexType.unique, "data") +@DefineIndex(IndexType.standard, "data") @BuildVertexEntity case class Data(data: String) diff --git a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala index a910840f40..8b5ce6c2e4 100644 --- a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala +++ b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala @@ -485,6 +485,7 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { .iterate() Success(()) } + .removeIndex("Data", IndexType.unique, "data") val reflectionClasses = new Reflections( new ConfigurationBuilder() diff --git a/thehive/app/org/thp/thehive/services/DataSrv.scala b/thehive/app/org/thp/thehive/services/DataSrv.scala index f40ce08c81..957ae3209e 100644 --- a/thehive/app/org/thp/thehive/services/DataSrv.scala +++ b/thehive/app/org/thp/thehive/services/DataSrv.scala @@ -3,6 +3,7 @@ package org.thp.thehive.services import akka.actor.ActorRef import org.apache.tinkerpop.gremlin.process.traversal.P import org.apache.tinkerpop.gremlin.structure.T +import org.thp.scalligraph.EntityId import org.thp.scalligraph.auth.AuthContext import org.thp.scalligraph.models.{Database, Entity} import org.thp.scalligraph.services.{VertexSrv, _} @@ -13,6 +14,7 @@ import org.thp.thehive.services.DataOps._ import java.lang.{Long => JLong} import javax.inject.{Inject, Named, Singleton} +import scala.collection.mutable import scala.util.{Success, Try} @Singleton @@ -58,6 +60,23 @@ object DataOps { } class DataIntegrityCheckOps @Inject() (val db: Database, val service: DataSrv) extends IntegrityCheckOps[Data] { + + override def findDuplicates(): Seq[Seq[Data with Entity]] = + db.roTransaction { implicit graph => + val map = mutable.Map.empty[String, mutable.Buffer[EntityId]] + service + .startTraversal + .foreach { data => + map.getOrElseUpdate(data.data, mutable.Buffer.empty[EntityId]) += EntityId(data._id) + } + map + .values + .collect { + case vertexIds if vertexIds.lengthCompare(1) > 0 => service.getByIds(vertexIds: _*).toList + } + .toSeq + } + override def resolve(entities: Seq[Data with Entity])(implicit graph: Graph): Try[Unit] = entities match { case head :: tail => diff --git a/thehive/app/org/thp/thehive/services/IntegrityCheckActor.scala b/thehive/app/org/thp/thehive/services/IntegrityCheckActor.scala index 1fc88d07c3..4e26cf9682 100644 --- a/thehive/app/org/thp/thehive/services/IntegrityCheckActor.scala +++ b/thehive/app/org/thp/thehive/services/IntegrityCheckActor.scala @@ -125,9 +125,9 @@ class IntegrityCheckActor() extends Actor { Success(integrityCheck.initialCheck()) } } -// integrityCheckOps.foreach { integrityCheck => -// self ! DuplicationCheck(integrityCheck.name) -// } + integrityCheckOps.foreach { integrityCheck => + self ! DuplicationCheck(integrityCheck.name) + } globalTimers = integrityCheckOps.map { integrityCheck => val interval = globalInterval(integrityCheck.name) val initialDelay = FiniteDuration((interval.toNanos * Random.nextDouble()).round, NANOSECONDS) From 0aa353f8c8b901c47bfe4dd64064758ca8e07d54 Mon Sep 17 00:00:00 2001 From: Nabil Adouani Date: Thu, 15 Apr 2021 16:33:16 +0200 Subject: [PATCH 14/17] #1956 Sort case template names in alert import dialg --- .../views/partials/alert/event.dialog.html | 50 +++++++++++-------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/frontend/app/views/partials/alert/event.dialog.html b/frontend/app/views/partials/alert/event.dialog.html index d5e950e1f9..64c2171363 100644 --- a/frontend/app/views/partials/alert/event.dialog.html +++ b/frontend/app/views/partials/alert/event.dialog.html @@ -1,6 +1,7 @@

- Alert Preview {{dialog.event.caseId ? 'Imported' : 'New'}} + Alert Preview {{dialog.event.caseId ? 'Imported' : 'New'}}

@@ -20,7 +21,8 @@

ID: - {{dialog.event._id}} + {{dialog.event._id}} Date: @@ -41,7 +43,8 @@

- None + None
@@ -89,7 +92,8 @@

- +
@@ -116,40 +120,46 @@

- - - - - -
+
- + - +
From 5b0e05bcc1c4f43d2263452a5228e8f6724777d4 Mon Sep 17 00:00:00 2001 From: To-om Date: Thu, 15 Apr 2021 16:37:14 +0200 Subject: [PATCH 15/17] #1973 Don't remove observables/tasks when a share is deleted --- thehive/app/org/thp/thehive/services/ShareSrv.scala | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/thehive/app/org/thp/thehive/services/ShareSrv.scala b/thehive/app/org/thp/thehive/services/ShareSrv.scala index f9cb89a6cd..557c15f873 100644 --- a/thehive/app/org/thp/thehive/services/ShareSrv.scala +++ b/thehive/app/org/thp/thehive/services/ShareSrv.scala @@ -86,15 +86,6 @@ class ShareSrv @Inject() (implicit } yield newShareProfile } - private def delete(shareId: EntityIdOrName)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = - for { - share <- getOrFail(shareId) - tasks = get(share).tasks.toSeq - obs = get(share).observables.toSeq - _ <- tasks.toTry(taskSrv.delete(_)) - _ <- obs.toTry(observableSrv.delete(_)) - } yield get(shareId).remove() - def unshareCase(shareId: EntityIdOrName)(implicit graph: Graph, authContext: AuthContext): Try[Unit] = for { organisation <- get(shareId).organisation.getOrFail("Organisation") @@ -102,8 +93,7 @@ class ShareSrv @Inject() (implicit _ = get(shareId).observables.removeValue(_.organisationIds, organisation._id).barrier().filterNot(_.shares.range(1, 2)).remove() _ = get(shareId).tasks.removeValue(_.organisationIds, organisation._id).barrier().filterNot(_.shares.range(1, 2)).remove() _ <- auditSrv.share.unshareCase(case0, organisation) - _ <- delete(shareId) - } yield () + } yield get(shareId).remove() def unshareTask( task: Task with Entity, From 293771fc1924e51226e6a8d8189f04577c12bf65 Mon Sep 17 00:00:00 2001 From: To-om Date: Thu, 15 Apr 2021 16:59:40 +0200 Subject: [PATCH 16/17] #1974 Fix tags field update --- thehive/app/org/thp/thehive/services/ObservableSrv.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/thehive/app/org/thp/thehive/services/ObservableSrv.scala b/thehive/app/org/thp/thehive/services/ObservableSrv.scala index a2e9f0d8cd..915eec1471 100644 --- a/thehive/app/org/thp/thehive/services/ObservableSrv.scala +++ b/thehive/app/org/thp/thehive/services/ObservableSrv.scala @@ -114,9 +114,11 @@ class ObservableSrv @Inject() ( .toSeq .map(_.toString) .toSet + val newTags = tags -- currentTags for { - createdTags <- (tags -- currentTags).filterNot(_.isEmpty).toTry(tagSrv.getOrCreate) + createdTags <- newTags.filterNot(_.isEmpty).toTry(tagSrv.getOrCreate) _ <- createdTags.toTry(observableTagSrv.create(ObservableTag(), observable, _)) + _ <- get(observable).update(_.tags, (currentTags ++ newTags).toSeq).getOrFail("Observable") // _ <- auditSrv.observable.update(observable, Json.obj("tags" -> (currentTags ++ tags))) } yield createdTags } From 7935ff5e14811b727c27f94d28ac1e3aad984b1c Mon Sep 17 00:00:00 2001 From: To-om Date: Thu, 15 Apr 2021 15:20:52 +0000 Subject: [PATCH 17/17] Release 4.1.4 --- CHANGELOG.md | 19 +++++++++++++++++++ build.sbt | 2 +- frontend/bower.json | 2 +- frontend/package.json | 2 +- 4 files changed, 22 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb8360bf07..03615d5be2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,24 @@ # Change Log +## [4.1.4](https://github.com/TheHive-Project/TheHive/milestone/73) (2021-04-15) + +**Implemented enhancements:** + +- [Feature Request] Sort case templates in alert Import drop down menu [\#1956](https://github.com/TheHive-Project/TheHive/issues/1956) +- [Enhancement] Make alert deletion more robust [\#1967](https://github.com/TheHive-Project/TheHive/issues/1967) + +**Fixed bugs:** + +- [Bug] Alert counter refresh not working [\#1911](https://github.com/TheHive-Project/TheHive/issues/1911) +- [Bug] Enabled or Disabled Taxonomies doesn't work [\#1957](https://github.com/TheHive-Project/TheHive/issues/1957) +- [Bug] TheHive 4.1.3-1 Task in Case is not visible [\#1964](https://github.com/TheHive-Project/TheHive/issues/1964) +- [Bug] Aggregation on custom fields provides incorect result (again) [\#1965](https://github.com/TheHive-Project/TheHive/issues/1965) +- [Bug] TheHive startup times out if schema evolution is long [\#1966](https://github.com/TheHive-Project/TheHive/issues/1966) +- [Bug] Default analyst rol cant add TTPs [\#1968](https://github.com/TheHive-Project/TheHive/issues/1968) +- [Bug] Links section should not be empty in Observables details view [\#1972](https://github.com/TheHive-Project/TheHive/issues/1972) +- [Bug] Deleting a shared rule case for org2 - deleting observables (sharing to org2) from the org1. [\#1973](https://github.com/TheHive-Project/TheHive/issues/1973) +- [Bug] AddTagToArtifact operation not working in 4.1.3 [\#1974](https://github.com/TheHive-Project/TheHive/issues/1974) + ## [4.1.3](https://github.com/TheHive-Project/TheHive/milestone/72) (2021-04-12) **Implemented enhancements:** diff --git a/build.sbt b/build.sbt index cf1d470605..97bf2e5b5b 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ import Dependencies._ import com.typesafe.sbt.packager.Keys.bashScriptDefines import org.thp.ghcl.Milestone -val thehiveVersion = "4.1.3-1" +val thehiveVersion = "4.1.4-1" val scala212 = "2.12.13" val scala213 = "2.13.1" val supportedScalaVersions = List(scala212, scala213) diff --git a/frontend/bower.json b/frontend/bower.json index 8e590b49d3..3725157055 100644 --- a/frontend/bower.json +++ b/frontend/bower.json @@ -1,6 +1,6 @@ { "name": "thehive", - "version": "4.1.3-1", + "version": "4.1.4-1", "license": "AGPL-3.0", "dependencies": { "jquery": "^3.4.1", diff --git a/frontend/package.json b/frontend/package.json index 56d225c3b1..652c7464c0 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "thehive", - "version": "4.1.3-1", + "version": "4.1.4-1", "license": "AGPL-3.0", "repository": { "type": "git",