TheHive-Project#2384 Add support of operations in analyzer reports

To-om · To-om · commit 5ad9b0593bf7 · 2022-05-11T11:03:17.000+02:00
diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/Conversion.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/Conversion.scala
@@ -47,6 +47,7 @@ object Conversion {
       .withFieldComputed(_.id, _._id.toString)
       .withFieldConst(_._type, "case_artifact_job")
       .withFieldConst(_.case_artifact, None)
+      .withFieldComputed(_.operations, a => JsArray(a.operations).toString)
       .enableMethodAccessors
       .transform
   )
@@ -80,6 +81,7 @@ object Conversion {
               Some(observableWithExtraOutput.toValue((richObservable, JsObject.empty, Some(Left(richCase)))))
           }
         )
+        .withFieldComputed(_.operations, a => JsArray(a.operations).toString)
         .enableMethodAccessors
         .transform
     }
diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/models/Job.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/models/Job.scala
@@ -29,7 +29,8 @@ case class Job(
     endDate: Date, // end date of the job or if it is not finished date of the last check
     report: Option[JsObject],
     cortexId: String,
-    cortexJobId: String
+    cortexJobId: String,
+    operations: Seq[JsObject]
 )
 
 case class RichJob(
@@ -50,5 +51,5 @@ case class RichJob(
   def report: Option[JsObject]   = job.report
   def cortexId: String           = job.cortexId
   def cortexJobId: String        = job.cortexJobId
-
+  def operations: Seq[JsObject]  = job.operations
 }
diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ActionSrv.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ActionSrv.scala
@@ -96,7 +96,7 @@ class ActionSrv @Inject() (
         job.report.flatMap(_.full),
         client.name,
         job.id,
-        job.report.fold[Seq[JsObject]](Nil)(_.operations)
+        Nil
       )
       createdAction <- Future.fromTry {
         db.tryTransaction { implicit graph =>
diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/JobSrv.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/JobSrv.scala
@@ -42,6 +42,7 @@ class JobSrv @Inject() (
     observableTypeSrv: ObservableTypeSrv,
     attachmentSrv: AttachmentSrv,
     reportTagSrv: ReportTagSrv,
+    actionOperationSrv: ActionOperationSrv,
     serviceHelper: ServiceHelper,
     auditSrv: CortexAuditSrv,
     organisationSrv: OrganisationSrv,
@@ -130,6 +131,7 @@ class JobSrv @Inject() (
       .withFieldConst(_.report, None)
       .withFieldConst(_.cortexId, "tbd")
       .withFieldComputed(_.cortexJobId, _.id)
+      .withFieldConst(_.operations, Nil)
       .transform
 
   /**
@@ -167,11 +169,31 @@ class JobSrv @Inject() (
           .availableCortexClients(connector.clients, authContext.organisation)
           .find(_.name == cortexId)
           .fold[Future[CortexClient]](Future.failed(NotFoundError(s"Cortex $cortexId not found")))(Future.successful)
-      job <- Future.fromTry(updateJobStatus(jobId, cortexJob))
-      _   <- importCortexArtifacts(job, cortexJob, cortexClient)
-      _   <- Future.fromTry(importAnalyzerTags(job, cortexJob))
+      operations <- Future.fromTry(executeOperations(jobId, cortexJob))
+      job        <- Future.fromTry(updateJobStatus(jobId, cortexJob, operations))
+      _          <- importCortexArtifacts(job, cortexJob, cortexClient)
+      _          <- Future.fromTry(importAnalyzerTags(job, cortexJob))
     } yield job
 
+  def executeOperations(jobId: EntityId, cortexJob: CortexJob)(implicit authContext: AuthContext): Try[Seq[ActionOperationStatus]] =
+    db.tryTransaction { implicit graph =>
+      get(jobId)
+        .observable
+        .project(_.by.by(_.`case`.option))
+        .getOrFail("Observable")
+        .map {
+          case (relatedObservable, relatedCase) =>
+            cortexJob
+              .report
+              .fold[Seq[ActionOperation]](Nil)(_.operations.map(_.as[ActionOperation]))
+              .map { operation =>
+                actionOperationSrv
+                  .execute(relatedObservable, operation, relatedCase, None)
+                  .fold(t => ActionOperationStatus(operation, success = false, t.getMessage), identity)
+              }
+        }
+    }
+
   /**
     * Update job status, set the endDate and remove artifacts from report
     *
@@ -180,7 +202,9 @@ class JobSrv @Inject() (
     * @param authContext the authentication context
     * @return the updated job
     */
-  private def updateJobStatus(jobId: EntityId, cortexJob: CortexJob)(implicit authContext: AuthContext): Try[Job with Entity] =
+  private def updateJobStatus(jobId: EntityId, cortexJob: CortexJob, operations: Seq[ActionOperationStatus])(implicit
+      authContext: AuthContext
+  ): Try[Job with Entity] =
     db.tryTransaction { implicit graph =>
       getOrFail(jobId).flatMap { job =>
         val report  = cortexJob.report.flatMap(r => r.full orElse r.errorMessage.map(m => Json.obj("errorMessage" -> m)))
@@ -193,6 +217,7 @@ class JobSrv @Inject() (
             .update(_.endDate, endDate)
             .update(_._updatedAt, Some(new Date))
             .update(_._updatedBy, Some(authContext.userId))
+            .update(_.operations, operations.map(o => Json.toJsObject(o)))
             .getOrFail("Job")
           observable <- get(job).observable.getOrFail("Observable")
           _ <-
diff --git a/cortex/connector/src/test/resources/cortex-jobs.json b/cortex/connector/src/test/resources/cortex-jobs.json
@@ -87,12 +87,21 @@
         {
           "data": "192.168.1.1",
           "message": "myIp",
-          "tags": [],
+          "tags": ["tag-test"],
           "tlp": 2,
           "dataType": "ip"
         }
       ],
       "operations": [
+        {
+          "type": "AddArtifactToCase",
+          "data": "myData",
+          "dataType": "other",
+          "message": "test-operation",
+          "tlp": 3,
+          "ignoreSimilarity": false,
+          "tags": ["tag1", "tag2"]
+        }
       ]
     },
     "tlp": 2,
diff --git a/cortex/connector/src/test/scala/org/thp/thehive/connector/cortex/services/JobSrvTest.scala b/cortex/connector/src/test/scala/org/thp/thehive/connector/cortex/services/JobSrvTest.scala
@@ -7,9 +7,10 @@ import org.thp.scalligraph.models.{Database, DummyUserSrv, Schema}
 import org.thp.scalligraph.traversal.TraversalOps._
 import org.thp.scalligraph.{AppBuilder, EntityName}
 import org.thp.thehive.TestAppBuilder
-import org.thp.thehive.connector.cortex.models.{Job, JobStatus, TheHiveCortexSchemaProvider}
+import org.thp.thehive.connector.cortex.models.{ActionOperationStatus, Job, JobStatus, TheHiveCortexSchemaProvider}
 import org.thp.thehive.connector.cortex.services.JobOps._
 import org.thp.thehive.models.Permissions
+import org.thp.thehive.services.CaseOps._
 import org.thp.thehive.services.ObservableOps._
 import org.thp.thehive.services.UserOps._
 import org.thp.thehive.services._
@@ -23,7 +24,8 @@ import scala.concurrent.duration.DurationInt
 import scala.io.Source
 
 class JobSrvTest extends PlaySpecification with TestAppBuilder {
-  implicit val authContext: AuthContext = DummyUserSrv(userId = "admin@thehive.local", permissions = Permissions.all).authContext
+  implicit val authContext: AuthContext =
+    DummyUserSrv(userId = "certuser@thehive.local", organisation = "cert", permissions = Permissions.all).authContext
   override def appConfigure: AppBuilder =
     super
       .appConfigure
@@ -33,53 +35,76 @@ class JobSrvTest extends PlaySpecification with TestAppBuilder {
       .`override`(_.bindToProvider[Schema, TheHiveCortexSchemaProvider])
 
   "job service" should {
+    val cortexOutputJob = {
+      val dataSource = Source.fromResource("cortex-jobs.json")
+      val data       = dataSource.mkString
+      dataSource.close()
+      Json.parse(data).as[List[OutputJob]].find(_.id == "ZWu85Q1OCVNx03hXK4df").get
+    }
+
     "handle creation and then finished job" in testApp { app =>
-//      val job = Job(
-//        workerId = "anaTest2",
-//        workerName = "anaTest2",
-//        workerDefinition = "test2",
-//        status = JobStatus.Waiting,
-//        startDate = new Date(1561625908856L),
-//        endDate = new Date(1561625908856L),
-//        report = None,
-//        cortexId = "test",
-//        cortexJobId = "LVyYKFstq3Rtrdc9DFmL"
-//      )
-//
-//      val cortexOutputJob = {
-//        val dataSource = Source.fromResource("cortex-jobs.json")
-//        val data       = dataSource.mkString
-//        dataSource.close()
-//        Json.parse(data).as[List[OutputJob]].find(_.id == "ZWu85Q1OCVNx03hXK4df").get
-//      }
-//
-//      val createdJobTry = app[Database].tryTransaction { implicit graph =>
-//        for {
-//          observable <- app[ObservableSrv].startTraversal.has(_.message, "hello world").getOrFail("Observable")
-//          createdJob <- app[JobSrv].create(job, observable)
-//        } yield createdJob
-//      }
-//      createdJobTry.map { createdJob =>
-//        Await.result(app[JobSrv].finished(app[CortexClient].name, createdJob._id, cortexOutputJob), 20.seconds)
-//      } must beASuccessfulTry.which { updatedJob =>
-//        updatedJob.status shouldEqual JobStatus.Success
-//        updatedJob.report must beSome
-//        (updatedJob.report.get \ "data").as[String] shouldEqual "imageedit_2_3904987689.jpg"
-//
-//        app[Database].roTransaction { implicit graph =>
-//          app[JobSrv].get(updatedJob).observable.has(_.message, "hello world").exists must beTrue
-//          app[JobSrv].get(updatedJob).reportObservables.toList.length must equalTo(2).updateMessage { s =>
-//            s"$s\nreport observables are : ${app[JobSrv].get(updatedJob).reportObservables.richObservable.toList.mkString("\n")}"
-//          }
-//
-//          for {
-//            audit        <- app[AuditSrv].startTraversal.has(_.objectId, updatedJob._id.toString).getOrFail("Audit")
-//            organisation <- app[OrganisationSrv].getByName("cert").getOrFail("Organisation")
-//            user         <- app[UserSrv].startTraversal.getByName("certuser@thehive.local").getOrFail("User")
-//          } yield new JobFinished().filter(audit, Some(updatedJob), organisation, Some(user))
-//        } must beASuccessfulTry(true)
-//      }
-      pending("flaky test")
+      val job = Job(
+        workerId = "anaTest2",
+        workerName = "anaTest2",
+        workerDefinition = "test2",
+        status = JobStatus.Waiting,
+        startDate = new Date(1561625908856L),
+        endDate = new Date(1561625908856L),
+        report = None,
+        cortexId = "test",
+        cortexJobId = "LVyYKFstq3Rtrdc9DFmL",
+        operations = Nil
+      )
+
+      val createdJobTry = app[Database].tryTransaction { implicit graph =>
+        for {
+          observable <- app[ObservableSrv].startTraversal.has(_.message, "hello world").getOrFail("Observable")
+          createdJob <- app[JobSrv].create(job, observable)
+        } yield createdJob
+      }
+      val finishedJobTry = createdJobTry.map { createdJob =>
+        Await.result(app[JobSrv].finished(app[CortexClient].name, createdJob._id, cortexOutputJob), 20.seconds)
+      }
+      finishedJobTry must beASuccessfulTry
+      val updatedJob = finishedJobTry.get
+      updatedJob.status shouldEqual JobStatus.Success
+      updatedJob.report must beSome
+      (updatedJob.report.get \ "data").as[String] shouldEqual "imageedit_2_3904987689.jpg"
+      updatedJob.operations must haveSize(1)
+      updatedJob.operations.map(o => (o \ "status").as[String]) must contain(beEqualTo("Success"))
+        .forall
+        .updateMessage(s => s"$s\nOperation has failed: ${updatedJob.operations.map("\n -" + _).mkString}")
+
+      app[Database].roTransaction { implicit graph =>
+        app[JobSrv].get(updatedJob).observable.has(_.message, "hello world").exists must beTrue
+        val reportObservables = app[JobSrv].get(updatedJob).reportObservables.toSeq
+        reportObservables.length must equalTo(2).updateMessage { s =>
+          s"$s\nreport observables are : ${app[JobSrv].get(updatedJob).reportObservables.richObservable.toList.mkString("\n")}"
+        }
+        val ipObservable = reportObservables.find(_.dataType == "ip").get
+        ipObservable.data    must beSome("192.168.1.1")
+        ipObservable.message must beSome("myIp")
+        ipObservable.tags    must contain(exactly("tag-test"))
+        ipObservable.tlp     must beEqualTo(2)
+
+        val operationObservableMaybe = app[JobSrv]
+          .get(updatedJob)
+          .observable
+          .`case`
+          .observables
+          .has(_.message, "test-operation")
+          .headOption
+        operationObservableMaybe must beSome.which { operationObservable =>
+          operationObservable.data must beSome("myData")
+          operationObservable.tlp  must beEqualTo(3)
+          operationObservable.tags must contain(exactly("tag1", "tag2"))
+        }
+        for {
+          audit        <- app[AuditSrv].startTraversal.has(_.objectId, updatedJob._id.toString).getOrFail("Audit")
+          organisation <- app[OrganisationSrv].getByName("cert").getOrFail("Organisation")
+          user         <- app[UserSrv].startTraversal.getByName("certadmin@thehive.local").getOrFail("User")
+        } yield JobFinished.filter(audit, Some(updatedJob), organisation, Some(user))
+      } must beASuccessfulTry(true).setMessage("The audit doesn't match the expected criteria")
     }
 
     "submit a job" in testApp { app =>
diff --git a/dto/src/main/scala/org/thp/thehive/connector/cortex/dto/v0/Job.scala b/dto/src/main/scala/org/thp/thehive/connector/cortex/dto/v0/Job.scala
@@ -17,7 +17,8 @@ case class OutputJob(
     cortexId: String,
     cortexJobId: String,
     id: String,
-    case_artifact: Option[OutputObservable]
+    case_artifact: Option[OutputObservable],
+    operations: String
 )
 
 object OutputJob {
diff --git a/migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala b/migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala
@@ -479,7 +479,8 @@ trait Conversion {
         endDate,
         report,
         cortexId,
-        cortexJobId
+        cortexJobId,
+        Nil
       )
     )
   }

Original file line number	Diff line number	Diff line change
`@@ -47,6 +47,7 @@ object Conversion {`
`47`	`47`	`.withFieldComputed(_.id, _._id.toString)`
`48`	`48`	`.withFieldConst(_._type, "case_artifact_job")`
`49`	`49`	`.withFieldConst(_.case_artifact, None)`
	`50`	`+ .withFieldComputed(_.operations, a => JsArray(a.operations).toString)`
`50`	`51`	`.enableMethodAccessors`
`51`	`52`	`.transform`
`52`	`53`	`)`
`@@ -80,6 +81,7 @@ object Conversion {`
`80`	`81`	`Some(observableWithExtraOutput.toValue((richObservable, JsObject.empty, Some(Left(richCase)))))`
`81`	`82`	`}`
`82`	`83`	`)`
	`84`	`+ .withFieldComputed(_.operations, a => JsArray(a.operations).toString)`
`83`	`85`	`.enableMethodAccessors`
`84`	`86`	`.transform`
`85`	`87`	`}`
Original file line number	Diff line number	Diff line change
`@@ -96,7 +96,7 @@ class ActionSrv @Inject() (`
`96`	`96`	`job.report.flatMap(_.full),`
`97`	`97`	`client.name,`
`98`	`98`	`job.id,`
`99`		`- job.report.fold[Seq[JsObject]](Nil)(_.operations)`
	`99`	`+ Nil`
`100`	`100`	`)`
`101`	`101`	`createdAction <- Future.fromTry {`
`102`	`102`	`db.tryTransaction { implicit graph =>`
Original file line number	Diff line number	Diff line change
`@@ -17,7 +17,8 @@ case class OutputJob(`
`17`	`17`	`cortexId: String,`
`18`	`18`	`cortexJobId: String,`
`19`	`19`	`id: String,`
`20`		`- case_artifact: Option[OutputObservable]`
	`20`	`+ case_artifact: Option[OutputObservable],`
	`21`	`+ operations: String`
`21`	`22`	`)`
`22`	`23`
`23`	`24`	`object OutputJob {`
Original file line number	Diff line number	Diff line change
`@@ -479,7 +479,8 @@ trait Conversion {`
`479`	`479`	`endDate,`
`480`	`480`	`report,`
`481`	`481`	`cortexId,`
`482`		`- cortexJobId`
	`482`	`+ cortexJobId,`
	`483`	`+ Nil`
`483`	`484`	`)`
`484`	`485`	`)`
`485`	`486`	`}`