difference rendering

docker-compose-dependecies.yml → docker-compose-dependencies.yml

@@ -63,7 +63,6 @@ services:
       - ./etc/promtail-local.yaml:/etc/promtail/promtail-local.yaml
       - ./data/logs:/app/logs
     depends_on:
-      - diffy
       - loki
 
   prometheus:
@@ -75,8 +74,6 @@ services:
       - --config.file=/etc/prometheus.yaml
     ports:
       - "9090:9090"
-    depends_on:
-      - diffy
 
   grafana:
     image: grafana/grafana:7.4.0-ubuntu

pom.xml

@@ -48,6 +48,13 @@
     </properties>
 
     <dependencies>
+        <dependency>
+            <groupId>org.projectlombok</groupId>
+            <artifactId>lombok</artifactId>
+            <version>1.18.28</version>
+            <scope>provided</scope>
+        </dependency>
+
         <dependency>
             <groupId>org.scala-lang</groupId>
             <artifactId>scala-library</artifactId>

src/main/scala/ai/diffy/analysis/DynamicAnalyzer.scala

@@ -3,13 +3,26 @@ package ai.diffy.analysis
 import ai.diffy.analysis.DynamicAnalyzer.decodeFieldMap
 import ai.diffy.lifter.{FieldMap, JsonLifter, Message}
 import ai.diffy.repository.DifferenceResultRepository
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.node.ObjectNode
+import scala.jdk.CollectionConverters.MapHasAsScala
 
 /**
  * Filters a DifferenceAnalyzer using a specified time range to output another DifferenceAnalyzer
  */
 object DynamicAnalyzer {
   def decodeFieldMap(payload: String): FieldMap = {
-    JsonLifter.decode(s"{\"value\":$payload}", classOf[FieldMap])
+    objectNodeToFieldMap(JsonLifter.decode(payload).asInstanceOf[ObjectNode])
+  }
+  def objectNodeToFieldMap(objectNode: ObjectNode): FieldMap ={
+    val acc = new java.util.HashMap[String, Object]()
+    objectNode.fields().forEachRemaining(entry => {
+      acc.put(entry.getKey(), entry.getValue())
+    })
+    if(acc.containsKey("headers")) {
+      acc.put("headers", objectNodeToFieldMap(acc.get("headers").asInstanceOf[ObjectNode]))
+    }
+    new FieldMap(acc.asScala.toMap)
   }
 }
 class DynamicAnalyzer(repository: DifferenceResultRepository) {

src/main/scala/ai/diffy/flat/FlatObject.scala

@@ -63,7 +63,7 @@ object FlatObject {
     case byteBuffer: ByteBuffer => lift(new String(byteBuffer.asReadOnlyBuffer().array))
     case jsonNode: JsonNode => lift(JsonLifter.lift(jsonNode))
     case null => FlatNull
-    case fm: FieldMap => lift(fm.value)
+    case fm: FieldMap => FlatStruct(lift(fm.value).asInstanceOf[FlatMap[FlatPrimitive[String], FlatObject]])
     case _ => FlatStruct(FlatMap(mkMap(a) map {case (k,v) => FlatPrimitive(k) -> lift(v)}))
   }
 }

src/main/scala/ai/diffy/interpreter/http/HttpLambdaServer.java

@@ -43,6 +43,9 @@ public HttpLambdaServer(int port, Source httpLambda) {
                 ).bindNow();
     }
 
+    public void shutdown() {
+        server.disposeNow();
+    }
     public static void main(String[] args) throws Exception {
         HttpLambdaServer primary = new HttpLambdaServer(Integer.parseInt(args[0]), new File("src/main/scala/ai/diffy/interpreter/http/master.js"));
         HttpLambdaServer secondary = new HttpLambdaServer(Integer.parseInt(args[1]), new File("src/main/scala/ai/diffy/interpreter/http/master.js"));

src/main/scala/ai/diffy/lifter/JsonLifter.scala

@@ -62,7 +62,7 @@ object JsonLifter {
   def encode(item: Any): String = Mapper.writer.writeValueAsString(item)
 
   def areMapInsteadofObjectKeys(fields: Set[String]): Boolean =
-    fields.size > 20 || fields.exists{ field =>
+    fields.size > 50 || fields.exists{ field =>
       field.length > 100 ||
       field.matches("[0-9].*") || // starts with a digit
       !field.matches("[_a-zA-Z0-9]*") // contains non-alphanumeric characters

src/main/scala/ai/diffy/proxy/ReactorHttpDifferenceProxy.java

@@ -6,6 +6,7 @@
 import ai.diffy.functional.endpoints.SeptaDependentEndpoint;
 import ai.diffy.functional.topology.Async;
 import ai.diffy.functional.topology.ControlFlowLogger;
+import ai.diffy.functional.topology.InvocationLogger;
 import ai.diffy.lifter.AnalysisRequest;
 import ai.diffy.lifter.HttpLifter;
 import ai.diffy.repository.DifferenceResultRepository;

src/test/resources/application.yml

@@ -0,0 +1,34 @@
+spring.application.name: "diffy"
+server:
+  port: ${http.port:8888}
+
+proxy:
+  port: 8880
+
+candidate: "localhost:9000"
+
+master:
+  primary: "localhost:9100"
+  secondary: "localhost:9200"
+
+service:
+  protocol : "http"
+
+serviceName : "Default Sample Service"
+
+allowHttpSideEffects: true
+
+spring:
+  mongodb:
+    embedded:
+      version: "5.0.6"
+management:
+  endpoints:
+    web:
+      exposure:
+        include: health,info,prometheus
+
+logging:
+  pattern:
+    console: "%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(%5p) %clr([diffy,%X{trace_id},%X{span_id}]) %clr(${PID:- }){magenta} %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m %n%wEx"
+    file: "%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(%5p) %clr([diffy,%X{trace_id},%X{span_id}]) %clr(${PID:- }){magenta} %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m %n%wEx"

src/test/resources/lambda.js

@@ -0,0 +1,89 @@
+(request) => {
+    const {
+        uri,
+        method,
+        path,
+        params,
+        headers,
+        body
+    } = request;
+
+    const response = {
+        status: '200 OK',
+        headers: {},
+        body: {
+                "message": "Success",
+                "response": {
+                  "id": 98758734857,
+                  "app": 12,
+                  "resource": {
+                    "id": 150,
+                    "app": null,
+                    "label": "Test label",
+                    "hfu": {
+                      "id": 898,
+                      "app": null,
+                      "createdDate": null,
+                      "lastModifiedDate": null,
+                      "name": "test_name",
+                      "uuid": null
+                    },
+                    "custom": false,
+                    "delhuf": false
+                  },
+                  "title": "Test title from candidate server",
+                  "abcd": "text",
+                  "related": [
+                    "ioi"
+                  ],
+                  "data": null,
+                  "flabel": "Test",
+                  "tcon": null,
+                  "type": "UYU",
+                  "gb": [],
+                  "meas": null,
+                  "audy": null,
+                  "stpe": null,
+                  "slim": 0,
+                  "zoomInData": null,
+                  "improvement": null,
+                  "isdterange": true,
+                  "meta": null,
+                  "intcof": null,
+                  "metqu": null,
+                  "fif": {
+                    "id": null,
+                    "requestResource": null,
+                    "resource": null,
+                    "updatable": false,
+                    "empty": true
+                  },
+                  "modified": false,
+                  "description": null,
+                  "initdat": null,
+                  "applyModifiedData": false,
+                  "sortConfiguration": null,
+                  "pasfor": null,
+                  "originalTitle": "Test original title",
+                  "uuid": "345345346",
+                  "cofn": null,
+                  "rol": null,
+                  "accessible": true,
+                  "hasDel": false,
+                  "witype": null,
+                  "empwi": false,
+                  "syd": null,
+                  "asgr": [],
+                  "ismod": false,
+                  "ughuygegg": false,
+                  "deleted": false,
+                  "newef": false,
+                  "efor": null
+                },
+                "path": null
+              }
+    }
+
+    response.body = body;
+    return response;
+}

src/test/resources/payload.json

@@ -0,0 +1,71 @@
+{
+  "message": "Success",
+  "response": {
+    "id": 98758734857,
+    "app": 12,
+    "resource": {
+      "id": 150,
+      "app": null,
+      "label": "Test label",
+      "hfu": {
+        "id": 898,
+        "app": null,
+        "createdDate": null,
+        "lastModifiedDate": null,
+        "name": "test_name",
+        "uuid": null
+      },
+      "custom": false,
+      "delhuf": false
+    },
+    "title": "Test title from candidate server",
+    "abcd": "text",
+    "related": [
+      "ioi"
+    ],
+    "data": null,
+    "flabel": "Test",
+    "tcon": null,
+    "type": "UYU",
+    "gb": [],
+    "meas": null,
+    "audy": null,
+    "stpe": null,
+    "slim": 0,
+    "zoomInData": null,
+    "improvement": null,
+    "isdterange": true,
+    "meta": null,
+    "intcof": null,
+    "metqu": null,
+    "fif": {
+      "id": null,
+      "requestResource": null,
+      "resource": null,
+      "updatable": false,
+      "empty": true
+    },
+    "modified": false,
+    "description": null,
+    "initdat": null,
+    "applyModifiedData": false,
+    "sortConfiguration": null,
+    "pasfor": null,
+    "originalTitle": "Test original title",
+    "uuid": "345345346",
+    "cofn": null,
+    "rol": null,
+    "accessible": true,
+    "hasDel": false,
+    "witype": null,
+    "empwi": false,
+    "syd": null,
+    "asgr": [],
+    "ismod": false,
+    "ughuygegg": false,
+    "deleted": false,
+    "newef": false,
+    "efor": null
+  },
+  "path": null
+}

src/test/scala/ai/diffy/IntegrationTest.java

@@ -0,0 +1,62 @@
+package ai.diffy;
+
+import ai.diffy.interpreter.http.HttpLambdaServer;
+import ai.diffy.proxy.ReactorHttpDifferenceProxy;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInstance;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.core.io.FileSystemResource;
+import org.springframework.http.HttpEntity;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.MediaType;
+import org.springframework.util.FileCopyUtils;
+import org.springframework.web.client.RestTemplate;
+import static org.junit.jupiter.api.Assertions.*;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStreamReader;
+
+@SpringBootTest
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+public class IntegrationTest {
+    RestTemplate restTemplate = new RestTemplate();
+    @Autowired
+    Settings settings;
+
+    @Autowired
+    ReactorHttpDifferenceProxy proxy;
+    private Integer extractPort(Downstream downstream) {
+        HostPort hostPort = (HostPort) downstream;
+        return hostPort.port();
+    }
+    HttpLambdaServer primary, secondary, candidate;
+    @BeforeAll
+    public void setup() throws IOException {
+        primary = new HttpLambdaServer(extractPort(settings.primary()), new File("src/test/resources/lambda.js"));
+        secondary = new HttpLambdaServer(extractPort(settings.secondary()), new File("src/test/resources/lambda.js"));
+        candidate = new HttpLambdaServer(extractPort(settings.candidate()), new File("src/test/resources/lambda.js"));
+    }
+
+    @AfterAll
+    public void shutdown() {
+        primary.shutdown();
+        secondary.shutdown();
+        candidate.shutdown();
+    }
+
+    @Test
+    public void warmup() throws Exception {
+        String proxyUrl = "http://localhost:"+settings.servicePort()+"/base";
+        HttpHeaders headers = new HttpHeaders();
+        headers.setContentType(MediaType.APPLICATION_JSON);
+
+        FileSystemResource payload = new FileSystemResource("src/test/resources/payload.json");
+        String json = FileCopyUtils.copyToString(new InputStreamReader(payload.getInputStream()));
+        String response = restTemplate.postForObject(proxyUrl, new HttpEntity<String>(json, headers), String.class);
+        assertEquals(json, response);
+    }
+}

src/test/scala/ai/diffy/compare/DifferenceTest.java

@@ -0,0 +1,25 @@
+package ai.diffy.compare;
+
+import lombok.extern.slf4j.Slf4j;
+import org.junit.jupiter.api.Test;
+import org.springframework.core.io.FileSystemResource;
+import org.springframework.util.FileCopyUtils;
+
+import java.io.IOException;
+import java.io.InputStreamReader;
+
+@Slf4j
+public class DifferenceTest {
+
+    @Test
+    public void payloadTest() throws IOException {
+        FileSystemResource payload = new FileSystemResource("src/test/resources/payload.json");
+        String json = FileCopyUtils.copyToString(new InputStreamReader(payload.getInputStream()));
+        log.info(json);
+        Difference diff = Difference.apply(json, json);
+        diff.flattened().foreach(tuple -> {
+            log.info("{} - {}", tuple._1(), tuple._2());
+            return null;
+        });
+    }
+}