Merge branch 'main' into jacek-new-reference

.github/workflows/pipeline.yml

@@ -148,6 +148,8 @@ jobs:
 
       - name: Verify if data is flowing
         working-directory: smoke-test
+        env:
+          GITHUB_ACTIONS: true
         run: go run main.go
 
       - name: Print docker status

.gitignore

@@ -16,7 +16,6 @@ heap.out
 **/terraform.tfstate*
 **/.terraform.tfstate*
 **/.terraform
-docker/ngrok/ngrok.yml
 docker/security/ca
 docker/security/es.local
 docker/security/certificate-bundle.zip

docker/README.md

@@ -13,5 +13,4 @@ or our own services like log-generator.
 * `kafka-demo.yml` - created specifically for Device demo, contains all services and data, including Kafka, which writes to Quesma via Elasticsearch Connector.
 * `opensearch.yml` - used for local development with OpenSearch instead of Elasticsearch. Work in progress.
 * `hydrolix.yml` - to be used with Hydrolix, requires `.env` file from 1Password.
-* `ngrok.yml` - used to expose our service to the internet, requires `ngrok/ngrok.yml` file from 1Password
 

docker/ci.yml

@@ -9,6 +9,7 @@ services:
       - QUESMA_port=8080
       - QUESMA_logging_path=/var/quesma/logs
       - QUESMA_clickhouse_url=clickhouse://clickhouse:9000
+      - QUESMA_logging_fileLogging=true
     depends_on:
       clickhouse:
         condition: service_healthy

docker/clean-clickhouse/schema/02-windows_logs.sql

@@ -0,0 +1,39 @@
+CREATE TABLE IF NOT EXISTS "windows_logs"
+(
+    "attributes_string_key" Array(String),
+    "attributes_string_value" Array(String),
+
+    "@timestamp" DateTime64 DEFAULT now64(),
+
+    "event::category" Nullable(String),
+    "event::type" Nullable(String),
+
+    "dll::name" Nullable(String),
+    "dll::path" Nullable(String),
+
+    "registry::path" Nullable(String),
+    "registry::value" Nullable(String),
+    "registry::key" Nullable(String),
+
+    "destination::address" Nullable(String),
+    "destination::port" Nullable(String),
+
+    "network::protocol" Nullable(String),
+    "network::direction" Nullable(String),
+
+    "source::address" Nullable(String),
+    "source::port" Nullable(String),
+
+    "process::pid" Nullable(Int64),
+    "process::entity_id" Nullable(String),
+    "process::executable" Nullable(String),
+    "process::name" Nullable(String),
+
+    "user::id" Nullable(String),
+    "user::domain" Nullable(String),
+    "user::full_name" Nullable(String),
+
+)
+ENGINE = MergeTree
+ORDER BY ("@timestamp")
+COMMENT 'Windows Security Logs. Created by clean-clickhouse.'

docker/deploy/quesma-all-in-one-vm.tf

@@ -55,6 +55,11 @@ resource "google_compute_instance" "vm_instance" {
   boot_disk {
     initialize_params {
       image = "debian-cloud/debian-10"
+      size  = 200
+      type = "pd-balanced"
+      labels = {
+        name = "quesma-demo-aio-vm"
+      }
     }
   }
 

docker/local-dev.yml

@@ -13,6 +13,7 @@ services:
       - QUESMA_logging_path=/var/quesma/logs
       - QUESMA_mode=dual-write-query-clickhouse
       - QUESMA_CONFIG_FILE=/config/local-dev.yaml
+      - QUESMA_logging_fileLogging=true
     depends_on:
       clean-clickhouse:
         condition: service_completed_successfully

quesma/clickhouse/quesma_communicator.go

@@ -26,17 +26,28 @@ func (lm *LogManager) Query(ctx context.Context, query string) (*sql.Rows, error
 	return rows, err
 }
 
-// ProcessSimpleSelectQuery - only WHERE clause
+// GetAllColumns - returns all columns for a given table including non-schema fields
+func (lm *LogManager) GetAllColumns(table *Table, query *model.Query) []string {
+	columns, err := table.extractColumns(query, true)
+	if err != nil {
+		logger.Error().Msgf("Failed to extract columns from query: %v", err)
+		return nil
+	}
+	return columns
+}
+
+// ProcessQuery - only WHERE clause
 // TODO query param should be type safe Query representing all parts of
 // sql statement that were already parsed and not string from which
 // we have to extract again different parts like where clause and columns to build a proper result
-func (lm *LogManager) ProcessSelectQuery(ctx context.Context, table *Table, query *model.Query) ([]model.QueryResultRow, error) {
+func (lm *LogManager) ProcessQuery(ctx context.Context, table *Table, query *model.Query, columns []string) ([]model.QueryResultRow, error) {
 	colNames, err := table.extractColumns(query, false)
 	rowToScan := make([]interface{}, len(colNames)+len(query.NonSchemaFields))
 	if err != nil {
 		return nil, err
 	}
-	rows, err := executeQuery(ctx, lm, table.Name, query.StringFromColumns(colNames), append(colNames, query.NonSchemaFields...), rowToScan)
+
+	rows, err := executeQuery(ctx, lm, table.Name, query.StringFromColumns(colNames), columns, rowToScan)
 	if err == nil {
 		for _, row := range rows {
 			row.Index = table.Name
@@ -45,16 +56,6 @@ func (lm *LogManager) ProcessSelectQuery(ctx context.Context, table *Table, quer
 	return rows, err
 }
 
-// TODO add support for autocomplete for attributes, if we'll find it needed
-func (lm *LogManager) ProcessFacetsQuery(ctx context.Context, table *Table, query *model.Query) ([]model.QueryResultRow, error) {
-	colNames, err := table.extractColumns(query, false)
-	if err != nil {
-		return nil, err
-	}
-	rowToScan := make([]interface{}, len(colNames)+len(query.NonSchemaFields))
-	return executeQuery(ctx, lm, table.Name, query.StringFromColumns(colNames), []string{"key", "doc_count"}, rowToScan)
-}
-
 var random = rand.New(rand.NewSource(time.Now().UnixNano()))
 
 const slowQueryThreshold = 30 * time.Second
@@ -114,24 +115,6 @@ func executeQuery(ctx context.Context, lm *LogManager, tableName string, queryAs
 	return res, err
 }
 
-func (lm *LogManager) ProcessAutocompleteSuggestionsQuery(ctx context.Context, table *Table, query *model.Query) ([]model.QueryResultRow, error) {
-	colNames, err := table.extractColumns(query, false)
-	if err != nil {
-		return nil, err
-	}
-	rowToScan := make([]interface{}, len(colNames)+len(query.NonSchemaFields))
-	return executeQuery(ctx, lm, table.Name, query.String(), query.Fields, rowToScan)
-}
-
-func (lm *LogManager) ProcessGeneralAggregationQuery(ctx context.Context, table *Table, query *model.Query) ([]model.QueryResultRow, error) {
-	colNames, err := table.extractColumns(query, true)
-	if err != nil {
-		return nil, err
-	}
-	rowToScan := make([]interface{}, len(colNames))
-	return executeQuery(ctx, lm, table.Name, query.String(), colNames, rowToScan)
-}
-
 // 'selectFields' are all values that we return from the query, both columns and non-schema fields,
 // like e.g. count(), or toInt8(boolField)
 func read(tableName string, rows *sql.Rows, selectFields []string, rowToScan []interface{}) ([]model.QueryResultRow, error) {

quesma/eql/README.md

@@ -0,0 +1,93 @@
+EQL support
+---
+
+
+This package contains the EQL parser and query transformer. 
+
+- The parser is generated using ANTLR4. The grammar is defined in `EQL.g4` file. The generated code is in `parser` directory. Do not review the generated code.
+- HTTP endpoint is implemented in `FIXME`
+- `query_translator.go` is the glue code that connects the parser with the Quesma search engine.
+- Sample EQL query as an HTTP request is in `http_request/eql_search.http` file.
+- A simple command line client is implemented in `playground` directory.
+- End-to-End tests are implemented in `e2e` directory. See file `e2e/eql_test.go` for more details.
+
+
+What is supported?
+---
+
+Comparison operators
+
+| operator | supported          | comment |
+|----------|--------------------|---------|
+| `==`     | :heavy_check_mark: |         |
+| `!=`     | :heavy_check_mark: |         |
+| `>`      | :heavy_check_mark: |         |
+| `>=`     | :heavy_check_mark: |         |
+| `<`      | :heavy_check_mark: |         |
+| `<=`     | :heavy_check_mark: |         | 
+| `:`      | :heavy_check_mark: |         |
+
+
+Lookup operators
+
+| operator  | supported          | comment |
+|-----------|--------------------|---------|
+| `in`      | :heavy_check_mark: |         |
+| `not in`  | :heavy_check_mark: |         |
+| `in~`     | :heavy_check_mark: |         |
+| `not in~` | :heavy_check_mark: |         |
+| `:`       | :heavy_check_mark: |         |
+| `like`    | :heavy_check_mark: |         |
+| `like~`   | :heavy_check_mark: |         |
+| `regex`   | :heavy_check_mark: |         |
+| `regex~`  | :heavy_check_mark: |         |
+
+
+Logical operators
+
+| operator | supported          | comment |
+|----------|--------------------|---------|
+| `and`    | :heavy_check_mark: |         |
+| `or`     | :heavy_check_mark: |         |
+| `not`    | :heavy_check_mark: |         |
+
+
+
+Supported functions
+
+
+| function          | supported          | comment                                |
+|-------------------|--------------------|----------------------------------------|
+| `add`             | :heavy_check_mark: |                                        |
+| `between`         | :x:                |                                        |
+| `cidrMatch`       | :cockroach:        |                                        |
+| `concat`          | :heavy_check_mark: |                                        |
+| `divide`          | :cockroach:        | division of integers should be rounded |
+| `endsWith`        | :heavy_check_mark: |                                        |
+| `endsWith~`       | :heavy_check_mark: |                                        |
+| `indexOf`         | :cockroach:        |                                        |
+| `indexOf~`        | :cockroach:        |                                        |
+| `length`          | :heavy_check_mark: |                                        |
+| `modulo`          | :heavy_check_mark: |                                        |
+| `multiply`        | :heavy_check_mark: |                                        |
+| `number`          | :cockroach:        |                                        |
+| `startsWith`      | :heavy_check_mark: |                                        |
+| `startsWith~`     | :heavy_check_mark: |                                        |
+| `string`          | :heavy_check_mark: |                                        |
+| `stringContains`  | :cockroach:        |                                        |
+| `stringContains~` | :cockroach:        |                                        |
+| `substring`       | :cockroach:        |                                        |
+| `subtract`        | :heavy_check_mark: |                                        |
+
+
+
+
+Known limitations
+---
+
+1. We support only simple EQL queries. Sequence and sample queries are not supported.
+2. Pipe operators are not supported. Syntax is parsed. Error is returned if pipe operator is used in the query. (https://www.elastic.co/guide/en/elasticsearch/reference/current/eql-syntax.html#eql-pipes)
+3. Optional fields are not supported. Field names are parsed. Error is returned if that field is used in the query. (https://www.elastic.co/guide/en/elasticsearch/reference/current/eql-syntax.html#eql-syntax-optional-fields)
+4. Backtick escaping is not supported. (https://www.elastic.co/guide/en/elasticsearch/reference/current/eql-syntax.html#eql-syntax-escape-a-field-name)
+5. Error handling is missing. Every error will be returned as na internal server error.
+