more tests + adapt README

Signed-off-by: odubajDT <[email protected]>
open-telemetry · Nov 26, 2024 · df49340 · df49340
1 parent a308e23
commit df49340
Show file tree

Hide file tree

Showing 4 changed files with 40 additions and 3 deletions.
diff --git a/pkg/stanza/fileconsumer/config_test.go b/pkg/stanza/fileconsumer/config_test.go
@@ -219,6 +219,23 @@ func TestUnmarshal(t *testing.T) {
 					return newMockOperatorConfig(cfg)
 				}(),
 			},
+			{
+				Name: "sort_by_group_by",
+				Expect: func() *mockOperatorConfig {
+					cfg := NewConfig()
+					cfg.OrderingCriteria = matcher.OrderingCriteria{
+						Regex:   `err\.(?P<file_num>[a-zA-Z])\.\d+\.\d{10}\.log`,
+						GroupBy: `err\.(?P<value>[a-z]+).[0-9]*.*log`,
+						SortBy: []matcher.Sort{
+							{
+								SortType: "numeric",
+								RegexKey: "file_num",
+							},
+						},
+					}
+					return newMockOperatorConfig(cfg)
+				}(),
+			},
 			{
 				Name: "poll_interval_no_units",
 				Expect: func() *mockOperatorConfig {

diff --git a/pkg/stanza/fileconsumer/matcher/matcher_test.go b/pkg/stanza/fileconsumer/matcher/matcher_test.go
@@ -440,6 +440,17 @@ func TestMatcher(t *testing.T) {
 			},
 			expected: []string{"err.a.123456789.log", "err.a.123456788.log", "err.a.123456787.log", "err.a.123456786.log", "err.b.123456789.log", "err.b.123456788.log"},
 		},
+		{
+			name:    "Grouping",
+			files:   []string{"err.a.123456788.log", "err.a.123456789.log", "err.a.123456787.log", "err.b.123456788.log", "err.a.123456786.log", "err.b.123456789.log"},
+			include: []string{"err.*.log"},
+			exclude: []string{},
+			filterCriteria: OrderingCriteria{
+				TopN:    6,
+				GroupBy: `err\.(?P<value>[a-z]+).[0-9]*.*log`,
+			},
+			expected: []string{"err.a.123456786.log", "err.a.123456787.log", "err.a.123456788.log", "err.a.123456789.log", "err.b.123456788.log", "err.b.123456789.log"},
+		},
 		{
 			name:    "Numeric Sorting Ascending",
 			files:   []string{"err.123456789.log", "err.123456788.log", "err.123456786.log", "err.123456787.log"},

diff --git a/pkg/stanza/fileconsumer/testdata/config.yaml b/pkg/stanza/fileconsumer/testdata/config.yaml
@@ -51,6 +51,14 @@ exclude_one:
     - "*.log"
   exclude:
     - one.log
+sort_by_group_by:
+  type: mock
+  ordering_criteria:
+    regex: 'err\.(?P<file_num>[a-zA-Z])\.\d+\.\d{10}\.log'
+    group_by: 'err\.(?P<value>[a-z]+).[0-9]*.*log'
+    sort_by:
+      - regex_key: file_num
+        sort_type: numeric
 sort_by_numeric:
   type: mock
   ordering_criteria:

diff --git a/receiver/filelogreceiver/README.md b/receiver/filelogreceiver/README.md
@@ -34,7 +34,7 @@ Tails and parses logs from files.
 | `include_file_path_resolved`          | `false`                              | Whether to add the file path after symlinks resolution as the attribute `log.file.path_resolved`.                                                                                                                                                               |
 | `include_file_owner_name`             | `false`                              | Whether to add the file owner name as the attribute `log.file.owner.name`. Not supported for windows.                                                                                                                                                           |
 | `include_file_owner_group_name`       | `false`                              | Whether to add the file group name as the attribute `log.file.owner.group.name`. Not supported for windows.                                                                                                                                                     |
-| `include_file_record_number`            | `false`                              | Whether to add the record number in the file as the attribute `log.file.record_number`.                                                                                                                                                                    |
+| `include_file_record_number`          | `false`                              | Whether to add the record number in the file as the attribute `log.file.record_number`.                                                                                                                                                                         |
 | `poll_interval`                       | 200ms                                | The [duration](#time-parameters) between filesystem polls.                                                                                                                                                                                                      |
 | `fingerprint_size`                    | `1kb`                                | The number of bytes with which to identify a file. The first bytes in the file are used as the fingerprint. Decreasing this value at any point will cause existing fingerprints to forgotten, meaning that all files will be read from the beginning (one time) |
 | `max_log_size`                        | `1MiB`                               | The maximum size of a log entry to read. A log entry will be truncated if it is larger than `max_log_size`. Protects against reading large amounts of data into memory.                                                                                         |
@@ -52,9 +52,10 @@ Tails and parses logs from files.
 | `retry_on_failure.enabled`            | `false`                              | If `true`, the receiver will pause reading a file and attempt to resend the current batch of logs if it encounters an error from downstream components.                                                                                                         |
 | `retry_on_failure.initial_interval`   | `1s`                                 | [Time](#time-parameters) to wait after the first failure before retrying.                                                                                                                                                                                       |
 | `retry_on_failure.max_interval`       | `30s`                                | Upper bound on retry backoff [interval](#time-parameters). Once this value is reached the delay between consecutive retries will remain constant at the specified value.                                                                                        |
-| `retry_on_failure.max_elapsed_time`   | `5m`                                 | Maximum amount of [time](#time-parameters) (including retries) spent trying to send a logs batch to a downstream consumer. Once this value is reached, the data is discarded. Retrying never stops if set to `0`.                                               
+| `retry_on_failure.max_elapsed_time`   | `5m`                                 | Maximum amount of [time](#time-parameters) (including retries) spent trying to send a logs batch to a downstream consumer. Once this value is reached, the data is discarded. Retrying never stops if set to `0`.                                               |
 | `ordering_criteria.regex`             |                                      | Regular expression used for sorting, should contain a named capture groups that are to be used in `regex_key`.                                                                                                                                                  |
-| `ordering_criteria.top_n`             | 1 | The number of files to track when using file ordering. The top N files are tracked after applying the ordering criteria.                                                                                                                                        |
+| `ordering_criteria.gropup_by`         |                                      | Regular expression used for grouping, which is done pre-sorting. Should contain a named capture groups.                                                                                                                                                         |
+| `ordering_criteria.top_n`             | 1                                    | The number of files to track when using file ordering. The top N files are tracked after applying the ordering criteria.                                                                                                                                        |
 | `ordering_criteria.sort_by.sort_type` |                                      | Type of sorting to be performed (e.g., `numeric`, `alphabetical`, `timestamp`, `mtime`)                                                                                                                                                                         |
 | `ordering_criteria.sort_by.location`  |                                      | Relevant if `sort_type` is set to `timestamp`. Defines the location of the timestamp of the file.                                                                                                                                                               |
 | `ordering_criteria.sort_by.format`    |                                      | Relevant if `sort_type` is set to `timestamp`. Defines the strptime format of the timestamp being sorted.                                                                                                                                                       |