Update file experiments to the use of logrotate FS

This commit updates the file_to_blackhole experiments to rely on the lading
logrotate FS generator, allowing us to assert on whether the log Agent loses
bytes in the course of its operation. I've added a new check asserting that no
bytes are lost, although I do not expect this to pass on higher latency intake
experiments yet.

REF #30744
REF DataDog/lading#1090

Signed-off-by: Brian L. Troutwine <[email protected]>

test/regression/cases/file_to_blackhole_0ms_latency/datadog-agent/conf.d/disk-listener.d/conf.yaml

@@ -1,5 +1,5 @@
 logs:
   - type: file
-    path: "/tmp/smp/*.log"
+    path: "/smp-shared/*.log"
     service: "my-service"
     source: "my-client-app"

test/regression/cases/file_to_blackhole_0ms_latency/experiment.yaml

@@ -31,3 +31,9 @@ checks:
       series: total_rss_bytes
       # The machine has 12GiB free.
       upper_bound: 1.2GiB
+
+  - name: lost_bytes
+    description: "Allowable bytes not polled by log Agent"
+    bounds:
+      series: lost_bytes
+      upper_bound: 0KB

test/regression/cases/file_to_blackhole_0ms_latency/lading/lading.yaml

@@ -1,16 +1,16 @@
 generator:
   - file_gen:
-      logrotate:
+      logrotate_fs:
         seed: [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53,
                59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131]
-        root: "/tmp/smp"
         concurrent_logs: 8
-        maximum_bytes_per_log: "500MiB"
+        maximum_bytes_per_log: 500MB
         total_rotations: 5
-        max_depth: 1 # flat, all logs are /tmp/smp/12345.log
+        max_depth: 0
         variant: "ascii"
-        bytes_per_second: "10MiB"
-        maximum_prebuild_cache_size_bytes: "300MiB"
+        bytes_per_second: 10MB
+        maximum_prebuild_cache_size_bytes: 300MB
+        mount_point: /smp-shared
 
 blackhole:
   - http:

test/regression/cases/file_to_blackhole_1000ms_latency/datadog-agent/conf.d/disk-listener.d/conf.yaml

@@ -1,5 +1,5 @@
 logs:
   - type: file
-    path: "/tmp/smp/*.log"
+    path: "/smp-shared/*.log"
     service: "my-service"
     source: "my-client-app"

test/regression/cases/file_to_blackhole_1000ms_latency/experiment.yaml

@@ -31,3 +31,9 @@ checks:
       series: total_rss_bytes
       # The machine has 12GiB free.
       upper_bound: 1.2GiB
+
+  - name: lost_bytes
+    description: "Allowable bytes not polled by log Agent"
+    bounds:
+      series: lost_bytes
+      upper_bound: 0KB

test/regression/cases/file_to_blackhole_1000ms_latency/lading/lading.yaml

@@ -1,16 +1,16 @@
 generator:
   - file_gen:
-      logrotate:
+      logrotate_fs:
         seed: [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53,
                59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131]
-        root: "/tmp/smp"
         concurrent_logs: 8
-        maximum_bytes_per_log: "500MiB"
+        maximum_bytes_per_log: 500MB
         total_rotations: 5
-        max_depth: 1 # flat, all logs are /tmp/smp/12345.log
+        max_depth: 0
         variant: "ascii"
-        bytes_per_second: "10MiB"
-        maximum_prebuild_cache_size_bytes: "300MiB"
+        bytes_per_second: 10MB
+        maximum_prebuild_cache_size_bytes: 300MB
+        mount_point: /smp-shared
 
 blackhole:
   - http:

test/regression/cases/file_to_blackhole_100ms_latency/datadog-agent/conf.d/disk-listener.d/conf.yaml

@@ -1,5 +1,5 @@
 logs:
   - type: file
-    path: "/tmp/smp/*.log"
+    path: "/smp-shared/*.log"
     service: "my-service"
     source: "my-client-app"

test/regression/cases/file_to_blackhole_100ms_latency/experiment.yaml

@@ -31,3 +31,9 @@ checks:
       series: total_rss_bytes
       # The machine has 12GiB free.
       upper_bound: 1.2GiB
+
+  - name: lost_bytes
+    description: "Allowable bytes not polled by log Agent"
+    bounds:
+      series: lost_bytes
+      upper_bound: 0KB

test/regression/cases/file_to_blackhole_100ms_latency/lading/lading.yaml

@@ -1,16 +1,16 @@
 generator:
   - file_gen:
-      logrotate:
+      logrotate_fs:
         seed: [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53,
                59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131]
-        root: "/tmp/smp"
         concurrent_logs: 8
-        maximum_bytes_per_log: "500MiB"
+        maximum_bytes_per_log: 500MB
         total_rotations: 5
-        max_depth: 1 # flat, all logs are /tmp/smp/12345.log
+        max_depth: 0
         variant: "ascii"
-        bytes_per_second: "10MiB"
-        maximum_prebuild_cache_size_bytes: "300MiB"
+        bytes_per_second: 10MB
+        maximum_prebuild_cache_size_bytes: 300MB
+        mount_point: /smp-shared
 
 blackhole:
   - http:

test/regression/cases/file_to_blackhole_300ms_latency/datadog-agent/conf.d/disk-listener.d/conf.yaml

@@ -1,5 +1,5 @@
 logs:
   - type: file
-    path: "/tmp/smp/*.log"
+    path: "/smp-shared/*.log"
     service: "my-service"
     source: "my-client-app"

test/regression/cases/file_to_blackhole_300ms_latency/experiment.yaml

@@ -31,3 +31,9 @@ checks:
       series: total_rss_bytes
       # The machine has 12GiB free.
       upper_bound: 1.2GiB
+
+  - name: lost_bytes
+    description: "Allowable bytes not polled by log Agent"
+    bounds:
+      series: lost_bytes
+      upper_bound: 0KB

test/regression/cases/file_to_blackhole_300ms_latency/lading/lading.yaml

@@ -1,16 +1,16 @@
 generator:
   - file_gen:
-      logrotate:
+      logrotate_fs:
         seed: [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53,
                59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131]
-        root: "/tmp/smp"
         concurrent_logs: 8
-        maximum_bytes_per_log: "500MiB"
+        maximum_bytes_per_log: 500MB
         total_rotations: 5
-        max_depth: 1 # flat, all logs are /tmp/smp/12345.log
+        max_depth: 0
         variant: "ascii"
-        bytes_per_second: "10MiB"
-        maximum_prebuild_cache_size_bytes: "300MiB"
+        bytes_per_second: 10MB
+        maximum_prebuild_cache_size_bytes: 300MB
+        mount_point: /smp-shared
 
 blackhole:
   - http:

test/regression/cases/file_to_blackhole_500ms_latency/datadog-agent/conf.d/disk-listener.d/conf.yaml

@@ -1,5 +1,5 @@
 logs:
   - type: file
-    path: "/tmp/smp/*.log"
+    path: "/smp-shared/*.log"
     service: "my-service"
     source: "my-client-app"

test/regression/cases/file_to_blackhole_500ms_latency/experiment.yaml

@@ -31,3 +31,9 @@ checks:
       series: total_rss_bytes
       # The machine has 12GiB free.
       upper_bound: 1.2GiB
+
+  - name: lost_bytes
+    description: "Allowable bytes not polled by log Agent"
+    bounds:
+      series: lost_bytes
+      upper_bound: 0KB

test/regression/cases/file_to_blackhole_500ms_latency/lading/lading.yaml

@@ -1,16 +1,16 @@
 generator:
   - file_gen:
-      logrotate:
+      logrotate_fs:
         seed: [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53,
                59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131]
-        root: "/tmp/smp"
         concurrent_logs: 8
-        maximum_bytes_per_log: "500MiB"
+        maximum_bytes_per_log: 500MB
         total_rotations: 5
-        max_depth: 1 # flat, all logs are /tmp/smp/12345.log
+        max_depth: 0
         variant: "ascii"
-        bytes_per_second: "10MiB"
-        maximum_prebuild_cache_size_bytes: "300MiB"
+        bytes_per_second: 10MB
+        maximum_prebuild_cache_size_bytes: 300MB
+        mount_point: /smp-shared
 
 blackhole:
   - http: