Skip to content

Commit

Permalink
[panw] Update event.created field to follow ECS spec (#10731)
Browse files Browse the repository at this point in the history 
The [ECS spec states](https://www.elastic.co/guide/en/ecs/current/ecs-event.html#field-event-created) `event.created` should be the time the event is first seen by the agent or ingested. The panw integration was not following this and was instead setting it to the PANOS event timestamp.

This corrects the integration so that it follows ECS properly.

These field changes have been made:
* `event.created` is set from filebeat's initial timestamp (before modification by the syslog processor or ingest pipeline).
*  `panw.panos.received_time` is now defined as the PANOS log timestamp (it is the same value that was previous in `event.created`)
* `panw.panos.generated_time` is added to hold the PANOS generated time
* `@timestamp` now holds the `panw.panos.high_resolution_timestamp` value, or if it isn't available `panw.panos.received_time`

This is also a major version upgrade, because of these changed field definitions.
  • Loading branch information
@mjwolf
mjwolf authored Aug 15, 2024
1 parent 30d69b2 commit 7e37f1c
Show file tree
Hide file tree
Showing 30 changed files with 1,060 additions and 1,018 deletions.
5 changes: 5 additions & 0 deletions packages/panw/changelog.yml
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
# newer versions go on top
- version: "4.0.0"
changes:
- description: Correct use of ECS event.created field
type: breaking-change
link: https://github.com/elastic/integrations/pull/10731
- version: "3.26.4"
changes:
- description: Use high-res timestamp for @timestamp and ensure time zone config is applied.
Expand Down
12 changes: 4 additions & 8 deletions .../panw/data_stream/panos/_dev/test/pipeline/test-panw-panos-audit-sample.log-expected.json
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,13 @@
{
"expected": [
{
"@timestamp": "2024-04-11T20:06:15.000-04:00",
"ecs": {
"version": "8.11.0"
},
"event": {
"category": [
"configuration"
],
"created": "2024-04-11T16:06:15.000-04:00",
"kind": "event",
"original": "Apr 11 20:06:15 192.168.0.1 01111111111,2024/04/11 20:06:15,audit,2561,gui-op,suser,\"<debug><dataplane><packet-diag><show><setting/></show></packet-diag></dataplane></debug>\",success",
"outcome": "success",
Expand All @@ -28,6 +26,7 @@
"cmd": "<debug><dataplane><packet-diag><show><setting/></show></packet-diag></dataplane></debug>",
"cmd_source": "gui-op",
"config_version": "2561",
"generated_time": "2024-04-11T20:06:15.000-04:00",
"type": "AUDIT"
}
},
Expand All @@ -44,15 +43,13 @@
}
},
{
"@timestamp": "2024-04-18T18:35:20.000-04:00",
"ecs": {
"version": "8.11.0"
},
"event": {
"category": [
"configuration"
],
"created": "2024-04-18T14:35:20.000-04:00",
"kind": "event",
"original": "Apr 18 18:35:20 10.1.1.1 003001000000,2024/04/18 18:35:20,audit,2561,gui-op,Mustang,\"<show><config-locks><vsys>all</vsys></config-locks></show>\",success",
"outcome": "success",
Expand All @@ -71,6 +68,7 @@
"cmd": "<show><config-locks><vsys>all</vsys></config-locks></show>",
"cmd_source": "gui-op",
"config_version": "2561",
"generated_time": "2024-04-18T18:35:20.000-04:00",
"type": "AUDIT"
}
},
Expand All @@ -87,15 +85,13 @@
}
},
{
"@timestamp": "2024-04-18T18:36:20.000-04:00",
"ecs": {
"version": "8.11.0"
},
"event": {
"category": [
"configuration"
],
"created": "2024-04-18T14:36:20.000-04:00",
"kind": "event",
"original": "Apr 18 18:36:20 test-hostname 003001000000,2024/04/18 18:36:20,audit,2561,gui-op,Mustang,\"<show><config-locks><vsys>all</vsys></config-locks></show>\",success",
"outcome": "success",
Expand All @@ -114,6 +110,7 @@
"cmd": "<show><config-locks><vsys>all</vsys></config-locks></show>",
"cmd_source": "gui-op",
"config_version": "2561",
"generated_time": "2024-04-18T18:36:20.000-04:00",
"type": "AUDIT"
}
},
Expand All @@ -130,15 +127,13 @@
}
},
{
"@timestamp": "2024-04-18T18:37:20.000-04:00",
"ecs": {
"version": "8.11.0"
},
"event": {
"category": [
"configuration"
],
"created": "2024-04-18T14:37:20.000-04:00",
"kind": "event",
"original": "Apr 18 18:37:20 test-hostname.test.intra 003001000000,2024/04/18 18:37:20,audit,2561,gui-op,Mustang,\"<show><config-locks><vsys>all</vsys></config-locks></show>\",success",
"outcome": "success",
Expand All @@ -157,6 +152,7 @@
"cmd": "<show><config-locks><vsys>all</vsys></config-locks></show>",
"cmd_source": "gui-op",
"config_version": "2561",
"generated_time": "2024-04-18T18:37:20.000-04:00",
"type": "AUDIT"
}
},
Expand Down
1 change: 0 additions & 1 deletion ...a_stream/panos/_dev/test/pipeline/test-panw-panos-authentication-sample.log-expected.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
"category": [
"authentication"
],
"created": "2019-11-23T00:44:44.000-04:30",
"kind": "event",
"original": "1,2019/11/23 00:44:44,01234567890,AUTHENTICATION,login,2561,2019/11/23 00:44:44,vsys1,fe80::4e7:1ab2:f6aa:82fa,user,normalize-user,object,auth-policy,12345,auth-id,vendor,log-action,server-profile,description,client-type,event-type,10,20,action-flag,0,0,0,0,vsys-name,device-name,vsys-id,auth-protocol,uuid,2021-11-23T01:03:05.498-08:00,src-category,src-profile,src-model,src-vendor,src-os-family,src-os-version,src-hostname,aa:aa:aa:aa:aa:aa,region,,\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36\",session-id",
"outcome": "success",
Expand Down
18 changes: 12 additions & 6 deletions ...panw/data_stream/panos/_dev/test/pipeline/test-panw-panos-config-sample.log-expected.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
"category": [
"configuration"
],
"created": "2021-10-25T20:25:39.000-04:00",
"kind": "event",
"original": "1,2021/10/25 20:25:39,,CONFIG,0,2561,2021/10/25 20:25:39,81.2.69.193,,set,admin,Web,Succeeded, config shared log-settings iptag match-list ip-tag,,\"iptag { match-list { ip-tag { send-syslog [ SYSLOG-1 ]; filter \"\"All Logs\"\"; } } } \",1234567890,0x0,0,0,0,0,,PA-VM,0,",
"outcome": "success",
Expand Down Expand Up @@ -40,7 +39,9 @@
"device_group_hierarchy3": "0",
"device_group_hierarchy4": "0",
"device_group_id": "0",
"generated_time": "2021-10-25T20:25:39.000-04:00",
"path": "config shared log-settings iptag match-list ip-tag",
"received_time": "2021-10-25T20:25:39.000-04:00",
"result": "Succeeded",
"sequence_number": "1234567890",
"sub_type": "0",
Expand Down Expand Up @@ -72,7 +73,6 @@
"category": [
"configuration"
],
"created": "2021-10-25T20:25:19.000-04:00",
"kind": "event",
"original": "1,2021/10/25 20:25:19,,CONFIG,0,2561,2021/10/25 20:25:19,81.2.69.193,,set,admin,Web,Succeeded, config shared log-settings globalprotect match-list globalProtect,,\"globalprotect { match-list { globalProtect { send-syslog [ SYSLOG-1 ]; filter \"\"All Logs\"\"; } } } \",1234567890,0x0,0,0,0,0,,PA-VM,0,",
"outcome": "success",
Expand Down Expand Up @@ -102,7 +102,9 @@
"device_group_hierarchy3": "0",
"device_group_hierarchy4": "0",
"device_group_id": "0",
"generated_time": "2021-10-25T20:25:19.000-04:00",
"path": "config shared log-settings globalprotect match-list globalProtect",
"received_time": "2021-10-25T20:25:19.000-04:00",
"result": "Succeeded",
"sequence_number": "1234567890",
"sub_type": "0",
Expand Down Expand Up @@ -134,7 +136,6 @@
"category": [
"configuration"
],
"created": "2023-10-04T08:52:10.000-04:00",
"kind": "event",
"original": "1,2023/10/04 08:52:10,007058000248010,CONFIG,0,2816,2023/10/04 08:52:10,81.2.69.193,,set,admin,Web,Succeeded, vsys vsys1 rulebase security rules reset-adult,,reset-adult 73a06abf-75ca-436f-9319-1a15b27fa692 { to [ public ]; from [ private ]; source [ any ]; destination [ any ]; source-user [ any ]; category [ adult ]; application [ any ]; service [ application-default ]; source-hip [ any ]; destination-hip [ any ]; action reset-client; icmp-unreachable yes; log-start yes; rule-type interzone; } ,7286123782408765488,0x0,0,0,0,0,,PA-VM,0,",
"outcome": "success",
Expand Down Expand Up @@ -165,7 +166,9 @@
"device_group_hierarchy3": "0",
"device_group_hierarchy4": "0",
"device_group_id": "0",
"generated_time": "2023-10-04T08:52:10.000-04:00",
"path": "vsys vsys1 rulebase security rules reset-adult",
"received_time": "2023-10-04T08:52:10.000-04:00",
"result": "Succeeded",
"sequence_number": "7286123782408765488",
"sub_type": "0",
Expand Down Expand Up @@ -197,7 +200,6 @@
"category": [
"configuration"
],
"created": "2023-10-04T08:50:28.000-04:00",
"kind": "event",
"original": "1,2023/10/04 08:50:28,007058000248010,CONFIG,0,2816,2023/10/04 08:50:28,81.2.69.193,,move,admin,Web,Succeeded, vsys vsys1 rulebase security rules block-1.1.1.1,,,7286123782408765487,0x0,0,0,0,0,,PA-VM,0,",
"outcome": "success",
Expand Down Expand Up @@ -227,7 +229,9 @@
"device_group_hierarchy3": "0",
"device_group_hierarchy4": "0",
"device_group_id": "0",
"generated_time": "2023-10-04T08:50:28.000-04:00",
"path": "vsys vsys1 rulebase security rules block-1.1.1.1",
"received_time": "2023-10-04T08:50:28.000-04:00",
"result": "Succeeded",
"sequence_number": "7286123782408765487",
"sub_type": "0",
Expand Down Expand Up @@ -258,7 +262,6 @@
"category": [
"configuration"
],
"created": "2023-10-04T08:27:38.000-04:00",
"kind": "event",
"original": "1,2023/10/04 08:27:38,007058000248010,CONFIG,0,2816,2023/10/04 08:27:38,81.2.69.193,,override,admin,Web,Failed, deviceconfig system device-telemetry,,,7286123782408765440,0x0,0,0,0,0,,PA-VM,0,",
"outcome": "failure",
Expand Down Expand Up @@ -288,7 +291,9 @@
"device_group_hierarchy3": "0",
"device_group_hierarchy4": "0",
"device_group_id": "0",
"generated_time": "2023-10-04T08:27:38.000-04:00",
"path": "deviceconfig system device-telemetry",
"received_time": "2023-10-04T08:27:38.000-04:00",
"result": "Failed",
"sequence_number": "7286123782408765440",
"sub_type": "0",
Expand Down Expand Up @@ -320,7 +325,6 @@
"category": [
"configuration"
],
"created": "2024-02-29T16:59:40.000-04:00",
"kind": "event",
"original": "1,2024/02/29 16:59:40,01234567890,CONFIG,0,2561,2024/02/29 16:59:40,81.2.69.193,,edit,admin,Web,Succeeded, vsys vsys1 address test123,\"test123 { description \"\"this, is a test. with, three comma, x4\"\"; } \",\"test123 { description \"\"this, is a test. with, three comma, x5\"\"; } \",7304387121517691189,0x0,0,0,0,0,,PA-VM,0,,0,2024-02-29T16:59:40.421+01:00",
"outcome": "success",
Expand Down Expand Up @@ -352,8 +356,10 @@
"device_group_hierarchy3": "0",
"device_group_hierarchy4": "0",
"device_group_id": "0",
"generated_time": "2024-02-29T16:59:40.000-04:00",
"high_resolution_timestamp": "2024-02-29T11:59:40.421-04:00",
"path": "vsys vsys1 address test123",
"received_time": "2024-02-29T16:59:40.000-04:00",
"result": "Succeeded",
"sequence_number": "7304387121517691189",
"sub_type": "0",
Expand Down
1 change: 0 additions & 1 deletion ...tream/panos/_dev/test/pipeline/test-panw-panos-correlated-events-sample.log-expected.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
"category": [
"network"
],
"created": "2019-10-09T10:20:15.000-02:30",
"kind": "event",
"original": "Nov 30 16:09:08 1,2019/10/09 10:20:15,001234567890002,CORRELATION,0,2304,2019/10/09 10:20:15,81.2.69.142,src-user,vsys,cat,4,0,0,0,0,vsys-name,d-name,vsys-id,o-name,o-id,evidence",
"outcome": "success",
Expand Down
9 changes: 6 additions & 3 deletions .../data_stream/panos/_dev/test/pipeline/test-panw-panos-decryption-sample.log-expected.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@
"category": [
"network"
],
"created": "2021-11-11T15:42:44.000-08:00",
"kind": "event",
"original": "<14>Nov 30 16:09:33 PA-220 1,2021/11/11 15:42:44,007051000184334,DECRYPTION,0,2561,2021/11/11 15:42:44,81.2.69.145,81.2.69.144,81.2.69.145,81.2.69.144,intrazone-default,,,incomplete,vsys1,LAN,LAN,ethernet1/2,ethernet1/2,,2021/11/11 15:42:44,33288,1,49908,443,49908,20077,0x1400000,tcp,allow,N/A,,,,,731a6a1a-9a62-4a92-a49a-0876025a9436,Server_Hello,Client_Hello,TLS1.2,ECDHE,AES_256_GCM,SHA384,,,Certificate,trusted,Trusted,GlobalProtect,ba67e84495b59512,a6a13e87221733b712ddbd0978da1ffdd69503dfd1f0a86f73d86bf90b743b85,2021/11/11 15:21:28,2022/11/11 15:21:28,V3,2048,9,9,0,0,:::::RSA,com.example.com,com.example.com,,,Received fatal alert CertificateUnknown from client,,,,,,,,2021-11-11T15:42:44.845-08:00,,,,,,,,,,,,,,,,,7028724914890736000,0x0,0,0,0,0,,PA-VM,1,unknown,unknown,unknown,1,,,incomplete,no",
"outcome": "failure",
Expand Down Expand Up @@ -94,6 +93,7 @@
"device_group_hierarchy4": "0",
"error_message": "Received fatal alert CertificateUnknown from client",
"flow_id": "33288",
"generated_time": "2021-11-11T15:42:44.000-08:00",
"high_resolution_timestamp": "2021-11-11T15:42:44.845-08:00",
"hs_stage_c2f": "Server_Hello",
"hs_stage_f2s": "Client_Hello",
Expand All @@ -107,6 +107,7 @@
}
},
"proxy_type": "GlobalProtect",
"received_time": "2021-11-11T15:42:44.000-08:00",
"repeat_count": 1,
"root_certificate_status": "trusted",
"root_common_name": {
Expand Down Expand Up @@ -228,7 +229,6 @@
"category": [
"network"
],
"created": "2021-11-11T15:42:44.000-08:00",
"kind": "event",
"original": "<134>1 2022-11-03T13:40:34+01:00 PA-220 1,2021/11/11 15:42:44,007051000184334,DECRYPTION,0,2561,2021/11/11 15:42:44,81.2.69.145,81.2.69.144,81.2.69.145,81.2.69.144,intrazone-default,,,incomplete,vsys1,LAN,LAN,ethernet1/2,ethernet1/2,,2021/11/11 15:42:44,33288,1,49908,443,49908,20077,0x1400000,tcp,allow,N/A,,,,,731a6a1a-9a62-4a92-a49a-0876025a9436,Server_Hello,Client_Hello,TLS1.2,ECDHE,AES_256_GCM,SHA384,,,Certificate,trusted,Trusted,GlobalProtect,ba67e84495b59512,a6a13e87221733b712ddbd0978da1ffdd69503dfd1f0a86f73d86bf90b743b85,2021/11/11 15:21:28,2022/11/11 15:21:28,V3,Unknown,9,9,0,0,:::::RSA,com.example.com,com.example.com,,,Received fatal alert CertificateUnknown from client,,,,,,,,2021-11-11T15:42:44.845-08:00,,,,,,,,,,,,,,,,,7028724914890736000,0x0,0,0,0,0,,PA-VM,1,unknown,unknown,unknown,1,,,incomplete,no",
"outcome": "failure",
Expand Down Expand Up @@ -294,6 +294,7 @@
"device_group_hierarchy4": "0",
"error_message": "Received fatal alert CertificateUnknown from client",
"flow_id": "33288",
"generated_time": "2021-11-11T15:42:44.000-08:00",
"high_resolution_timestamp": "2021-11-11T15:42:44.845-08:00",
"hs_stage_c2f": "Server_Hello",
"hs_stage_f2s": "Client_Hello",
Expand All @@ -307,6 +308,7 @@
}
},
"proxy_type": "GlobalProtect",
"received_time": "2021-11-11T15:42:44.000-08:00",
"repeat_count": 1,
"root_certificate_status": "trusted",
"root_common_name": {
Expand Down Expand Up @@ -427,7 +429,6 @@
"category": [
"network"
],
"created": "2024-05-30T15:42:44.000-08:00",
"kind": "event",
"original": "<14>Nov 30 16:09:33 PA-220 1,2024/05/30 15:42:44,007051000184334,DECRYPTION,0,2561,2024/05/30 15:42:44,81.2.69.145,81.2.69.144,81.2.69.145,81.2.69.144,intrazone-default,,,ssl,vsys1,LAN,PROXY,ae1,ethernet1/5,TEST-Log,2024/05/30 15:46:50,35508943,1,60312,9400,0,0,0x400,tcp,allow,N/A,,,,,731a6a1a-9a62-4a92-a49a-0876025a9436,Unknown,Unknown,TLS1.3,ECDHE,AES_256_GCM,SHA384,SSL Exception Destination Hosts,,None,uninspected,Uninspected,No Decrypt,,,,,V1,0,0,0,0,0,:::::NONE,,,,,,,,,,,,,2024-05-27T15:46:51.539+02:00,,,,,,,,,,,,,,,,,7335860982980205586,0x8000000000000000,12,0,0,0,,TESTFW01,1,encrypted-tunnel,networking,browser-based,4,\\\"used-by-malware,able-to-transfer-file,has-known-vulnerability,tunnel-other-application,pervasive-use\\\",,ssl,no,no",
"outcome": "failure",
Expand Down Expand Up @@ -485,6 +486,7 @@
"device_group_hierarchy3": "0",
"device_group_hierarchy4": "0",
"flow_id": "35508943",
"generated_time": "2024-05-30T15:42:44.000-08:00",
"high_resolution_timestamp": "2024-05-27T05:46:51.539-08:00",
"hs_stage_c2f": "Unknown",
"hs_stage_f2s": "Unknown",
Expand All @@ -497,6 +499,7 @@
"name": "SSL Exception Destination Hosts"
},
"proxy_type": "No Decrypt",
"received_time": "2024-05-30T15:42:44.000-08:00",
"repeat_count": 1,
"root_certificate_status": "uninspected",
"root_common_name": {
Expand Down
Loading

0 comments on commit 7e37f1c

Please sign in to comment.