azure: add related.entity field to auditlogs and signinlogs default ingest pipeline

packages/azure/changelog.yml

@@ -1,3 +1,8 @@
+- version: "1.18.0"
+  changes:
+    - description: Add entity identifiers to `related.entity`.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/11344
 - version: "1.17.0"
   changes:
     - description: Add `event.reason` log field for exact data found in requests that matches a firewall rule.

packages/azure/data_stream/auditlogs/_dev/test/pipeline/test-audit-logs-edgecases.log-expected.json

@@ -74,6 +74,10 @@
                 "outcome": "success"
             },
             "related": {
+                "entity": [
+                    "b9814691-9ca1-4e55-a1ac-8ef5dd010ec0",
+                    "a7d5dcbe-0627-4ddf-a2f4-86b6785bcc42"
+                ],
                 "ip": [
                     "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6"
                 ]
@@ -164,6 +168,12 @@
                 "original": "{ \"time\": \"2022-01-22T18:15:02.3875429Z\", \"resourceId\": \"/tenants/4bbb79f7-5724-4c9e-95f3-de075f6ec090/providers/Microsoft.aadiam\", \"operationName\": \"Update service principal\", \"operationVersion\": \"1.0\", \"category\": \"AuditLogs\", \"tenantId\": \"4bbb79f7-5724-4c9e-95f3-de075f6ec090\", \"resultSignature\": \"None\", \"durationMs\": 0, \"callerIpAddress\": \"::2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6\", \"correlationId\": \"87979703-118b-498f-99c2-ccd1a56f1a5a\", \"identity\": \"Managed Service Identity\", \"Level\": 4, \"properties\": {\"id\":\"Directory_87979703-118b-498f-99c2-ccd1a56f1a5a_ULAYA_144938566\",\"category\":\"ApplicationManagement\",\"correlationId\":\"87979703-118b-498f-99c2-ccd1a56f1a5a\",\"result\":\"success\",\"resultReason\":\"\",\"activityDisplayName\":\"Update service principal\",\"activityDateTime\":\"2022-01-22T18:15:02.3875429+00:00\",\"loggedByService\":\"Core Directory\",\"operationType\":\"Update\",\"userAgent\":null,\"initiatedBy\":{\"app\":{\"appId\":null,\"displayName\":\"Managed Service Identity\",\"servicePrincipalId\":\"b9814691-9ca1-4e55-a1ac-8ef5dd010ec0\",\"servicePrincipalName\":null}},\"targetResources\":[{\"id\":\"a7d5dcbe-0627-4ddf-a2f4-86b6785bcc42\",\"displayName\":\"billing-test-wus\",\"type\":\"ServicePrincipal\",\"modifiedProperties\":[{\"displayName\":\"TargetId.ServicePrincipalNames\",\"oldValue\":null,\"newValue\":\"\\\"a70a7931-c387-4dce-9f35-fbf95bdcc91e;https://identity.azure.net/N8CUySpCeRFU3iB/PEuFlON4zd8+n8d3qgzrF1MviSY=\\\"\"}],\"administrativeUnits\":[]}],\"additionalDetails\":[{\"key\":\"User-Agent\",\"value\":\"Microsoft Azure Graph Client Library 2.1.17-internal\"},{\"key\":\"AppId\",\"value\":\"a70a7931-c387-4dce-9f35-fbf95bdcc91e\"}]}}",
                 "outcome": "success"
             },
+            "related": {
+                "entity": [
+                    "b9814691-9ca1-4e55-a1ac-8ef5dd010ec0",
+                    "a7d5dcbe-0627-4ddf-a2f4-86b6785bcc42"
+                ]
+            },
             "source": {
                 "address": "::2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6"
             },

packages/azure/data_stream/auditlogs/_dev/test/pipeline/test-audit-logs-sample.log-expected.json

@@ -83,6 +83,10 @@
                 "outcome": "success"
             },
             "related": {
+                "entity": [
+                    "b9814691-9ca1-4e55-a1ac-8ef5dd010ec0",
+                    "a7d5dcbe-0627-4ddf-a2f4-86b6785bcc42"
+                ],
                 "ip": [
                     "1.128.3.4"
                 ]
@@ -178,6 +182,10 @@
                 "outcome": "success"
             },
             "related": {
+                "entity": [
+                    "b9814691-9ca1-4e55-a1ac-8ef5dd010ec0",
+                    "a7d5dcbe-0627-4ddf-a2f4-86b6785bcc42"
+                ],
                 "ip": [
                     "1.128.3.4"
                 ]
@@ -269,6 +277,10 @@
                 "outcome": "success"
             },
             "related": {
+                "entity": [
+                    "b9814691-9ca1-4e55-a1ac-8ef5dd010ec0",
+                    "a7d5dcbe-0627-4ddf-a2f4-86b6785bcc42"
+                ],
                 "ip": [
                     "1.128.3.4"
                 ]

packages/azure/data_stream/auditlogs/_dev/test/pipeline/test-auditlogs-raw.log-expected.json

@@ -65,6 +65,11 @@
             "log": {
                 "level": "Informational"
             },
+            "related": {
+                "entity": [
+                    "8a4de8b5-095c-47d0-a96f-a75130c61d53"
+                ]
+            },
             "tags": [
                 "preserve_original_event"
             ]
@@ -135,6 +140,11 @@
             "log": {
                 "level": "Informational"
             },
+            "related": {
+                "entity": [
+                    "8a4de8b5-095c-47d0-a96f-a75130c61d53"
+                ]
+            },
             "tags": [
                 "preserve_original_event"
             ]
@@ -204,6 +214,11 @@
             "log": {
                 "level": "Informational"
             },
+            "related": {
+                "entity": [
+                    "8a4de8b5-095c-47d0-a96f-a75130c61d53"
+                ]
+            },
             "tags": [
                 "preserve_original_event"
             ]

packages/azure/data_stream/auditlogs/elasticsearch/ingest_pipeline/default.yml

@@ -229,6 +229,35 @@ processors:
   - set:
       field: event.kind
       value: event
+  - append:
+      field: related.entity
+      value: '{{{ azure.auditlogs.properties.initiated_by.user.id }}}'
+      allow_duplicates: false
+      if: ctx.azure?.auditlogs?.properties?.initiated_by?.user?.id != null && ctx.azure.auditlogs.properties.initiated_by.user.id != ''
+  - append:
+      field: related.entity
+      value: '{{{ azure.auditlogs.properties.initiated_by.app.servicePrincipalId }}}'
+      allow_duplicates: false
+      if: ctx.azure?.auditlogs?.properties?.initiated_by?.app?.servicePrincipalId != null && ctx.azure.auditlogs.properties.initiated_by.app.servicePrincipalId != ''
+  - script:
+      description: Appends target resources identifiers to `related.entity`
+      lang: painless
+      source: |
+        ctx.related = ctx.related ?: [:];
+        ctx.related.entity = ctx.related.entity ?: [];
+        if (ctx.azure.auditlogs.properties.target_resources != null) {
+            for (String k : ctx.azure.auditlogs.properties.target_resources.keySet()) {
+                def resource = ctx.azure.auditlogs.properties.target_resources[k];
+                if (resource?.id != null && resource.id != '' && !ctx.related.entity.contains(resource.id)) {
+                        ctx.related.entity.add(resource.id);
+                }
+            }
+        }
+      on_failure: 
+        - set:
+            description: Add error reason
+            field: error.message
+            value: "{{{ _ingest.on_failure_message }}}"
   - pipeline:
       name: '{{ IngestPipeline "azure-shared-pipeline" }}'
 on_failure:

packages/azure/data_stream/auditlogs/fields/fields.yml

@@ -191,3 +191,9 @@
     - name: city_name
       type: keyword
       description: City name.
+- name: related.entity
+  description: >-
+    All the entity identifiers related to the document. If the document contains
+    multiple entities, identifiers belonging to different entities will be present.
+    Example identifiers include cloud resource IDs, ARNs, email addresses, or hostnames.
+  type: keyword