Scripts to capture dns packets send by specific pod.

scripts/captureDnsPackets/DnsPktCapture2019.yaml

@@ -0,0 +1,112 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: dns-pkt-capture
+  labels:
+    app: dns-pkt-capture
+spec:
+  selector:
+    matchLabels:
+      name: dns-pkt-capture
+  template:
+    metadata:
+      labels:
+        name: dns-pkt-capture
+    spec:
+      securityContext:
+        windowsOptions:
+          hostProcess: true
+          runAsUserName: "NT AUTHORITY\\SYSTEM"
+      hostNetwork: true
+      containers:
+      - name: dns-pkt-capture
+        image: mcr.microsoft.com/windows/nanoserver:1809
+        lifecycle:
+          preStop:
+            exec:
+              command: ["pktmon", "stop"]
+        command:
+        - powershell.exe
+        - -command
+        - |
+            $podPrefix = "tcp-server"
+            $pktmonLogs = "C:\pktmonLogs"
+
+            Write-Host "Stop pktmon if running..."
+            pktmon stop
+
+            $pods = (crictl pods -o json | ConvertFrom-Json).items
+            $podIPs = @()
+            $macAddrs = @()
+
+            foreach($pod in $pods) {
+              if($pod.metadata.name -like "$podPrefix*") { 
+                $podInspect = (crictl inspectp $pod.id | ConvertFrom-Json)
+                $podIP = $podInspect.status.network.ip
+                $podIPs += $podIP
+                $macAddrs += (Get-HnsEndpoint | where IPAddress -EQ $podIP).MacAddress
+              } 
+            }
+
+            if(($macAddrs).Count -Eq 0) {
+              Write-Host "No matching pods. No mac addresses found..."
+              While($true) {
+                Start-Sleep -Seconds 60
+              }
+              return
+            }
+
+            Write-Host "POD IPS : $podIPs"
+            Write-Host "MAC ADDRESSES : $macAddrs"
+
+            $compIds = ""
+
+            foreach($mac in $macAddrs) {
+              $grepped = pktmon list | Select-String $mac
+              $compId = $grepped.ToString().Trim().Split(" ")[0]
+              if($compId -ne "") {
+                if($compIds -eq "") {
+                  $compIds = $compId
+                } else {
+                  $compIds += ","
+                  $compIds += $compId
+                }
+              }
+            }
+
+            if($compIds -Eq "") {
+              Write-Host "No matching pods. No component IDs found..."
+              While($true) {
+                Start-Sleep -Seconds 60
+              }
+              return
+            }
+
+            Write-Host "COMPONENT IDS : $compIds"
+
+            Write-Host "Removing all pktmon filters if anything existing..."
+            pktmon filter remove
+
+            Write-Host "Create DNS Port filter..."
+            pktmon filter add DNSFilter -p 53
+
+            Write-Host "Create a directory for pktmon logs..."
+            remove-item -Recurse -Force $pktmonLogs -ErrorAction Ignore
+            mkdir $pktmonLogs
+            Set-Location $pktmonLogs
+
+            Write-Host "Start pktmon. Command : [pktmon start -c --comp $compIds --pkt-size 0 -m multi-file] ..."
+            pktmon start -c --comp $compIds --pkt-size 0 -m multi-file
+
+            Write-Host "Logs will be available in $pktmonLogs"
+
+            While($true) {
+              Start-Sleep -Seconds 21600
+              Write-Host "Stop pktmon if running..."
+              pktmon stop
+            }
+
+        securityContext:
+          privileged: true
+      nodeSelector:
+        kubernetes.azure.com/os-sku: Windows2019

scripts/captureDnsPackets/DnsPktCapture2022.yaml

@@ -0,0 +1,112 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: dns-pkt-capture
+  labels:
+    app: dns-pkt-capture
+spec:
+  selector:
+    matchLabels:
+      name: dns-pkt-capture
+  template:
+    metadata:
+      labels:
+        name: dns-pkt-capture
+    spec:
+      securityContext:
+        windowsOptions:
+          hostProcess: true
+          runAsUserName: "NT AUTHORITY\\SYSTEM"
+      hostNetwork: true
+      containers:
+      - name: dns-pkt-capture
+        image: mcr.microsoft.com/windows/servercore:ltsc2022
+        lifecycle:
+          preStop:
+            exec:
+              command: ["pktmon", "stop"]
+        command:
+        - powershell.exe
+        - -command
+        - |
+            $podPrefix = "tcp-server"
+            $pktmonLogs = "C:\pktmonLogs"
+
+            Write-Host "Stop pktmon if running..."
+            pktmon stop
+
+            $pods = (crictl pods -o json | ConvertFrom-Json).items
+            $podIPs = @()
+            $macAddrs = @()
+
+            foreach($pod in $pods) {
+              if($pod.metadata.name -like "$podPrefix*") { 
+                $podInspect = (crictl inspectp $pod.id | ConvertFrom-Json)
+                $podIP = $podInspect.status.network.ip
+                $podIPs += $podIP
+                $macAddrs += (Get-HnsEndpoint | where IPAddress -EQ $podIP).MacAddress
+              } 
+            }
+
+            if(($macAddrs).Count -Eq 0) {
+              Write-Host "No matching pods. No mac addresses found..."
+              While($true) {
+                Start-Sleep -Seconds 60
+              }
+              return
+            }
+
+            Write-Host "POD IPS : $podIPs"
+            Write-Host "MAC ADDRESSES : $macAddrs"
+
+            $compIds = ""
+
+            foreach($mac in $macAddrs) {
+              $grepped = pktmon list | Select-String $mac
+              $compId = $grepped.ToString().Trim().Split(" ")[0]
+              if($compId -ne "") {
+                if($compIds -eq "") {
+                  $compIds = $compId
+                } else {
+                  $compIds += ","
+                  $compIds += $compId
+                }
+              }
+            }
+
+            if($compIds -Eq "") {
+              Write-Host "No matching pods. No component IDs found..."
+              While($true) {
+                Start-Sleep -Seconds 60
+              }
+              return
+            }
+
+            Write-Host "COMPONENT IDS : $compIds"
+
+            Write-Host "Removing all pktmon filters if anything existing..."
+            pktmon filter remove
+
+            Write-Host "Create DNS Port filter..."
+            pktmon filter add DNSFilter -p 53
+
+            Write-Host "Create a directory for pktmon logs..."
+            remove-item -Recurse -Force $pktmonLogs -ErrorAction Ignore
+            mkdir $pktmonLogs
+            Set-Location $pktmonLogs
+
+            Write-Host "Start pktmon. Command : [pktmon start -c --comp $compIds --pkt-size 0 -m multi-file] ..."
+            pktmon start -c --comp $compIds --pkt-size 0 -m multi-file
+
+            Write-Host "Logs will be available in $pktmonLogs"
+
+            While($true) {
+              Start-Sleep -Seconds 21600
+              Write-Host "Stop pktmon if running..."
+              pktmon stop
+            }
+
+        securityContext:
+          privileged: true
+      nodeSelector:
+        kubernetes.azure.com/os-sku: Windows2022

scripts/captureDnsPackets/README.md

@@ -0,0 +1,23 @@
+# Capture DNS Packets
+
+> This yaml spawns hostprocess daemonset containers in every node once created. The script inside the container will start pktmon capture for DNS packets originating from every pod with prefix mentioned in powershell variable $podPrefix [Eg: "tcp-server"]
+
+## Start DNS Packet Capture
+
+Update the $podPrefix in DnsPktCapture2019.yaml/DnsPktCapture2022.yaml with right values.
+```
+Line: 28   $podPrefix = "DnsPinger"
+```
+Start pktmon by creating daemon set: DnsPktCapture2019.yaml.
+```
+kubectl create -f .\DnsPktCapture2019.yaml
+```
+Keep the containers running less than 5 hours.
+
+## Stop DNS Packet Capture
+
+Once the issue is reproduced or pktmon running time exceeds 5 hours, stop pktmon by deleting daemon set: DnsPktCapture2019.yaml and wait for 5 minutes.
+```
+kubectl delete -f .\DnsPktCapture2019.yaml
+```
+Packet capture will be generated in “C:\pktmonLogs” directory of each node after 5 minutes. Copy the capture logs out of the node.