Skip to content

Commit 4c71df3

Browse files
pk910pk910
committed
add summary script
1 parent 2c5a00c commit 4c71df3

File tree

4 files changed

+182
-1
lines changed

4 files changed

+182
-1
lines changed

‎.github/workflows/_shared-run-test.yaml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -312,6 +312,8 @@ jobs:
312312
cd $tempdir/dump
313313
echo "dump_dir=$tempdir/dump" >> $GITHUB_OUTPUT
314314
315+
echo "dump_artifact=enclave-dump-assertoor-${{ github.run_id }}-${{ inputs.id }}" >> $GITHUB_OUTPUT
316+
315317
kurtosis enclave dump $enclave_name
316318
317319
- name: Generate failure summary via ChatGPT (on failure)
@@ -561,6 +563,14 @@ jobs:
561563
EOF
562564
)
563565
566+
summary_json="{}"
567+
summary_json=$(echo "$summary_json" | jq -c --arg value "$test_result" '.test_result = $value')
568+
summary_json=$(echo "$summary_json" | jq -c --arg value "$test_status" '.test_status = $value')
569+
summary_json=$(echo "$summary_json" | jq -c --arg value "$failed_test_status" '.failed_test_status = $value')
570+
summary_json=$(echo "$summary_json" | jq -c --arg value "$chatgpt_summary" '.chatgpt_summary = $value')
571+
summary_json=$(echo "$summary_json" | jq -c --arg value "${{ steps.enclave_dump.outputs.dump_artifact }}" '.enclave_dump_artifact = $value')
572+
echo "$summary_json" > ./temp/summary.json
573+
564574
echo "Test Result: $test_result"
565575
echo "$test_status"
566576
@@ -582,4 +592,9 @@ jobs:
582592
583593
exit 1 # fail action
584594
fi
595+
- name: Upload summary artifact
596+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
597+
with:
598+
name: "summary-${{ inputs.id }}"
599+
path: "./temp/summary.json"
585600

‎.github/workflows/_shared-run.yaml‎

Lines changed: 54 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,20 +34,24 @@ jobs:
3434
name: ${{ steps.config.outputs.name }}
3535
clients: ${{ steps.config.outputs.clients }}
3636
kurtosis: ${{ steps.config.outputs.kurtosis }}
37+
kurtosis_version: ${{ steps.config.outputs.kurtosis_version }}
3738
kurtosis_branch: ${{ steps.config.outputs.kurtosis_branch }}
3839
worker: ${{ steps.config.outputs.worker }}
3940
backend: ${{ steps.config.outputs.backend }}
41+
send_notification: ${{ steps.config.outputs.send_notification }}
4042
pairs: ${{ steps.config.outputs.pairs }}
4143
pairsidx: ${{ steps.config.outputs.pairsidx }}
4244
assertoor_tests: ${{ steps.config.outputs.assertoor_tests }}
4345
kubernetes_cluster: ${{ steps.config.outputs.kubernetes_cluster }}
4446
kubernetes_storageclass: ${{ steps.config.outputs.kubernetes_storageclass }}
4547
kubeconfig: ${{ steps.kubeconfig.outputs.kubeconfig }}
48+
summary_script: ${{ steps.config.outputs.summary_script }}
4649
steps:
4750
- name: "Parse test configuration"
4851
id: config
4952
shell: bash
5053
run: |
54+
send_notification="${{ inputs.send_notification }}"
5155
testcfg=$(
5256
cat <<"EOF"
5357
${{ inputs.config }}
@@ -62,10 +66,12 @@ jobs:
6266
kurtosis_branch="$(echo "$testcfg" | jq -r '.kurtosis_branch // ""')"
6367
worker="$(echo "$testcfg" | jq -c .worker)"
6468
backend="$(echo "$testcfg" | jq -r .backend)"
69+
skip_notifications="$(echo "$testcfg" | jq -r .skip_notifications)"
6570
kubernetes_cluster="$(echo "$testcfg" | jq -r .kubernetes.cluster)"
6671
kubernetes_storageclass="$(echo "$testcfg" | jq -r .kubernetes.storageClass)"
6772
pairs="$(echo "$testcfg" | jq -c .clientPairs)"
6873
assertoor_tests="$(echo "$testcfg" | jq -c .assertoorTests)"
74+
summary_script="$(echo "$testcfg" | jq -c .summaryScript)"
6975
7076
echo "ID: $id"
7177
echo "id=$(echo "$id")" >> $GITHUB_OUTPUT
@@ -92,13 +98,22 @@ jobs:
9298
echo "kubernetes_cluster=" >> $GITHUB_OUTPUT
9399
echo "kubernetes_storageclass=" >> $GITHUB_OUTPUT
94100
fi
101+
if [ "$skip_notifications" == "true" ]; then
102+
echo "Notifications: false"
103+
echo "send_notification=false" >> $GITHUB_OUTPUT
104+
else
105+
echo "Notifications: $send_notification"
106+
echo "send_notification=$send_notification" >> $GITHUB_OUTPUT
107+
fi
95108
echo "Client Pairs:"
96109
echo "$pairs" | yq -P
97110
echo "pairs=$(echo "$pairs" | jq -c 'to_entries | map({pairs:.value, index:.key})')" >> $GITHUB_OUTPUT
98111
echo "pairsidx=$(echo "$pairs" | jq -c 'to_entries | map(.key)')" >> $GITHUB_OUTPUT
99112
echo "Assertoor Tests:"
100113
echo "$assertoor_tests" | yq -P
101114
echo "assertoor_tests=$(echo "$assertoor_tests")" >> $GITHUB_OUTPUT
115+
echo "Summary Script: $summary_script"
116+
echo "summary_script=$(echo "$summary_script")" >> $GITHUB_OUTPUT
102117
103118
# Generate kubeconfig for kubernetes backend
104119
- name: "Install rancher CLI"
@@ -147,10 +162,48 @@ jobs:
147162
kubeStorageClass: ${{ needs.get_test.outputs.kubernetes_storageclass }}
148163
clients: ${{ needs.get_test.outputs.clients }}
149164
assertoor_tests: ${{ needs.get_test.outputs.assertoor_tests }}
150-
send_notification: ${{ inputs.send_notification }}
165+
send_notification: ${{ needs.get_test.outputs.send_notification }}
151166
use_chatgpt: ${{ inputs.use_chatgpt }}
152167
secrets:
153168
KUBECONFIG: ${{ needs.get_test.outputs.kubeconfig }}
154169
DISCORD_HOOK: ${{ secrets.DISCORD_HOOK }}
155170
CHATGPT_KEY: ${{ secrets.CHATGPT_KEY }}
156171

172+
generate_summary:
173+
name: "Generate Summary"
174+
if: ${{ always() && needs.get_test.outputs.summary_script != '' }}
175+
runs-on: ubuntu-latest
176+
needs: [get_test, run_test]
177+
steps:
178+
- name: Checkout Repository
179+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
180+
- name: Get summary artifacts
181+
id: "summary_ids"
182+
run: |
183+
artifacts=$(gh api /repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts --jq '[.artifacts[] | {id, name}]')
184+
summary_artifacts=$(echo "$artifacts" | jq -c --arg id "${{ needs.get_test.outputs.id }}" '[.[] | select(.name | test("^summary-" + $id + "-")) | .id]' | tr -d "[]")
185+
echo "artifact_ids=$summary_artifacts" >> $GITHUB_OUTPUT
186+
mkdir ./summaries
187+
env:
188+
GH_TOKEN: ${{ github.token }}
189+
- name: Download summary artifacts
190+
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
191+
with:
192+
artifact-ids: ${{ steps.summary_ids.outputs.artifact_ids }}
193+
path: "./summaries"
194+
- name: "Parse test configuration"
195+
id: config
196+
shell: bash
197+
run: |
198+
summary_script="${{ needs.get_test.outputs.summary_script }}"
199+
pairs=$(
200+
cat <<"EOF"
201+
${{ needs.get_test.outputs.pairs }}
202+
EOF
203+
)
204+
205+
if [ -f "$summary_script" ]; then
206+
source $summary_script
207+
generate_summary
208+
fi
209+

‎summary-scripts/vc-compatibility.sh‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
#!/usr/bin/env bash
2+
set -eo pipefail
3+
4+
generate_summary() {
5+
declare -A result
6+
declare -A seen_bn
7+
declare -A seen_vc
8+
9+
# Normalize a client name (capitalize first letter)
10+
normalize() {
11+
echo "$1" | awk '{print toupper(substr($0,1,1)) tolower(substr($0,2))}'
12+
}
13+
14+
# Loop through all pair entries
15+
while read -r item; do
16+
index=$(echo "$item" | jq -r '.index')
17+
pair_string=$(echo "$item" | jq -r '.pairs')
18+
19+
# Check if summary exists and was successful
20+
summary_path="./summaries/summary-vc-compatibility-$index/summary.json"
21+
if [[ -f "$summary_path" ]]; then
22+
test_result=$(jq -r '.test_result' "$summary_path")
23+
else
24+
test_result="missing"
25+
fi
26+
27+
IFS=',' read -ra pairs <<< "$pair_string"
28+
if [[ "$test_result" == "success" || "$test_result" == "failed" ]]; then
29+
for pair in "${pairs[@]}"; do
30+
if [[ "$pair" == */* ]]; then
31+
bn_raw=$(echo "$pair" | cut -d'/' -f1)
32+
vc_raw=$(echo "$pair" | cut -d'/' -f2 | cut -d'-' -f1)
33+
else
34+
bn_raw=$(echo "$pair" | cut -d'-' -f1)
35+
vc_raw="$bn_raw"
36+
fi
37+
38+
bn=$(normalize "$bn_raw")
39+
vc=$(normalize "$vc_raw")
40+
41+
seen_bn["$bn"]=1
42+
seen_vc["$vc"]=1
43+
result["$bn,$vc"]="✅"
44+
done
45+
fi
46+
47+
if [[ "$test_result" == "failed" ]]; then
48+
# Parse failed client pairs from failed_test_status
49+
if [[ -f "$summary_path" ]]; then
50+
failed_lines=$(jq -r '.failed_test_status' "$summary_path" | grep 'check_consensus_block_proposals' | grep 'failed')
51+
52+
while read -r line; do
53+
# Extract from "Wait for block proposal from X-<el>-<bn>[-<vc>]"
54+
client_part=$(echo "$line" | grep -oE '[0-9]+-[a-z0-9-]+-[a-z0-9-]+$' | cut -d'-' -f2-)
55+
56+
# Split out bn and optional vc
57+
IFS='-' read -ra parts <<< "$client_part"
58+
bn_raw="${parts[0]}"
59+
vc_raw="${parts[1]:-${parts[0]}}" # fallback to bn if vc not specified
60+
61+
bn=$(normalize "$bn_raw")
62+
vc=$(normalize "$vc_raw")
63+
64+
seen_bn["$bn"]=1
65+
seen_vc["$vc"]=1
66+
result["$bn,$vc"]="❌"
67+
done <<< "$failed_lines"
68+
fi
69+
fi
70+
done < <(echo "$pairs" | jq -c '.[]')
71+
72+
# Collect sorted BNs and VCs
73+
bns=($(printf "%s\n" "${!seen_bn[@]}" | sort))
74+
vcs=($(printf "%s\n" "${!seen_vc[@]}" | sort))
75+
76+
# Initialize missing pairs as ❌
77+
for bn in "${bns[@]}"; do
78+
for vc in "${vcs[@]}"; do
79+
result["$bn,$vc"]="${result["$bn,$vc"]:-❌}"
80+
done
81+
done
82+
83+
# Generate markdown summary
84+
header="| |"
85+
for vc in "${vcs[@]}"; do
86+
header+=" ${vc} VC |"
87+
done
88+
89+
separator="|---------------|"
90+
for _ in "${vcs[@]}"; do
91+
separator+="-------------|"
92+
done
93+
94+
rows=""
95+
for bn in "${bns[@]}"; do
96+
row="| ${bn} BN"
97+
for vc in "${vcs[@]}"; do
98+
row+=" | ${result["$bn,$vc"]}"
99+
done
100+
row+=" |"
101+
rows+="${row}"$'\n'
102+
done
103+
104+
# Output to GitHub Actions summary
105+
{
106+
echo "$header"
107+
echo "$separator"
108+
echo "$rows"
109+
} >> "$GITHUB_STEP_SUMMARY"
110+
111+
}

‎tests.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ tests:
5353
kurtosis_branch: "5.0.1"
5454
worker: ubuntu-latest
5555
backend: docker
56+
skip_notifications: true
5657
clientPairs:
5758
# EL: [geth]
5859
# CL: [lighthouse,prysm,teku,lodestar,nimbus,grandine]
@@ -65,6 +66,7 @@ tests:
6566
- grandine/lighthouse-geth,grandine/prysm-geth,grandine/teku-geth,grandine/lodestar-geth,grandine/nimbus-geth
6667
assertoorTests:
6768
- assertoor-tests/block-proposal-check.yaml
69+
summaryScript: "summary-scripts/vc-compatibility.sh"
6870

6971
# Check validator client combinations with stable clients
7072
- id: "vc-compatibility-stable"

0 commit comments

Comments
 (0)