From f476b9f24006d4ae0f1ef304dc0a4f9ffdd775ae Mon Sep 17 00:00:00 2001 From: Vince JV <1276544+vincejv@users.noreply.github.com> Date: Tue, 16 Sep 2025 00:53:07 +0800 Subject: [PATCH 1/4] ech-rotate: add rollback mechanism --- ech-rotate.sh | 97 ++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 85 insertions(+), 12 deletions(-) diff --git a/ech-rotate.sh b/ech-rotate.sh index ee22f68..bc62629 100644 --- a/ech-rotate.sh +++ b/ech-rotate.sh @@ -24,6 +24,20 @@ SUBDOMAINS="${SUBDOMAINS:?Must set SUBDOMAINS (space-separated list)}" ECH_ROTATION="${ECH_ROTATION:-false}" # default: disabled KEEP_KEYS="${KEEP_KEYS:-3}" # number of old timestamped keys to keep +reload_nginx() { + if [[ -f "$PIDFILE" ]]; then + PID=$(cat "$PIDFILE") + if kill -0 "$PID" 2>/dev/null; then + kill -SIGHUP "$PID" + log "Reloaded nginx (pid $PID)" + else + log "Nginx PID $PID is not yet running" + fi + else + log "PID file not found: $PIDFILE" + fi +} + rotate_ech() { log "Rotating ECH keys..." mkdir -p "$ECH_DIR" || log "Failed to create $ECH_DIR" @@ -35,27 +49,86 @@ rotate_ech() { # 2. Ensure symlinks exist, fill missing ones with latest cd "$ECH_DIR" || return 1 + + # Before rotation, capture current symlinks for rollback + old_latest=$(readlink -f "$DOMAIN.ech" 2>/dev/null || true) + old_previous=$(readlink -f "$DOMAIN.previous.ech" 2>/dev/null || true) + old_stale=$(readlink -f "$DOMAIN.stale.ech" 2>/dev/null || true) + ln -sf "$(readlink "$DOMAIN.previous.ech")" "$DOMAIN.stale.ech" ln -sf "$(readlink "$DOMAIN.ech")" "$DOMAIN.previous.ech" ln -sf "$(basename "$NEW_KEY")" "$DOMAIN.ech" log "Symlinks rotated: ech -> $(readlink "$DOMAIN.ech"), previous.ech -> $(readlink "$DOMAIN.previous.ech"), stale.ech -> $(readlink "$DOMAIN.stale.ech")" # 4. Reload nginx - if [[ -f "$PIDFILE" ]]; then - PID=$(cat "$PIDFILE") - if kill -0 "$PID" 2>/dev/null; then - kill -SIGHUP "$PID" - log "Reloaded nginx (pid $PID)" - else - log "Nginx PID $PID is not yet running" - fi - else - log "PID file not found: $PIDFILE" - fi + reload_nginx + + # 5. Backup DNS records for rollback + backup_file=$(mktemp) + curl -s -X GET "$CF_ZONE_URL/$CF_ZONE_ID/dns_records?type=HTTPS" \ + -H "Authorization: Bearer $CF_API_TOKEN" \ + -H "Content-Type: application/json" \ + > "$backup_file" # 5-6. Update DNS Records source /usr/local/bin/update_https_records.sh - update_https_records || { log "Error: Failed to update HTTPS DNS records"; return 1; } + # DNS update + if ! update_https_records; then + log "Error: Failed to update HTTPS DNS records, rolling back ECH keys in nginx..." + + # Roll back symlinks to old state + [[ -n "$old_latest" ]] && ln -sf "$(basename "$old_latest")" "$DOMAIN.ech" + [[ -n "$old_previous" ]] && ln -sf "$(basename "$old_previous")" "$DOMAIN.previous.ech" + [[ -n "$old_stale" ]] && ln -sf "$(basename "$old_stale")" "$DOMAIN.stale.ech" + + # Optionally delete the new key if not needed + rm -f -- "$NEW_KEY" + log "Deleted the newly generated key: ${NEW_KEY}" + reload_nginx + + log "Rolling back DNS updates" + # Get current state + current_file=$(mktemp) + curl -s -X GET "$CF_ZONE_URL/$CF_ZONE_ID/dns_records?type=HTTPS" \ + -H "Authorization: Bearer $CF_API_TOKEN" \ + -H "Content-Type: application/json" \ + > "$current_file" + + # Collect rollback candidates + ROLLBACK=() + while IFS= read -r rec; do + rec_id=$(jq -r '.id' <<<"$rec") + cur=$(jq -c --arg id "$rec_id" '.result[] | select(.id==$id)' "$current_file") + + if [[ "$rec" != "$cur" ]]; then + log "Will restore record $rec_id" + ROLLBACK+=("$rec") + fi + done < <(jq -c '.result[]' "$backup_file") + + if [ "${#ROLLBACK[@]}" -gt 0 ]; then + # Build batch body with puts + PUTS_JSON=$(printf '%s\n' "${ROLLBACK[@]}" | jq -s '.') + BATCH=$(jq -n --argjson puts "$PUTS_JSON" '{puts:$puts}') + + log "Submitting rollback batch with ${#ROLLBACK[@]} records" + CF_RESULT=$(curl -s -X POST "$CF_ZONE_URL/$CF_ZONE_ID/dns_records/batch" \ + -H "Authorization: Bearer $CF_API_TOKEN" \ + -H "Content-Type: application/json" \ + --data "$BATCH") + + if echo "$CF_RESULT" | grep -q '"success":true'; then + log "Rollback batch applied successfully" + else + log "Rollback batch failed: $CF_RESULT" + fi + else + log "No changes detected, nothing to rollback" + fi + + log "ECH key rotation failed, rollback successful" + return 1 + fi # 7. Cleanup old keys (keep latest N timestamped files, skip symlink targets) cd "$ECH_DIR" || return 1 From d1b85e80aa1688edef8e98930c4e564914056f12 Mon Sep 17 00:00:00 2001 From: Vince JV <1276544+vincejv@users.noreply.github.com> Date: Tue, 16 Sep 2025 00:53:21 +0800 Subject: [PATCH 2/4] dns-update: use batch api --- update_https_records.sh | 83 +++++++++++++++++++++-------------------- 1 file changed, 43 insertions(+), 40 deletions(-) diff --git a/update_https_records.sh b/update_https_records.sh index 8ed8881..b64d1cb 100644 --- a/update_https_records.sh +++ b/update_https_records.sh @@ -3,7 +3,7 @@ update_https_records() { log "Updating DNS HTTPS records..." - + # 1. Extract ECHConfig from new key ECHCONFIG=$(awk '/-----BEGIN ECHCONFIG-----/{flag=1;next}/-----END ECHCONFIG-----/{flag=0}flag' "$NEW_KEY" | tr -d '\n') if [[ -z "$ECHCONFIG" ]]; then @@ -23,53 +23,56 @@ update_https_records() { # Common curl options # 3. Publish HTTPS DNS record to Cloudflare (update only ech field) + # Common curl options + # use the DNS batch API schema (posts, patches, puts, deletes) CURL_OPTS=(-s --retry 5 --retry-delay 2 --retry-connrefused) + + POSTS=() + PATCHES=() + for d in "${SUBDOMAINS_ARR[@]}"; do - RECORD=$(curl "${CURL_OPTS[@]}" -X GET "$CF_ZONE_URL/$CF_ZONE_ID/dns_records?type=HTTPS&name=$d" \ + # fetch existing HTTPS record (if any) + RECORD_RAW=$(curl "${CURL_OPTS[@]}" -G \ + --data-urlencode "type=HTTPS" --data-urlencode "name=$d" \ -H "Authorization: Bearer $CF_API_TOKEN" \ - -H "Content-Type: application/json") - - RECORD_ID=$(echo "$RECORD" | jq -r '.result[0].id') - RECORD_DATA=$(echo "$RECORD" | jq '.result[0].data') + -H "Content-Type: application/json" \ + "$CF_ZONE_URL/$CF_ZONE_ID/dns_records") - if [[ "$RECORD_ID" == "null" ]]; then - log "No HTTPS record found for $d, inserting new HTTPS record" - UPDATED_DATA=$(jq -n --arg ech "$ECHCONFIG" '{ - value: "ech=\"\($ech)\"", - priority: "1", - target: ".", - }') - METHOD="POST" - URL="$CF_ZONE_URL/$CF_ZONE_ID/dns_records" + # no existing record -> create a posts entry + if [ "$(echo "$RECORD_RAW" | jq -r '.result | length')" = "0" ]; then + POSTS+=( "$(jq -n --arg name "$d" --arg ech "$ECHCONFIG" '{ + name: $name, + type: "HTTPS", + data: { value: ("ech=\"" + $ech + "\""), priority: "1", target: "." } + }')" ) else - log "HTTPS record found for $d, updating ech public key" - # Replace the ech record in HTTPS DNS record - UPDATED_DATA=$(echo "$RECORD_DATA" \ - | jq --arg ECH "$ECHCONFIG" ' - if .value | test("ech=") - then .value |= sub("ech=\"[^\"]*\""; "ech=\"\($ECH)\"") - else .value += " ech=\"\($ECH)\"" - end - ') - METHOD="PUT" - URL="$CF_ZONE_URL/$CF_ZONE_ID/dns_records/$RECORD_ID" + # existing record -> produce a patch object by taking the whole record and only updating data.value + PATCHES+=( "$(echo "$RECORD_RAW" | jq --arg ECH "$ECHCONFIG" ' + .result[0] | + # ensure .data.value exists and then replace or append ech="..." + (.data.value // "") as $v | + if ($v | test("ech=")) then + .data.value |= sub("ech=\"[^\"]*\""; "ech=\"\($ECH)\"") + else + .data.value |= (. + " ech=\"\($ECH)\"") + end + ')" ) fi + done - UPDATED_DATA=$(jq -n --arg name "$d" --argjson data "$UPDATED_DATA" '{type:"HTTPS", name:$name, data:$data}') - log "Pushing updated HTTPS record for $d: $UPDATED_DATA" + # build JSON arrays (empty arrays if there are no items) + POSTS_JSON=$(printf '%s\n' "${POSTS[@]}" | jq -s '.' ) + PATCHES_JSON=$(printf '%s\n' "${PATCHES[@]}" | jq -s '.' ) - sleep 0.3 + # final batch body: include only the arrays you need (Cloudflare accepts empty arrays) + BATCH=$(jq -n --argjson posts "$POSTS_JSON" --argjson patches "$PATCHES_JSON" '{posts:$posts, patches:$patches}') - CF_RESULT=$(curl "${CURL_OPTS[@]}" -X "$METHOD" "$URL" \ - -H "Authorization: Bearer $CF_API_TOKEN" \ - -H "Content-Type: application/json" \ - --data "$UPDATED_DATA") || log "Failed to push DNS record for $d" + # send the batch + CF_RESULT=$(curl "${CURL_OPTS[@]}" -X POST "$CF_ZONE_URL/$CF_ZONE_ID/dns_records/batch" \ + -H "Authorization: Bearer $CF_API_TOKEN" \ + -H "Content-Type: application/json" \ + --data "$BATCH") - if echo "$CF_RESULT" | grep -q '"success":true'; then - log "Updated ech for $d (record $RECORD_ID)" - else - log "Failed to update ech for $d: $CF_RESULT" - fi - sleep 0.3 - done + # show response + echo "$CF_RESULT" | jq -C . } \ No newline at end of file From 6952222a10c74ceb228f6e87dc9ff204c920b2d0 Mon Sep 17 00:00:00 2001 From: Vince JV <1276544+vincejv@users.noreply.github.com> Date: Tue, 16 Sep 2025 01:14:07 +0800 Subject: [PATCH 3/4] improve logging of dns-update and ech-rotate --- ech-rotate.sh | 1 + update_https_records.sh | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ech-rotate.sh b/ech-rotate.sh index bc62629..527ad1a 100644 --- a/ech-rotate.sh +++ b/ech-rotate.sh @@ -112,6 +112,7 @@ rotate_ech() { BATCH=$(jq -n --argjson puts "$PUTS_JSON" '{puts:$puts}') log "Submitting rollback batch with ${#ROLLBACK[@]} records" + log "Rollback batch records: $BATCH" CF_RESULT=$(curl -s -X POST "$CF_ZONE_URL/$CF_ZONE_ID/dns_records/batch" \ -H "Authorization: Bearer $CF_API_TOKEN" \ -H "Content-Type: application/json" \ diff --git a/update_https_records.sh b/update_https_records.sh index b64d1cb..d9ba6a8 100644 --- a/update_https_records.sh +++ b/update_https_records.sh @@ -66,6 +66,7 @@ update_https_records() { # final batch body: include only the arrays you need (Cloudflare accepts empty arrays) BATCH=$(jq -n --argjson posts "$POSTS_JSON" --argjson patches "$PATCHES_JSON" '{posts:$posts, patches:$patches}') + log "Submitting API curl batch update: $BATCH" # send the batch CF_RESULT=$(curl "${CURL_OPTS[@]}" -X POST "$CF_ZONE_URL/$CF_ZONE_ID/dns_records/batch" \ @@ -74,5 +75,5 @@ update_https_records() { --data "$BATCH") # show response - echo "$CF_RESULT" | jq -C . + log "$CF_RESULT" | jq -C . } \ No newline at end of file From 7da8848d57053dea01ed4c4a1dbb8792d758fa72 Mon Sep 17 00:00:00 2001 From: Vince JV <1276544+vincejv@users.noreply.github.com> Date: Tue, 16 Sep 2025 12:18:37 +0800 Subject: [PATCH 4/4] enhance changes --- ech-rotate.sh | 4 +++- update_https_records.sh | 45 +++++++++++++++++++++++------------------ 2 files changed, 28 insertions(+), 21 deletions(-) diff --git a/ech-rotate.sh b/ech-rotate.sh index 527ad1a..650a9de 100644 --- a/ech-rotate.sh +++ b/ech-rotate.sh @@ -102,7 +102,9 @@ rotate_ech() { if [[ "$rec" != "$cur" ]]; then log "Will restore record $rec_id" - ROLLBACK+=("$rec") + # Make sure to keep only fields CF accepts, including id + clean=$(jq '{id, type, name, content, ttl, proxied, priority, data, comment, tags}' <<<"$rec") + ROLLBACK+=("$clean") fi done < <(jq -c '.result[]' "$backup_file") diff --git a/update_https_records.sh b/update_https_records.sh index d9ba6a8..1e10249 100644 --- a/update_https_records.sh +++ b/update_https_records.sh @@ -30,32 +30,37 @@ update_https_records() { POSTS=() PATCHES=() + # Fetch all HTTPS records once + ALL_RECORDS=$(curl "${CURL_OPTS[@]}" -G \ + --data-urlencode "type=HTTPS" \ + -H "Authorization: Bearer $CF_API_TOKEN" \ + -H "Content-Type: application/json" \ + "$CF_ZONE_URL/$CF_ZONE_ID/dns_records") + for d in "${SUBDOMAINS_ARR[@]}"; do - # fetch existing HTTPS record (if any) - RECORD_RAW=$(curl "${CURL_OPTS[@]}" -G \ - --data-urlencode "type=HTTPS" --data-urlencode "name=$d" \ - -H "Authorization: Bearer $CF_API_TOKEN" \ - -H "Content-Type: application/json" \ - "$CF_ZONE_URL/$CF_ZONE_ID/dns_records") + RECORD_RAW=$(jq --arg name "$d" '.result[] | select(.name==$name)' <<<"$ALL_RECORDS") - # no existing record -> create a posts entry - if [ "$(echo "$RECORD_RAW" | jq -r '.result | length')" = "0" ]; then + if [ -z "$RECORD_RAW" ]; then + # no existing record -> prepare POST POSTS+=( "$(jq -n --arg name "$d" --arg ech "$ECHCONFIG" '{ - name: $name, - type: "HTTPS", - data: { value: ("ech=\"" + $ech + "\""), priority: "1", target: "." } + name: $name, + type: "HTTPS", + data: { value: ("ech=\"" + $ech + "\""), priority: "1", target: "." } }')" ) else - # existing record -> produce a patch object by taking the whole record and only updating data.value + # existing record -> produce a minimal patch object PATCHES+=( "$(echo "$RECORD_RAW" | jq --arg ECH "$ECHCONFIG" ' - .result[0] | - # ensure .data.value exists and then replace or append ech="..." - (.data.value // "") as $v | - if ($v | test("ech=")) then - .data.value |= sub("ech=\"[^\"]*\""; "ech=\"\($ECH)\"") - else - .data.value |= (. + " ech=\"\($ECH)\"") - end + .result[0] | { + id, + data: ( + if (.data.value // "") | test("ech=") then + .data.value |= sub("ech=\"[^\"]*\""; "ech=\"\($ECH)\"") + else + .data.value |= (. + " ech=\"\($ECH)\"") + end + | . + ) + } ')" ) fi done