Fix: Prevent 'argument list too long' error in hooks

.githooks/pre-commit/list-staged-files

@@ -1,4 +1,9 @@
 #!/bin/sh
+
+if [ -n "${STAGED_FILES_REFERENCE}" ]; then
+    STAGED_FILES="$(cat "${STAGED_FILES_REFERENCE}")"
+fi
+
 # shellcheck disable=SC2153
 if [ -n "${STAGED_FILES}" ]; then
     echo "* Staged files:"

.githooks/pre-commit/no-set-x

@@ -17,6 +17,8 @@ checkSetX() {
 
 if [ -n "$GITHOOKS_ON_DEMAND_EXEC" ]; then
     STAGED_FILES=$(find . -name '*.sh')
+elif [ -n "${STAGED_FILES_REFERENCE}" ]; then
+    STAGED_FILES="$(cat "${STAGED_FILES_REFERENCE}")"
 fi
 
 for FILE in $STAGED_FILES; do

.githooks/pre-commit/no-tabs

@@ -17,6 +17,8 @@ checkTab() {
 
 if [ -n "$GITHOOKS_ON_DEMAND_EXEC" ]; then
     STAGED_FILES=$(find . -name '*.sh')
+elif [ -n "${STAGED_FILES_REFERENCE}" ]; then
+    STAGED_FILES="$(cat "${STAGED_FILES_REFERENCE}")"
 fi
 
 for FILE in $STAGED_FILES; do

.githooks/pre-commit/no-todo-or-fixme

@@ -18,6 +18,8 @@ checkTodo() {
 
 if [ -n "$GITHOOKS_ON_DEMAND_EXEC" ]; then
     STAGED_FILES=$(find . -type f)
+elif [ -n "${STAGED_FILES_REFERENCE}" ]; then
+    STAGED_FILES="$(cat "${STAGED_FILES_REFERENCE}")"
 fi
 
 for FILE in $STAGED_FILES; do

.githooks/pre-commit/shellcheck

@@ -12,6 +12,8 @@ fi
 SUCCESS=0
 if [ -n "$GITHOOKS_ON_DEMAND_EXEC" ]; then
     STAGED_FILES=$(find . -name '*.sh')
+elif [ -n "${STAGED_FILES_REFERENCE}" ]; then
+    STAGED_FILES="$(cat "${STAGED_FILES_REFERENCE}")"
 fi
 
 for FILE in $STAGED_FILES; do

.githooks/pre-commit/shellcheck-ignore-format

@@ -7,6 +7,8 @@ assertStaged
 SUCCESS=0
 if [ -n "$GITHOOKS_ON_DEMAND_EXEC" ]; then
     STAGED_FILES=$(find . -name '*.sh')
+elif [ -n "${STAGED_FILES_REFERENCE}" ]; then
+    STAGED_FILES="$(cat "${STAGED_FILES_REFERENCE}")"
 fi
 
 for FILE in $STAGED_FILES; do

.githooks/pre-commit/shfmt

@@ -12,6 +12,8 @@ fi
 SUCCESS=0
 if [ -n "$GITHOOKS_ON_DEMAND_EXEC" ]; then
     STAGED_FILES=$(find . -name '*.sh')
+elif [ -n "${STAGED_FILES_REFERENCE}" ]; then
+    STAGED_FILES="$(cat "${STAGED_FILES_REFERENCE}")"
 fi
 
 for FILE in $STAGED_FILES; do

README.md

@@ -58,6 +58,18 @@ done
 
 The `ACMR` filter in the `git diff` will include staged files that are added, copied, modified or renamed.
 
+__Note__: if the list of changes is over 100k characters, then instead of `${STAGED_FILES}` you will get the `${STAGED_FILES_REFERENCE}` variable set instead which will point to a temporary file containing this list. This is to avoid `Argument list too long` errors when executing hooks and other parts of the framework. If you have a large enough repository where this is a concern, you should probably start your hook files by examining if this reference is set, like shown below.
+
+```shell
+if [ -n "${STAGED_FILES_REFERENCE}" ]; then
+    STAGED_FILES="$(cat "${STAGED_FILES_REFERENCE}")"
+fi
+
+for STAGED in ${STAGED_FILES}; do
+    ...
+done
+```
+
 ## Supported hooks
 
 The supported hooks are listed below. Refer to the [Git documentation](https://git-scm.com/docs/githooks) for information on what they do and what parameters they receive.

base-template.sh

@@ -33,6 +33,7 @@ process_git_hook() {
     execute_old_hook_if_available "$@" || return 1
     execute_all_shared_hooks "$@" || return 1
     execute_all_hooks_in "$(pwd)/.githooks" "$@" || return 1
+    cleanup_staged_files_reference_if_exists
 }
 
 #####################################################
@@ -167,7 +168,8 @@ register_repo() {
 #   when available, so hooks can use it if
 #   they want to.
 #
-# Sets the ${STAGED_FILES} variable
+# Sets the ${STAGED_FILES} or the
+#   ${STAGED_FILES_REFERENCE} variable.
 #
 # Returns:
 #   None
@@ -181,7 +183,28 @@ export_staged_files() {
 
     # shellcheck disable=SC2181
     if [ $? -eq 0 ]; then
-        export STAGED_FILES="$CHANGED_FILES"
+        # if the changed files list is over 100k then write it into a temporary file instead
+        # to avoid "Argument list too long" errors
+        if [ "${#CHANGED_FILES}" -gt 100000 ]; then
+            STAGED_FILES_TMP=$(mktemp)
+            echo "$CHANGED_FILES" >"$STAGED_FILES_TMP"
+            export STAGED_FILES_REFERENCE="$STAGED_FILES_TMP"
+        else
+            export STAGED_FILES="$CHANGED_FILES"
+        fi
+    fi
+}
+
+#####################################################
+# Deletes the temporary file that references
+#   the staged files list when it was too big.
+#
+# Returns:
+#   None
+#####################################################
+cleanup_staged_files_reference_if_exists() {
+    if [ -n "$STAGED_FILES_REFERENCE" ] && [ -f "$STAGED_FILES_REFERENCE" ]; then
+        rm "$STAGED_FILES_REFERENCE"
     fi
 }
 

tests/step-101.sh

@@ -0,0 +1,64 @@
+#!/bin/sh
+# Test:
+#   Test that a large number of staged files does not cause an "argument list too long" error.
+
+# run the default install
+sh /var/lib/githooks/install.sh --non-interactive || exit 1
+
+mkdir -p /tmp/test101 && cd /tmp/test101 || exit 2
+git init || exit 3
+
+# set up a pre-commit hook
+# shellcheck disable=SC2016
+mkdir -p .githooks/pre-commit &&
+    echo 'echo "RefFile: $STAGED_FILES_REFERENCE" >> /tmp/test101.out;
+if [ -n "$STAGED_FILES_REFERENCE" ]; then
+     export STAGED_FILES="$(cat $STAGED_FILES_REFERENCE)";
+fi;
+echo "Hook executed for $STAGED_FILES" >> /tmp/test101.out
+' >.githooks/pre-commit/test &&
+    git hooks accept test || exit 4
+
+# Create a large number of files
+mkdir -p some/quite/long/directory/to/put/these/test/files/so/that/our/test/here/can/verify/lengths/better/and/again/some/quite/long/directory/to/put/these/test/files/so/that/our/test/here/can/verify/lengths/better/and/again/some/quite/long/directory/to/put/these/test/files/so/that/our/test/here/can/verify/lengths/better/and/again
+for i in $(seq 1 5000); do
+    touch "some/quite/long/directory/to/put/these/test/files/so/that/our/test/here/can/verify/lengths/better/and/again/some/quite/long/directory/to/put/these/test/files/so/that/our/test/here/can/verify/lengths/better/and/again/some/quite/long/directory/to/put/these/test/files/so/that/our/test/here/can/verify/lengths/better/and/again/some_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_long_file_name_$i.txt"
+done
+
+# Stage the files
+git add . || exit 5
+
+# Commit the files
+# This should not fail with "argument list too long"
+OUTPUT=$(git commit -m "Test commit with a large number of files." 2>&1)
+if echo "$OUTPUT" | grep -q "Argument list too long"; then
+    echo "! $OUTPUT"
+    echo "! The commit failed with 'Argument list too long'"
+    exit 6
+fi
+
+# Check that the commit was successful
+if ! git log -1 | grep -q "Test commit with a large number of files."; then
+    echo "! $OUTPUT"
+    echo "! The commit was not successful"
+    exit 7
+fi
+
+# Check that the hook was executed
+if ! grep -q 'Hook executed for' /tmp/test101.out || ! grep -q '_long_file_name_1012.txt' /tmp/test101.out; then
+    echo "! $OUTPUT"
+    echo "! Could not verify the hook execution"
+    exit 8
+fi
+
+# Make sure the temporary staged files reference file was deleted
+REF_FILE=$(grep 'RefFile:' /tmp/test101.out | awk '{print $2}')
+if [ -z "$REF_FILE" ]; then
+    echo "! Staged files reference file not found in the output"
+    exit 9
+elif [ -f "$REF_FILE" ]; then
+    echo "! Staged files reference file was not cleaned up"
+    exit 9
+fi
+
+exit 0