more experimental stuff

diff --git a/git-remote-gcrypt b/git-remote-gcrypt
index ee68c8606e81470b6cd8a47bce9a98609a06374b..e9b267283e2e6ffbf0b706de850102abdff067fd 100755 (executable)
--- a/git-remote-gcrypt
+++ b/git-remote-gcrypt
@@ -399,6 +399,13 @@ gitception_new_repo()
 GET()
 {
        print_debug "GET $1 $2 $3"
+       # Check for pre-fetched batch file
+       if [ -f "$Tempdir/batch/$2" ]; then
+               print_debug "Using pre-fetched file $2"
+               cat "$Tempdir/batch/$2" > "$3"
+               return 0
+       fi
+
        if isurl sftp "$1"
        then
                (exec 0</dev/null; curl -s -S -k "$1/$2") > "$3"
@@ -884,8 +891,48 @@ get_verify_decrypt_pack()
 get_pack_files()
 {
        # shellcheck disable=SC3043
-       local pack_id="" r_pack_key_line="" htype_="" pack_="" key_=""
-       while IFS=': ' read -r _ htype_ pack_ # <<here-document
+       local pack_id="" r_pack_key_line="" htype_="" pack_="" key_="" input_data=""
+
+       # Capture input data first (to allow multiple passes)
+       input_data=$(cat)
+
+       # Batch Fetch Optimization for rsync
+       if isurl rsync "$URL" && isnonnull "$input_data"; then
+               mkdir -p "$Tempdir/batch"
+               # Extract pack filenames (field 3) from input lines.
+               # Format: "pack :SHA256:HASH ..." -> we need HASH (which is $pack_)
+               # Input data comes from $pneed_ which was filtered from $Packlist.
+               # Format matches Packlist: "pack :HTYPE:HASH KEY"
+               # xecho "$input_data"
+
+               # We construct a list of files to fetch.
+               # Iterate over input_data to extract pack names
+               (
+                       IFS=$Newline
+                       for line in $input_data; do
+                               # parse "pack :HTYPE:HASH KEY"
+                               # shellcheck disable=SC2086
+                               set -- $line
+                               # $1=pack, $2=:HTYPE:HASH, $3=KEY
+                               # extract HASH from $2
+                               p_hash=${2##*:}
+                               echo "$p_hash"
+                       done
+               ) > "$Tempdir/batch_list"
+
+               if [ -s "$Tempdir/batch_list" ]; then
+                       print_debug "Batch fetching $(line_count "$(cat "$Tempdir/batch_list")") packs..."
+                       (
+                               if [ -n "${GCRYPT_TRACE:-}" ]; then set -x; fi
+                               exec 0</dev/null
+                               # Use --files-from=- to download all packs in one go
+                               rsync -I -W --files-from="$Tempdir/batch_list" "$(rsynclocation "$URL")/" "$Tempdir/batch/" >&2
+                       )
+               fi
+       fi
+
+       # Process packs (sequentially, using cache if available)
+       xecho "$input_data" | while IFS=': ' read -r _ htype_ pack_ # <<here-document
        do
                isnonnull "$pack_" || continue
 

diff --git a/tests/test-rsync-batch.sh b/tests/test-rsync-batch.sh
new file mode 100644 (file)
index 0000000..3c4ceaf
--- /dev/null
+++ b/tests/test-rsync-batch.sh
@@ -0,0 +1,116 @@
+#!/bin/bash
+set -e
+
+# Mock rsync to verify batching
+cat <<'EOF' >"$Tempdir/rsync"
+#!/bin/bash
+TRACELOG="${TRACELOG_PATH}"
+echo "MOCK RSYNC CALLED with args: $*" >> "$TRACELOG"
+# Simulate download if --files-from is used
+if [[ "$*" == *"--files-from"* ]]; then
+    # Parse --files-from argument value
+    # simple parsing
+    for arg in "$@"; do
+        if [[ "$arg" == --files-from=* ]]; then
+            listfile="${arg#*=}"
+            # Create dummy batch files
+            batch_dir="$BatchDir"
+            chmod 700 "$batch_dir"
+            while read -r file; do
+                    mkdir -p "$batch_dir"
+                    echo "dummy content for $file" > "$batch_dir/$file"
+            done < "$listfile"
+        fi
+    done
+fi
+EOF
+chmod +x "$Tempdir/rsync"
+PATH="$Tempdir:$PATH"
+export TRACELOG_PATH="$TRACELOG"
+export BatchDir="$Tempdir/batch"
+
+# Helper to source git-remote-gcrypt functions
+source_script() {
+       # Find the line number where the main logic starts
+       # We look for the "if [ "$NAME" = "dummy-gcrypt-check" ]; then" line
+       start_line=$(grep -n "dummy-gcrypt-check" ./git-remote-gcrypt | cut -d: -f1 || echo "")
+       echo "DEBUG: start_line=$start_line"
+       if [ -z "$start_line" ]; then
+               echo "ERROR: Could not find start line for sourcing."
+               grep "dummy-gcrypt-check" ./git-remote-gcrypt || echo "Grep failed to find pattern"
+               exit 1
+       fi
+       # Extract everything up to that line (exclusive)
+       head -n "$((start_line - 1))" ./git-remote-gcrypt >"$Tempdir/funcs.sh"
+       source "$Tempdir/funcs.sh"
+}
+
+# Test Setup
+Tempdir=$(mktemp -d)
+Localdir="$Tempdir/local"
+mkdir -p "$Localdir"
+mkdir -p "$Localdir/objects/pack"
+TRACELOG="$Tempdir/trace.log"
+URL="rsync://example.com/repo"
+GITCEPTION=""
+Hex40="0000000000000000000000000000000000000000"
+
+source_script
+
+# Redefine other deps we don't want to run
+gpg_hash() { echo "mockhash"; }
+DECRYPT() { cat; }
+check_safety() { :; }
+# Mock get_verify_decrypt_pack to simplify (since we care about the rsync call)
+get_verify_decrypt_pack() {
+       # Just call GET to trigger our logic
+       GET "$URL" "$2" "$Tempdir/packF"
+}
+# Mock git index-pack
+git() { :; }
+
+echo "Running Batch Test..."
+
+# Create a mock Packlist with 3 packs
+cat <<EOF >"$Tempdir/input_packs"
+pack :SHA256:pack1 hash1
+pack :SHA256:pack2 hash2
+pack :SHA256:pack3 hash3
+EOF
+
+Packlist=$(cat "$Tempdir/input_packs")
+echo "DEBUG input_packs content:"
+cat "$Tempdir/input_packs"
+
+echo "DEBUG Running get_pack_files..."
+# Run get_pack_files with input from the "missing packs" (simulated)
+# In real flow, this input comes from pneed_
+cat "$Tempdir/input_packs" | get_pack_files
+
+echo "DEBUG batch_list content:"
+if [ -f "$Tempdir/batch/batch_list" ]; then
+       cat "$Tempdir/batch/batch_list"
+else
+       # logic uses $Tempdir/batch_list not $Tempdir/batch/batch_list
+       if [ -f "$Tempdir/batch_list" ]; then
+               cat "$Tempdir/batch_list"
+       else
+               echo "batch_list not found!"
+       fi
+fi
+
+# Verify trace
+count=$(grep -c "MOCK RSYNC CALLED" "$TRACELOG")
+echo "Rsync called $count times."
+
+if [ "$count" -eq 1 ]; then
+       echo "SUCCESS: Rsync called exactly once."
+       cat "$TRACELOG"
+else
+       echo "FAILURE: Rsync called $count times (expected 1)."
+       cat "$TRACELOG"
+       exit 1
+fi
+
+# rm -rf "$Tempdir"
+echo "Trace log at $TRACELOG"