From: Shane Jaroch <chown_tee@proton.me>
Date: Wed, 7 Jan 2026 22:49:27 +0000 (-0500)
Subject: more experimental stuff
X-Git-Url: https://git.nutra.tk/v2?a=commitdiff_plain;h=refs%2Fheads%2Ffeature%2Fbatch-rsync;p=gamesguru%2Fgit-remote-gcrypt.git

more experimental stuff
---

diff --git a/git-remote-gcrypt b/git-remote-gcrypt
index ee68c86..e9b2672 100755
--- a/git-remote-gcrypt
+++ b/git-remote-gcrypt
@@ -399,6 +399,13 @@ gitception_new_repo()
 GET()
 {
 	print_debug "GET $1 $2 $3"
+	# Check for pre-fetched batch file
+	if [ -f "$Tempdir/batch/$2" ]; then
+		print_debug "Using pre-fetched file $2"
+		cat "$Tempdir/batch/$2" > "$3"
+		return 0
+	fi
+
 	if isurl sftp "$1"
 	then
 		(exec 0</dev/null; curl -s -S -k "$1/$2") > "$3"
@@ -884,8 +891,48 @@ get_verify_decrypt_pack()
 get_pack_files()
 {
 	# shellcheck disable=SC3043
-	local pack_id="" r_pack_key_line="" htype_="" pack_="" key_=""
-	while IFS=': ' read -r _ htype_ pack_ # <<here-document
+	local pack_id="" r_pack_key_line="" htype_="" pack_="" key_="" input_data=""
+
+	# Capture input data first (to allow multiple passes)
+	input_data=$(cat)
+
+	# Batch Fetch Optimization for rsync
+	if isurl rsync "$URL" && isnonnull "$input_data"; then
+		mkdir -p "$Tempdir/batch"
+		# Extract pack filenames (field 3) from input lines.
+		# Format: "pack :SHA256:HASH ..." -> we need HASH (which is $pack_)
+		# Input data comes from $pneed_ which was filtered from $Packlist.
+		# Format matches Packlist: "pack :HTYPE:HASH KEY"
+		# xecho "$input_data"
+
+		# We construct a list of files to fetch.
+		# Iterate over input_data to extract pack names
+		(
+			IFS=$Newline
+			for line in $input_data; do
+				# parse "pack :HTYPE:HASH KEY"
+				# shellcheck disable=SC2086
+				set -- $line
+				# $1=pack, $2=:HTYPE:HASH, $3=KEY
+				# extract HASH from $2
+				p_hash=${2##*:}
+				echo "$p_hash"
+			done
+		) > "$Tempdir/batch_list"
+
+		if [ -s "$Tempdir/batch_list" ]; then
+			print_debug "Batch fetching $(line_count "$(cat "$Tempdir/batch_list")") packs..."
+			(
+				if [ -n "${GCRYPT_TRACE:-}" ]; then set -x; fi
+				exec 0</dev/null
+				# Use --files-from=- to download all packs in one go
+				rsync -I -W --files-from="$Tempdir/batch_list" "$(rsynclocation "$URL")/" "$Tempdir/batch/" >&2
+			)
+		fi
+	fi
+
+	# Process packs (sequentially, using cache if available)
+	xecho "$input_data" | while IFS=': ' read -r _ htype_ pack_ # <<here-document
 	do
 		isnonnull "$pack_" || continue
 
diff --git a/tests/test-rsync-batch.sh b/tests/test-rsync-batch.sh
new file mode 100644
index 0000000..3c4ceaf
--- /dev/null
+++ b/tests/test-rsync-batch.sh
@@ -0,0 +1,116 @@
+#!/bin/bash
+set -e
+
+# Mock rsync to verify batching
+cat <<'EOF' >"$Tempdir/rsync"
+#!/bin/bash
+TRACELOG="${TRACELOG_PATH}"
+echo "MOCK RSYNC CALLED with args: $*" >> "$TRACELOG"
+# Simulate download if --files-from is used
+if [[ "$*" == *"--files-from"* ]]; then
+    # Parse --files-from argument value
+    # simple parsing
+    for arg in "$@"; do
+        if [[ "$arg" == --files-from=* ]]; then
+            listfile="${arg#*=}"
+            # Create dummy batch files
+            batch_dir="$BatchDir"
+            chmod 700 "$batch_dir"
+            while read -r file; do
+                    mkdir -p "$batch_dir"
+                    echo "dummy content for $file" > "$batch_dir/$file"
+            done < "$listfile"
+        fi
+    done
+fi
+EOF
+chmod +x "$Tempdir/rsync"
+PATH="$Tempdir:$PATH"
+export TRACELOG_PATH="$TRACELOG"
+export BatchDir="$Tempdir/batch"
+
+# Helper to source git-remote-gcrypt functions
+source_script() {
+	# Find the line number where the main logic starts
+	# We look for the "if [ "$NAME" = "dummy-gcrypt-check" ]; then" line
+	start_line=$(grep -n "dummy-gcrypt-check" ./git-remote-gcrypt | cut -d: -f1 || echo "")
+	echo "DEBUG: start_line=$start_line"
+	if [ -z "$start_line" ]; then
+		echo "ERROR: Could not find start line for sourcing."
+		grep "dummy-gcrypt-check" ./git-remote-gcrypt || echo "Grep failed to find pattern"
+		exit 1
+	fi
+	# Extract everything up to that line (exclusive)
+	head -n "$((start_line - 1))" ./git-remote-gcrypt >"$Tempdir/funcs.sh"
+	source "$Tempdir/funcs.sh"
+}
+
+# Test Setup
+Tempdir=$(mktemp -d)
+Localdir="$Tempdir/local"
+mkdir -p "$Localdir"
+mkdir -p "$Localdir/objects/pack"
+TRACELOG="$Tempdir/trace.log"
+URL="rsync://example.com/repo"
+GITCEPTION=""
+Hex40="0000000000000000000000000000000000000000"
+
+source_script
+
+# Redefine other deps we don't want to run
+gpg_hash() { echo "mockhash"; }
+DECRYPT() { cat; }
+check_safety() { :; }
+# Mock get_verify_decrypt_pack to simplify (since we care about the rsync call)
+get_verify_decrypt_pack() {
+	# Just call GET to trigger our logic
+	GET "$URL" "$2" "$Tempdir/packF"
+}
+# Mock git index-pack
+git() { :; }
+
+echo "Running Batch Test..."
+
+# Create a mock Packlist with 3 packs
+cat <<EOF >"$Tempdir/input_packs"
+pack :SHA256:pack1 hash1
+pack :SHA256:pack2 hash2
+pack :SHA256:pack3 hash3
+EOF
+
+Packlist=$(cat "$Tempdir/input_packs")
+echo "DEBUG input_packs content:"
+cat "$Tempdir/input_packs"
+
+echo "DEBUG Running get_pack_files..."
+# Run get_pack_files with input from the "missing packs" (simulated)
+# In real flow, this input comes from pneed_
+cat "$Tempdir/input_packs" | get_pack_files
+
+echo "DEBUG batch_list content:"
+if [ -f "$Tempdir/batch/batch_list" ]; then
+	cat "$Tempdir/batch/batch_list"
+else
+	# logic uses $Tempdir/batch_list not $Tempdir/batch/batch_list
+	if [ -f "$Tempdir/batch_list" ]; then
+		cat "$Tempdir/batch_list"
+	else
+		echo "batch_list not found!"
+	fi
+fi
+
+# Verify trace
+count=$(grep -c "MOCK RSYNC CALLED" "$TRACELOG")
+echo "Rsync called $count times."
+
+if [ "$count" -eq 1 ]; then
+	echo "SUCCESS: Rsync called exactly once."
+	cat "$TRACELOG"
+else
+	echo "FAILURE: Rsync called $count times (expected 1)."
+	cat "$TRACELOG"
+	exit 1
+fi
+
+# rm -rf "$Tempdir"
+echo "Trace log at $TRACELOG"