From fcb222a7bdf2877bc570ae2e104ab57e6eb4e30f Mon Sep 17 00:00:00 2001 From: Shane Jaroch Date: Mon, 19 Jan 2026 13:31:24 -0500 Subject: [PATCH] wip --- Makefile | 5 + etc/nginx/conf.d/default.dev.conf | 6 +- etc/nginx/conf.d/default.prod.conf | 51 +--- etc/nginx/conf.d/klaus.conf | 41 --- .../{stalwart.conf => stalwart.dev.conf} | 8 +- scripts/deploy.sh | 52 ++-- scripts/gen_services_map.py | 19 +- scripts/klaus_app.py | 8 +- scripts/manage_repos.py | 285 +++++++++++------- 9 files changed, 247 insertions(+), 228 deletions(-) delete mode 100644 etc/nginx/conf.d/klaus.conf rename etc/nginx/conf.d/{stalwart.conf => stalwart.dev.conf} (86%) diff --git a/Makefile b/Makefile index 9177dd1..1e8402d 100644 --- a/Makefile +++ b/Makefile @@ -191,3 +191,8 @@ git/list: ##H @Local List tracked repositories .PHONY: git/sync git/sync: ##H @Local Sync remote repositories to local JSON @python3 scripts/manage_repos.py --remote $(VPS) sync + +.PHONY: format +format: ##H @Local Format python and shell scripts + git ls-files '*.py' | xargs black + git ls-files '*.sh' | xargs shfmt -l -w diff --git a/etc/nginx/conf.d/default.dev.conf b/etc/nginx/conf.d/default.dev.conf index 1621935..2a7b71d 100644 --- a/etc/nginx/conf.d/default.dev.conf +++ b/etc/nginx/conf.d/default.dev.conf @@ -101,9 +101,11 @@ server { #ssl_stapling_verify on; # Services Map (Homepage) + root /var/www; + index homepage.html; + location / { - alias /var/www/homepage.html; - default_type text/html; + try_files $uri $uri/ =404; } # # Blog / Sphinx diff --git a/etc/nginx/conf.d/default.prod.conf b/etc/nginx/conf.d/default.prod.conf index ed48274..1420045 100644 --- a/etc/nginx/conf.d/default.prod.conf +++ b/etc/nginx/conf.d/default.prod.conf @@ -174,53 +174,6 @@ server { return 301 https://nutra.tk$request_uri; } -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -# Listen on 443 with matrix / synapse -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -server { - listen 443 ssl; - listen 443 quic; - http2 on; - http3 on; - add_header Alt-Svc 'h3=":443"; ma=86400' always; - add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" always; - ssl_trusted_certificate /etc/ssl/private/ca-certs.pem; - server_name matrix.nutra.tk chat.nutra.tk; - - location / { - # Service: Matrix Chat | https://chat.nutra.tk - proxy_pass http://127.0.0.1:8008; - proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto; - proxy_set_header X-Forwarded-For $remote_addr; - } - - location /favicon.ico { - alias /var/www/favicon.gif; - } -} +# (Matrix Chat removed: Dev only) -# Open matrix chat on 8448 -server { - listen 8448 ssl default_server; - listen [::]:8448 ssl default_server; - listen 8448 quic default_server; - listen [::]:8448 quic default_server; - http2 on; - http3 on; - add_header Alt-Svc 'h3=":8448"; ma=86400' always; - add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" always; - ssl_trusted_certificate /etc/ssl/private/ca-certs.pem; - server_name nutra.tk; - - location / { - proxy_pass http://127.0.0.1:8008; - proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto; - proxy_set_header X-Forwarded-For $remote_addr; - } - - # HTTPS / SSL - ssl_certificate /etc/letsencrypt/live/nutra.tk/fullchain.pem; # managed by Certbot - ssl_certificate_key /etc/letsencrypt/live/nutra.tk/privkey.pem; # managed by Certbot - include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot - ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot -} +# (Matrix Chat on 8448 removed: Dev only) diff --git a/etc/nginx/conf.d/klaus.conf b/etc/nginx/conf.d/klaus.conf deleted file mode 100644 index a4691d4..0000000 --- a/etc/nginx/conf.d/klaus.conf +++ /dev/null @@ -1,41 +0,0 @@ -server { - listen 80; - listen [::]:80; - server_name git.nutra.tk; - return 301 https://$host$request_uri; -} - -server { - listen 443 ssl; - listen 443 quic; - listen [::]:443 ssl; - listen [::]:443 quic; - http2 on; - http3 on; - add_header Alt-Svc 'h3=":443"; ma=86400' always; - server_name git.nutra.tk; - - ssl_certificate /etc/letsencrypt/live/earthyenergy.mooo.com/fullchain.pem; # managed by Certbot - ssl_certificate_key /etc/letsencrypt/live/earthyenergy.mooo.com/privkey.pem; # managed by Certbot - include /etc/letsencrypt/options-ssl-nginx.conf; - ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; - - # Password Protection (Uncomment to enable) - # sudo apt-get install apache2-utils - # sudo htpasswd -c /etc/nginx/.htpasswd username - # auth_basic "Restricted Access"; - # auth_basic_user_file /etc/nginx/.htpasswd; - - location / { - proxy_pass http://127.0.0.1:8080; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - } - - # Optional: Serve static files directly if we locate where klaus installed them - # location /static { - # alias /usr/local/lib/python3.x/dist-packages/klaus/static; - # } -} diff --git a/etc/nginx/conf.d/stalwart.conf b/etc/nginx/conf.d/stalwart.dev.conf similarity index 86% rename from etc/nginx/conf.d/stalwart.conf rename to etc/nginx/conf.d/stalwart.dev.conf index dde6028..1f9883a 100644 --- a/etc/nginx/conf.d/stalwart.conf +++ b/etc/nginx/conf.d/stalwart.dev.conf @@ -2,13 +2,13 @@ server { listen 80; listen [::]:80; - server_name mail.yourdomain.com; + server_name mail.nutra.tk; return 301 https://$host$request_uri; } # Main Server (HTTPS + HTTP/3) server { - server_name mail.yourdomain.com; + server_name mail.nutra.tk; # HTTP/3 (QUIC) - UDP # Note: No 'reuseport' here because dev.nutra.tk already has it @@ -34,8 +34,8 @@ server { # SSL Configuration # (Ensure you point these to the actual certs generated for your mail domain) - ssl_certificate /etc/letsencrypt/live/mail.yourdomain.com/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/live/mail.yourdomain.com/privkey.pem; + ssl_certificate /etc/letsencrypt/live/mail.nutra.tk/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/mail.nutra.tk/privkey.pem; include /etc/letsencrypt/options-ssl-nginx.conf; ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 1c61652..70ab884 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -28,9 +28,14 @@ if [ "$1" = "diff" ]; then # Logic to check against default.conf TARGET_FILE="$DEST_CONF_DIR/$BASENAME" - if [[ "$BASENAME" == "default.dev.conf" || "$BASENAME" == "default.prod.conf" || "$BASENAME" == "default.conf" ]]; then - if [ "$BASENAME" == "default.${ENV}.conf" ]; then - TARGET_FILE="$DEST_CONF_DIR/default.conf" + + # Generic handling for *.dev.conf and *.prod.conf + if [[ "$BASENAME" =~ ^(.*)\.(dev|prod)\.conf$ ]]; then + STEM="${BASH_REMATCH[1]}" + CONF_ENV="${BASH_REMATCH[2]}" + + if [ "$CONF_ENV" == "$ENV" ]; then + TARGET_FILE="$DEST_CONF_DIR/$STEM.conf" else continue fi @@ -63,13 +68,14 @@ if [ "$1" = "test" ]; then continue fi - # Handle default configuration switching - if [[ "$BASENAME" == "default.dev.conf" || "$BASENAME" == "default.prod.conf" || "$BASENAME" == "default.conf" ]]; then - if [ "$BASENAME" == "default.${ENV}.conf" ]; then - # Rename to default.conf - cp "$FILE" "$TMP_CONF_D/default.conf" + # Generic handling for *.dev.conf and *.prod.conf + if [[ "$BASENAME" =~ ^(.*)\.(dev|prod)\.conf$ ]]; then + STEM="${BASH_REMATCH[1]}" + CONF_ENV="${BASH_REMATCH[2]}" + + if [ "$CONF_ENV" == "$ENV" ]; then + cp "$FILE" "$TMP_CONF_D/$STEM.conf" else - # Skip other environment configs continue fi else @@ -83,11 +89,14 @@ if [ "$1" = "test" ]; then if sudo nginx -t -c "$TMP_NGINX_CONF"; then echo "✓ Pre-flight validation passed." - sudo nginx -T -c "$TMP_NGINX_CONF" + if [ -n "$DEBUG" ]; then + sudo nginx -T -c "$TMP_NGINX_CONF" + fi rm -rf "$TMP_WORK_DIR" exit 0 else echo "✗ Pre-flight validation FAILED." + sudo nginx -T -c "$TMP_NGINX_CONF" rm -rf "$TMP_WORK_DIR" exit 1 fi @@ -109,21 +118,23 @@ echo "Deploying for environment: $ENV" echo "Installing new configurations..." for FILE in "$NGINX_CONF_SRC"/*.conf; do BASENAME=$(basename "$FILE") - + # Skip encrypted secrets if [ "$BASENAME" = "secrets.conf" ] && ! is_text_file "$FILE"; then echo "Skipping encrypted secrets.conf..." continue fi - # Handle default configuration switching - if [[ "$BASENAME" == "default.dev.conf" || "$BASENAME" == "default.prod.conf" || "$BASENAME" == "default.conf" ]]; then - if [ "$BASENAME" == "default.${ENV}.conf" ]; then - echo "Installing $BASENAME as default.conf..." - sudo cp "$FILE" "$DEST_CONF_DIR/default.conf" + # Generic handling for *.dev.conf and *.prod.conf + if [[ "$BASENAME" =~ ^(.*)\.(dev|prod)\.conf$ ]]; then + STEM="${BASH_REMATCH[1]}" + CONF_ENV="${BASH_REMATCH[2]}" + + if [ "$CONF_ENV" == "$ENV" ]; then + echo "Installing $BASENAME as $STEM.conf..." + sudo cp "$FILE" "$DEST_CONF_DIR/$STEM.conf" else - # Skip other environment configs and the raw default.conf if it exists - echo "Skipping mismatch/raw config: $BASENAME" + echo "Skipping mismatch config: $BASENAME" continue fi else @@ -153,9 +164,9 @@ if sudo nginx -t; then if [ -d "$REPO_ROOT/scripts/gitweb-simplefrontend" ]; then echo "Generating services map..." if [ -f "$REPO_ROOT/scripts/gen_services_map.py" ]; then - python3 "$REPO_ROOT/scripts/gen_services_map.py" + python3 "$REPO_ROOT/scripts/gen_services_map.py" fi - + echo "Deploying Gitweb frontend..." sudo cp -r "$REPO_ROOT/scripts/gitweb-simplefrontend/"* /srv/git/ sudo chown -R www-data:www-data /srv/git/ @@ -167,6 +178,7 @@ if sudo nginx -t; then sudo mkdir -p /var/www sudo cp "$REPO_ROOT/scripts/homepage.html" /var/www/homepage.html sudo chown www-data:www-data /var/www/homepage.html + sudo chmod 644 /var/www/homepage.html fi echo "✓ Deployment successful." diff --git a/scripts/gen_services_map.py b/scripts/gen_services_map.py index 3b5f22d..de4ec7c 100755 --- a/scripts/gen_services_map.py +++ b/scripts/gen_services_map.py @@ -35,12 +35,11 @@ HTML_TEMPLATE = """ """ - def parse_file(path, pattern, is_version=False): if not path.exists(): - print(f"Warning: Could not find config at {path}") - return [] - + print(f"Warning: Could not find config at {path}") + return [] + with open(path, "r") as f: content = f.read() @@ -54,10 +53,14 @@ def parse_file(path, pattern, is_version=False): vid = f"v{version_id}" else: vid = version_id - items.append({"id": vid, "url": f"/{vid}", "description": description.strip()}) + items.append( + {"id": vid, "url": f"/{vid}", "description": description.strip()} + ) else: name, url = m - items.append({"id": name.strip(), "url": url.strip(), "description": name.strip()}) + items.append( + {"id": name.strip(), "url": url.strip(), "description": name.strip()} + ) return items @@ -67,13 +70,13 @@ def get_all_services(): service_pattern = re.compile(r"^\s*#\s*Service:\s*(.+?)\s*\|\s*(.+)$", re.MULTILINE) services_git = parse_file(NGINX_CONF, version_pattern, is_version=True) - + # Locate default.conf # On Server: Read the live deployed config live_default = Path("/etc/nginx/conf.d/default.conf") # On Local: Read default.dev.conf local_dev = REPO_ROOT / "etc/nginx/conf.d/default.dev.conf" - + if live_default.exists(): DEFAULT_CONF = live_default print(f"Using live config: {DEFAULT_CONF}") diff --git a/scripts/klaus_app.py b/scripts/klaus_app.py index 6f3561a..ace43fc 100644 --- a/scripts/klaus_app.py +++ b/scripts/klaus_app.py @@ -3,8 +3,9 @@ import klaus from klaus.contrib.wsgi import make_app # Root directory for repositories -REPO_ROOT = os.environ.get('KLAUS_REPOS_ROOT', '/srv/git') -SITE_NAME = os.environ.get('KLAUS_SITE_NAME', 'Git Repos') +REPO_ROOT = os.environ.get("KLAUS_REPOS_ROOT", "/srv/git") +SITE_NAME = os.environ.get("KLAUS_SITE_NAME", "Git Repos") + def find_git_repos(root_dir): """ @@ -14,11 +15,12 @@ def find_git_repos(root_dir): for root, dirs, files in os.walk(root_dir): # Scan directories for d in dirs: - if d.endswith('.git'): + if d.endswith(".git"): full_path = os.path.join(root, d) repos.append(full_path) return sorted(repos) + # Discover repositories repositories = find_git_repos(REPO_ROOT) diff --git a/scripts/manage_repos.py b/scripts/manage_repos.py index 22f371d..b7f9a6c 100755 --- a/scripts/manage_repos.py +++ b/scripts/manage_repos.py @@ -5,6 +5,7 @@ import os import subprocess import sys import shlex + try: import argcomplete except ImportError: @@ -17,9 +18,10 @@ REPO_JSON = os.path.join(SCRIPT_DIR, "repos.json") REPO_CSV = os.path.join(SCRIPT_DIR, "repo_metadata.csv") GIT_ROOT = "/srv/git" + def load_repos(): if os.path.exists(REPO_JSON): - with open(REPO_JSON, 'r') as f: + with open(REPO_JSON, "r") as f: try: return json.load(f) except json.JSONDecodeError: @@ -27,79 +29,91 @@ def load_repos(): return {} return {} + def save_repos(data): - with open(REPO_JSON, 'w') as f: + with open(REPO_JSON, "w") as f: json.dump(data, f, indent=2) + def remote_exists(remote, path): if remote: - cmd = ['ssh', remote, f'test -e {shlex.quote(path)}'] + cmd = ["ssh", remote, f"test -e {shlex.quote(path)}"] return subprocess.call(cmd) == 0 else: return os.path.exists(path) + def remote_isdir(remote, path): if remote: - cmd = ['ssh', remote, f'test -d {shlex.quote(path)}'] + cmd = ["ssh", remote, f"test -d {shlex.quote(path)}"] return subprocess.call(cmd) == 0 else: return os.path.isdir(path) + def remote_makedirs(remote, path): if remote: - subprocess.check_call(['ssh', remote, f'mkdir -p {shlex.quote(path)}']) + subprocess.check_call(["ssh", remote, f"mkdir -p {shlex.quote(path)}"]) else: os.makedirs(path, exist_ok=True) + def remote_run(remote, cmd_list, check=True): if remote: - cmd_str = ' '.join(shlex.quote(c) for c in cmd_list) - subprocess.check_call(['ssh', remote, cmd_str]) + cmd_str = " ".join(shlex.quote(c) for c in cmd_list) + subprocess.check_call(["ssh", remote, cmd_str]) else: subprocess.run(cmd_list, check=check) + def remote_write(remote, path, content): if remote: # Use simple cat redirection - p = subprocess.Popen(['ssh', remote, f'cat > {shlex.quote(path)}'], stdin=subprocess.PIPE) - p.communicate(input=content.encode('utf-8')) + p = subprocess.Popen( + ["ssh", remote, f"cat > {shlex.quote(path)}"], stdin=subprocess.PIPE + ) + p.communicate(input=content.encode("utf-8")) if p.returncode != 0: raise subprocess.CalledProcessError(p.returncode, "remote write") else: - with open(path, 'w') as f: + with open(path, "w") as f: f.write(content) + def normalize_repo_path(name): """Ensures path ends in .git and usually has a projects/ prefix if simplistic.""" # Logic: If it has slashes, trust the user. If not, prepend projects/. # Always append .git if missing. name = name.strip() - if '/' not in name: + if "/" not in name: name = f"projects/{name}" - if not name.endswith('.git'): - name += '.git' + if not name.endswith(".git"): + name += ".git" return name + def get_current_dir_name(): return os.path.basename(os.getcwd()) + # ----------------- Commands ----------------- + def cmd_add_clone(args, remote): url = args.url data = load_repos() - + # Determine repo name if args.name: repo_name = args.name else: - repo_name = url.rstrip('/').rsplit('/', 1)[-1] - if repo_name.endswith('.git'): + repo_name = url.rstrip("/").rsplit("/", 1)[-1] + if repo_name.endswith(".git"): repo_name = repo_name[:-4] repo_rel_path = normalize_repo_path(repo_name) full_path = os.path.join(GIT_ROOT, repo_rel_path) - + print(f"Adding Clone: {url}") print(f"Target: {full_path} (Remote: {remote if remote else 'Local'})") @@ -111,12 +125,21 @@ def cmd_add_clone(args, remote): # Clone if not remote_exists(remote, full_path): print(f"Cloning into {full_path}...") - remote_run(remote, ['git', 'clone', '--mirror', url, full_path]) + remote_run(remote, ["git", "clone", "--mirror", url, full_path]) else: print("Repository already exists. Updating metadata...") # Configure - configure_repo(remote, repo_rel_path, full_path, args.desc, args.owner, origin_url=url, data=data) + configure_repo( + remote, + repo_rel_path, + full_path, + args.desc, + args.owner, + origin_url=url, + data=data, + ) + def cmd_init(args, remote): data = load_repos() @@ -125,7 +148,7 @@ def cmd_init(args, remote): if not name: # Try to infer from current directory name = get_current_dir_name() - + repo_rel_path = normalize_repo_path(name) full_path = os.path.join(GIT_ROOT, repo_rel_path) @@ -136,18 +159,20 @@ def cmd_init(args, remote): remote_makedirs(remote, full_path) # Git Init - remote_run(remote, ['git', '-C', full_path, 'init', '--bare']) - + remote_run(remote, ["git", "-C", full_path, "init", "--bare"]) + # Daemon ok - # We can use remote_run with touch, or remote_write. + # We can use remote_run with touch, or remote_write. # touch is simpler. - remote_run(remote, ['touch', os.path.join(full_path, 'git-daemon-export-ok')]) + remote_run(remote, ["touch", os.path.join(full_path, "git-daemon-export-ok")]) # Safe Directory - # We need to run this globally or on the system level, but running it locally for the user often works - # if the user accessing it is the one running this. + # We need to run this globally or on the system level, but running it locally for the user often works + # if the user accessing it is the one running this. # However, 'git config --global' on remote affects the remote user (gg). - remote_run(remote, ['git', 'config', '--global', '--add', 'safe.directory', full_path]) + remote_run( + remote, ["git", "config", "--global", "--add", "safe.directory", full_path] + ) # Configure configure_repo(remote, repo_rel_path, full_path, args.desc, args.owner, data=data) @@ -156,24 +181,30 @@ def cmd_init(args, remote): if args.auto_remote: # Check if we are in a git repo if os.path.exists(".git"): - remote_url = f"ssh://{remote}/{full_path.lstrip('/')}" if remote else full_path - remote_name = "helio-web" # standardizing on this name from makefile? - + remote_url = ( + f"ssh://{remote}/{full_path.lstrip('/')}" if remote else full_path + ) + remote_name = "helio-web" # standardizing on this name from makefile? + print(f"Configuring local remote '{remote_name}' -> {remote_url}") # Try adding, if fails, try setting - res = subprocess.call(['git', 'remote', 'add', remote_name, remote_url], stderr=subprocess.DEVNULL) + res = subprocess.call( + ["git", "remote", "add", remote_name, remote_url], + stderr=subprocess.DEVNULL, + ) if res != 0: print(f"Remote '{remote_name}' exists, setting url...") - subprocess.call(['git', 'remote', 'set-url', remote_name, remote_url]) - + subprocess.call(["git", "remote", "set-url", remote_name, remote_url]) + print("You can now push: git push -u helio-web main") + def cmd_rename(args, remote): data = load_repos() - + old_rel = normalize_repo_path(args.old) new_rel = normalize_repo_path(args.new) - + old_full = os.path.join(GIT_ROOT, old_rel) new_full = os.path.join(GIT_ROOT, new_rel) @@ -182,16 +213,16 @@ def cmd_rename(args, remote): if not remote_exists(remote, old_full): print(f"Error: Source repo {old_rel} does not exist.") sys.exit(1) - + if remote_exists(remote, new_full): print(f"Error: Destination repo {new_rel} already exists.") sys.exit(1) # Create new parent remote_makedirs(remote, os.path.dirname(new_full)) - + # Move - remote_run(remote, ['mv', old_full, new_full]) + remote_run(remote, ["mv", old_full, new_full]) # Update Json if old_rel in data: @@ -202,39 +233,52 @@ def cmd_rename(args, remote): print(f"Warning: {old_rel} was not found in repos.json. No metadata moved.") # Safe Directory for new path - remote_run(remote, ['git', 'config', '--global', '--add', 'safe.directory', new_full]) - + remote_run( + remote, ["git", "config", "--global", "--add", "safe.directory", new_full] + ) + print(f"Success. Check your local remotes if you were pushing to {old_rel}.") + def cmd_update(args, remote): data = load_repos() - + target = args.name if target: # Update/configure single repo_rel_path = normalize_repo_path(target) full_path = os.path.join(GIT_ROOT, repo_rel_path) - - if not remote_exists(remote, full_path): - print(f"Warning: {repo_rel_path} does not exist on remote. Skipping.") - return - configure_repo(remote, repo_rel_path, full_path, args.desc, args.owner, origin_url=args.origin, data=data) + if not remote_exists(remote, full_path): + print(f"Warning: {repo_rel_path} does not exist on remote. Skipping.") + return + + configure_repo( + remote, + repo_rel_path, + full_path, + args.desc, + args.owner, + origin_url=args.origin, + data=data, + ) else: print("Error: Name required.") + def repo_completer(prefix, parsed_args, **kwargs): # Load repos for completion data = load_repos() return [k for k in data.keys() if k.startswith(prefix)] + def cmd_sync(args, remote): if not remote: print("Error: --remote is required for sync (or set VPS_REMOTE env var)") sys.exit(1) - + print(f"Scanning {remote}:{GIT_ROOT}...") - + # Remote python script to gather all metadata in one go remote_script = f""" import os, json, subprocess @@ -284,10 +328,10 @@ for dirpath, dirnames, filenames in os.walk(root): print(json.dumps(results)) """ - + # Run the script remotely via SSH - cmd = ['ssh', remote, 'python3', '-c', shlex.quote(remote_script)] - + cmd = ["ssh", remote, "python3", "-c", shlex.quote(remote_script)] + try: output = subprocess.check_output(cmd, universal_newlines=True) remote_data = json.loads(output) @@ -303,7 +347,7 @@ print(json.dumps(results)) data = load_repos() updated_count = 0 new_count = 0 - + for rel_path, info in remote_data.items(): if rel_path not in data: data[rel_path] = {} @@ -311,64 +355,88 @@ print(json.dumps(results)) print(f" [NEW] Found {rel_path}") else: updated_count += 1 - - data[rel_path]['description'] = info['description'] - data[rel_path]['owner'] = info['owner'] - if info.get('remotes'): - if 'remotes' not in data[rel_path]: data[rel_path]['remotes'] = {} - data[rel_path]['remotes'].update(info['remotes']) + + data[rel_path]["description"] = info["description"] + data[rel_path]["owner"] = info["owner"] + if info.get("remotes"): + if "remotes" not in data[rel_path]: + data[rel_path]["remotes"] = {} + data[rel_path]["remotes"].update(info["remotes"]) save_repos(data) - print(f"\nSync complete. Added {new_count}, Scanned {updated_count}. (Single SSH connection used)") + print( + f"\nSync complete. Added {new_count}, Scanned {updated_count}. (Single SSH connection used)" + ) + def cmd_list(args, remote): data = migrate_csv_if_needed() print(json.dumps(data, indent=2)) + # ----------------- Helpers ----------------- -def configure_repo(remote, repo_rel_path, full_path, description, owner, origin_url=None, data=None): - if data is None: data = {} - + +def configure_repo( + remote, repo_rel_path, full_path, description, owner, origin_url=None, data=None +): + if data is None: + data = {} + msg_parts = [] - + # Description if description: - desc_path = os.path.join(full_path, 'description') + desc_path = os.path.join(full_path, "description") remote_write(remote, desc_path, description + "\n") msg_parts.append("description") # Owner if owner: - config_path = os.path.join(full_path, 'config') - remote_run(remote, ['git', 'config', '--file', config_path, 'gitweb.owner', owner]) + config_path = os.path.join(full_path, "config") + remote_run( + remote, ["git", "config", "--file", config_path, "gitweb.owner", owner] + ) msg_parts.append("owner") # JSON Metadata if repo_rel_path not in data: data[repo_rel_path] = {} - if description: data[repo_rel_path]['description'] = description - if owner: data[repo_rel_path]['owner'] = owner - + if description: + data[repo_rel_path]["description"] = description + if owner: + data[repo_rel_path]["owner"] = owner + if origin_url: - if 'remotes' not in data[repo_rel_path]: - data[repo_rel_path]['remotes'] = {} - data[repo_rel_path]['remotes']['origin'] = origin_url - + if "remotes" not in data[repo_rel_path]: + data[repo_rel_path]["remotes"] = {} + data[repo_rel_path]["remotes"]["origin"] = origin_url + # Also update on remote if possible try: - config_path = os.path.join(full_path, 'config') - # check if remote exists - # It's hard to know if 'origin' exists in the config without checking, - # but we can just try setting it. - # If it doesn't exist, we might need to add it? Bare repos don't usually have remotes unless mirrored. - # Let's try setting. - remote_run(remote, ['git', 'config', '--file', config_path, 'remote.origin.url', origin_url], check=False) - # Also ensure fetch is set? (Optional, usually implied or not needed for just tracking URL) + config_path = os.path.join(full_path, "config") + # check if remote exists + # It's hard to know if 'origin' exists in the config without checking, + # but we can just try setting it. + # If it doesn't exist, we might need to add it? Bare repos don't usually have remotes unless mirrored. + # Let's try setting. + remote_run( + remote, + [ + "git", + "config", + "--file", + config_path, + "remote.origin.url", + origin_url, + ], + check=False, + ) + # Also ensure fetch is set? (Optional, usually implied or not needed for just tracking URL) except Exception: - pass - + pass + # Always save! save_repos(data) print(f"Configuration updated ({', '.join(msg_parts)}) and saved to repos.json") @@ -384,35 +452,40 @@ def migrate_csv_if_needed(): return data print(f"Migrating existing metadata from {REPO_CSV} to {REPO_JSON}...") - with open(REPO_CSV, 'r') as f: + with open(REPO_CSV, "r") as f: reader = csv.DictReader(f) reader.fieldnames = [name.strip() for name in reader.fieldnames] - + for row in reader: - path = row.get('repo_path', '').strip() - if not path: continue - + path = row.get("repo_path", "").strip() + if not path: + continue + if path not in data: data[path] = { - "owner": row.get('owner', '').strip(), - "description": row.get('description', '').strip(), - "remotes": {} + "owner": row.get("owner", "").strip(), + "description": row.get("description", "").strip(), + "remotes": {}, } - + # We skip the complex remote scanning for now to keep it fast save_repos(data) return data + # ----------------- Main ----------------- + def main(): - parser = argparse.ArgumentParser(description="Manage git repos in /srv/git and track metadata") + parser = argparse.ArgumentParser( + description="Manage git repos in /srv/git and track metadata" + ) parser.add_argument( "--remote", help="SSH remote (e.g. gg@dev.nutra.tk) or blank for local (env: VPS_REMOTE)", - default=os.environ.get('VPS_REMOTE') + default=os.environ.get("VPS_REMOTE"), ) - + subparsers = parser.add_subparsers(dest="command", required=True) # ADD @@ -425,10 +498,17 @@ def main(): # INIT p_init = subparsers.add_parser("init", help="Initialize new bare repo") - p_init.add_argument("--name", help="Repository name (e.g. 'cli' or 'projects/cli'). Defaults to current dir name.") + p_init.add_argument( + "--name", + help="Repository name (e.g. 'cli' or 'projects/cli'). Defaults to current dir name.", + ) p_init.add_argument("--desc", help="Description for gitweb", default="") p_init.add_argument("--owner", help="Owner for gitweb", default="Shane") - p_init.add_argument("--auto-remote", action="store_true", help="Add this as a remote to current git repo") + p_init.add_argument( + "--auto-remote", + action="store_true", + help="Add this as a remote to current git repo", + ) p_init.set_defaults(func=cmd_init) # RENAME @@ -454,21 +534,24 @@ def main(): p_mig.set_defaults(func=lambda args, r: migrate_csv_if_needed()) # SYNC - p_sync = subparsers.add_parser("sync", help="Sync/Import remote repositories to local JSON") + p_sync = subparsers.add_parser( + "sync", help="Sync/Import remote repositories to local JSON" + ) p_sync.set_defaults(func=cmd_sync) if argcomplete: argcomplete.autocomplete(parser) args = parser.parse_args() - + # Migration check before any command? # Or just let them run. load_repos handles empty json nicely. - - if hasattr(args, 'func'): + + if hasattr(args, "func"): args.func(args, args.remote) else: parser.print_help() + if __name__ == "__main__": main() -- 2.52.0