feat: initial import — SVN↔Git bi-directional mirror for Gitea
Server-side sidecar daemon. Uses own SQLite mapping table (not git svn) so Git commit hashes never change. Pure Python, zero framework deps. Includes: - SVN→Git sync engine with nested-directory tree reconstruction - Git→SVN push with author mapping and working-copy management - Branch/tag sync (svn copy → git branch/tag and reverse) - Gitea integration (on-disk push/fetch, webhook receiver) - Reconciliation tool for divergence recovery (assess + auto-fix) - Full end-to-end regression test (bash, 34 checks) - systemd unit and deployment docs Only external dependency: PyYAML (config parsing).
This commit is contained in:
@@ -0,0 +1,3 @@
|
||||
from .cli import main
|
||||
|
||||
main()
|
||||
@@ -0,0 +1,324 @@
|
||||
"""CLI entry point for svn-mirror."""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from .config import AppConfig, ConfigError
|
||||
from .mirror import Mirror, MirrorError
|
||||
from .gitea import push_to_gitea, GiteaError
|
||||
from .git_to_svn import sync_git_to_svn, GitToSVNError
|
||||
from .sync import sync_all, run_daemon_loop
|
||||
from .reconcile import reconcile, assess as reconcile_assess, print_assessment, ReconcileError
|
||||
from .webhook import run_webhook_server
|
||||
|
||||
|
||||
def setup_logging(verbose: bool = False):
|
||||
level = logging.DEBUG if verbose else logging.INFO
|
||||
logging.basicConfig(
|
||||
level=level,
|
||||
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
)
|
||||
|
||||
|
||||
def cmd_create(args):
|
||||
config = AppConfig.load(args.config)
|
||||
mirrors = [config.mirrors[mid] for mid in (args.mirror and [args.mirror] or config.mirrors)]
|
||||
for mc in mirrors:
|
||||
m = Mirror(mc, Path(config.data_dir))
|
||||
try:
|
||||
m.create()
|
||||
print(f" ✓ {mc.id}")
|
||||
except MirrorError as e:
|
||||
print(f" ✗ {mc.id}: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def cmd_init(args):
|
||||
config = AppConfig.load(args.config)
|
||||
mirrors = [config.mirrors[mid] for mid in (args.mirror and [args.mirror] or config.mirrors)]
|
||||
for mc in mirrors:
|
||||
m = Mirror(mc, Path(config.data_dir))
|
||||
try:
|
||||
m.init_import()
|
||||
print(f" ✓ {mc.id}")
|
||||
except MirrorError as e:
|
||||
print(f" ✗ {mc.id}: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def cmd_destroy(args):
|
||||
config = AppConfig.load(args.config)
|
||||
mc = config.mirrors.get(args.mirror)
|
||||
if not mc:
|
||||
print(f"Error: mirror '{args.mirror}' not found in config", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
m = Mirror(mc, Path(config.data_dir))
|
||||
try:
|
||||
m.destroy()
|
||||
print(f" ✓ {args.mirror}")
|
||||
except MirrorError as e:
|
||||
print(f" ✗ {args.mirror}: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def cmd_status(args):
|
||||
config = AppConfig.load(args.config)
|
||||
mirrors = [config.mirrors[mid] for mid in (args.mirror and [args.mirror] or config.mirrors)]
|
||||
for mc in mirrors:
|
||||
m = Mirror(mc, Path(config.data_dir))
|
||||
s = m.status()
|
||||
_print_status(s)
|
||||
|
||||
|
||||
def _print_status(s: dict):
|
||||
print(f"Mirror: {s['id']}")
|
||||
print(f" Status: {s['status']}")
|
||||
print(f" SVN URL: {s['svn_url']}")
|
||||
print(f" Enabled: {s.get('enabled', True)}")
|
||||
|
||||
if s['status'] == 'not_created':
|
||||
print(f" (not yet created on disk)")
|
||||
return
|
||||
|
||||
if s.get('imported'):
|
||||
print(f" Import: done")
|
||||
print(f" Last SVN: {s.get('last_svn_revision', '?')}")
|
||||
print(f" Mappings: {s.get('mapping_count', 0)}")
|
||||
else:
|
||||
print(f" Import: pending")
|
||||
|
||||
print(f" Created: {s.get('created_at', '?')}")
|
||||
print(f" Last sync: {s.get('last_sync_at', 'never')}")
|
||||
print(f" Location: {s.get('base_dir', '?')}")
|
||||
print()
|
||||
|
||||
|
||||
def cmd_sync(args):
|
||||
"""Run a sync cycle for each mirror."""
|
||||
config = AppConfig.load(args.config)
|
||||
mirrors = [config.mirrors[mid] for mid in (args.mirror and [args.mirror] or config.mirrors)]
|
||||
for mc in mirrors:
|
||||
m = Mirror(mc, Path(config.data_dir))
|
||||
if not m.exists:
|
||||
print(f" - {mc.id}: not created yet", file=sys.stderr)
|
||||
continue
|
||||
state = m._read_state()
|
||||
if not state.get("initial_import_done"):
|
||||
print(f" - {mc.id}: import not done yet", file=sys.stderr)
|
||||
continue
|
||||
try:
|
||||
total = 0
|
||||
svn_count = 0
|
||||
git_count = 0
|
||||
if args.direction in ("both", "svn-to-git"):
|
||||
svn_count = sync_all(m)
|
||||
total += svn_count
|
||||
if svn_count:
|
||||
try:
|
||||
push_to_gitea(m)
|
||||
except GiteaError as ge:
|
||||
print(f" ⚠ {mc.id}: push to Gitea failed: {ge}", file=sys.stderr)
|
||||
if args.direction in ("both", "git-to-svn"):
|
||||
git_count = sync_git_to_svn(m)
|
||||
total += git_count
|
||||
if svn_count:
|
||||
print(f" → {mc.id}: {svn_count} SVN rev(s) synced to Git")
|
||||
if git_count:
|
||||
print(f" ← {mc.id}: {git_count} Git commit(s) pushed to SVN")
|
||||
if not total:
|
||||
print(f" ✓ {mc.id}: up to date")
|
||||
except Exception as e:
|
||||
print(f" ✗ {mc.id}: {e}", file=sys.stderr)
|
||||
|
||||
|
||||
def cmd_push(args):
|
||||
"""Run a single Git→SVN push cycle for each mirror."""
|
||||
config = AppConfig.load(args.config)
|
||||
mirrors = [config.mirrors[mid] for mid in (args.mirror and [args.mirror] or config.mirrors)]
|
||||
for mc in mirrors:
|
||||
m = Mirror(mc, Path(config.data_dir))
|
||||
if not m.exists:
|
||||
print(f" - {mc.id}: not created yet", file=sys.stderr)
|
||||
continue
|
||||
state = m._read_state()
|
||||
if not state.get("initial_import_done"):
|
||||
print(f" - {mc.id}: import not done yet", file=sys.stderr)
|
||||
continue
|
||||
try:
|
||||
count = sync_git_to_svn(m)
|
||||
if count:
|
||||
print(f" ✓ {mc.id}: {count} Git commit(s) pushed to SVN")
|
||||
else:
|
||||
print(f" ✓ {mc.id}: up to date")
|
||||
except Exception as e:
|
||||
print(f" ✗ {mc.id}: {e}", file=sys.stderr)
|
||||
|
||||
|
||||
def cmd_webhook(args):
|
||||
"""Start the Gitea webhook receiver server."""
|
||||
config = AppConfig.load(args.config)
|
||||
mirrors = {}
|
||||
for mid, mc in config.mirrors.items():
|
||||
if not mc.enabled:
|
||||
continue
|
||||
m = Mirror(mc, Path(config.data_dir))
|
||||
mirrors[mid] = m
|
||||
if not mirrors:
|
||||
print("No enabled mirrors found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
# Use first mirror's webhook config (they should all be the same host/port)
|
||||
first = next(iter(mirrors.values()))
|
||||
host = first.config.gitea.webhook_host
|
||||
port = first.config.gitea.webhook_port
|
||||
print(f"Starting webhook server on {host}:{port} …")
|
||||
run_webhook_server(mirrors, host=host, port=port)
|
||||
|
||||
|
||||
def cmd_daemon(args):
|
||||
"""Run continuous sync daemon."""
|
||||
config = AppConfig.load(args.config)
|
||||
mirrors = {}
|
||||
for mid, mc in config.mirrors.items():
|
||||
if not mc.enabled:
|
||||
continue
|
||||
m = Mirror(mc, Path(config.data_dir))
|
||||
mirrors[mid] = m
|
||||
if not mirrors:
|
||||
print("No enabled mirrors found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
run_daemon_loop(mirrors)
|
||||
|
||||
|
||||
def cmd_reconcile(args):
|
||||
"""Reconcile a diverged mirror (both sides have commits)."""
|
||||
config = AppConfig.load(args.config)
|
||||
mc = config.mirrors.get(args.mirror)
|
||||
if not mc:
|
||||
print(f"Error: mirror '{args.mirror}' not found in config", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
m = Mirror(mc, Path(config.data_dir))
|
||||
if not m.exists:
|
||||
print(f"Error: mirror '{args.mirror}' not created yet", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
state = m._read_state()
|
||||
if not state.get("initial_import_done"):
|
||||
print(f"Error: mirror '{args.mirror}' import not done yet", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
if args.assess:
|
||||
report = reconcile_assess(m)
|
||||
print_assessment(report)
|
||||
else:
|
||||
strategy = "git-wins"
|
||||
if args.merge:
|
||||
strategy = "merge"
|
||||
rev = reconcile(m, strategy=strategy, dry_run=args.dry_run)
|
||||
if rev:
|
||||
print(f" ✓ Reconciliation commit: SVN r{rev}")
|
||||
except ReconcileError as e:
|
||||
print(f" ✗ {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def cmd_check(args):
|
||||
"""Validate that the config is parseable and git svn is available."""
|
||||
try:
|
||||
config = AppConfig.load(args.config)
|
||||
print(f"Config OK: {len(config.mirrors)} mirror(s) defined")
|
||||
print(f" Data dir: {config.data_dir}")
|
||||
for mid, mc in config.mirrors.items():
|
||||
print(f" Mirror '{mid}': {mc.svn.url} → {mc.gitea.owner}/{mc.gitea.repo}")
|
||||
except ConfigError as e:
|
||||
print(f"Config error: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
# optional: check git svn
|
||||
import subprocess
|
||||
try:
|
||||
r = subprocess.run(["git", "svn", "--version"], capture_output=True, text=True, timeout=30)
|
||||
if r.returncode == 0:
|
||||
print(f"git svn: {r.stdout.strip()}")
|
||||
else:
|
||||
print(f"git svn: NOT available (exit {r.returncode})", file=sys.stderr)
|
||||
except FileNotFoundError:
|
||||
print(f"git svn: NOT installed", file=sys.stderr)
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="gitea-svn-mirror – Subgit-like SVN↔Git mirror daemon",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--config", "-c",
|
||||
default="/etc/svn-mirror/config.yml",
|
||||
help="Config file path (default: %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--verbose", "-v",
|
||||
action="store_true",
|
||||
help="Verbose output (debug logging)",
|
||||
)
|
||||
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
p = sub.add_parser("create", help="Create mirror directory structure and DB")
|
||||
p.add_argument("--mirror", "-m", help="Mirror ID (omit for all)")
|
||||
p.set_defaults(func=cmd_create)
|
||||
|
||||
p = sub.add_parser("init", help="Run initial SVN import via git svn clone")
|
||||
p.add_argument("--mirror", "-m", help="Mirror ID (omit for all)")
|
||||
p.set_defaults(func=cmd_init)
|
||||
|
||||
p = sub.add_parser("destroy", help="Delete all mirror data from disk")
|
||||
p.add_argument("--mirror", "-m", required=True, help="Mirror ID")
|
||||
p.set_defaults(func=cmd_destroy)
|
||||
|
||||
p = sub.add_parser("status", help="Show mirror status")
|
||||
p.add_argument("--mirror", "-m", help="Mirror ID (omit for all)")
|
||||
p.set_defaults(func=cmd_status)
|
||||
|
||||
p = sub.add_parser("sync", help="Bidirectional sync (SVN→Git + Git→SVN)")
|
||||
p.add_argument("--mirror", "-m", help="Mirror ID (omit for all)")
|
||||
p.add_argument("--direction", choices=["svn-to-git", "git-to-svn", "both"],
|
||||
default="both", help="Sync direction (default: both)")
|
||||
p.set_defaults(func=cmd_sync)
|
||||
|
||||
p = sub.add_parser("push", help="Run a single Git→SVN push cycle")
|
||||
p.add_argument("--mirror", "-m", help="Mirror ID (omit for all)")
|
||||
p.set_defaults(func=cmd_push)
|
||||
|
||||
p = sub.add_parser("daemon", help="Run continuous sync daemon")
|
||||
p.set_defaults(func=cmd_daemon)
|
||||
|
||||
p = sub.add_parser("webhook", help="Start Gitea webhook receiver")
|
||||
p.set_defaults(func=cmd_webhook)
|
||||
|
||||
p = sub.add_parser("reconcile", help="Reconcile diverged SVN↔Git mirror")
|
||||
p.add_argument("--mirror", "-m", required=True, help="Mirror ID")
|
||||
p.add_argument("--dry-run", "-n", action="store_true",
|
||||
help="Show assessment without making changes")
|
||||
p.add_argument("--assess", action="store_true",
|
||||
help="Show divergence assessment report")
|
||||
p.add_argument("--merge", action="store_true",
|
||||
help="Use three-way merge for conflicting files (experimental)")
|
||||
p.set_defaults(func=cmd_reconcile)
|
||||
|
||||
p = sub.add_parser("check", help="Validate config and prerequisites")
|
||||
p.set_defaults(func=cmd_check)
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
def main():
|
||||
parser = build_parser()
|
||||
args = parser.parse_args()
|
||||
setup_logging(args.verbose)
|
||||
args.func(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,203 @@
|
||||
"""Configuration loading and validation."""
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict, Optional
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
class ConfigError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def _check_type(val, typ, path: str):
|
||||
if not isinstance(val, typ):
|
||||
raise ConfigError(f"{path}: expected {typ.__name__}, got {type(val).__name__}")
|
||||
|
||||
|
||||
def _check_optional(val, typ, path: str):
|
||||
if val is not None and not isinstance(val, typ):
|
||||
raise ConfigError(f"{path}: expected {typ.__name__} or null, got {type(val).__name__}")
|
||||
|
||||
|
||||
@dataclass
|
||||
class SVNConfig:
|
||||
url: str
|
||||
layout: str = "std"
|
||||
trunk: str = "trunk"
|
||||
branches: str = "branches"
|
||||
tags: str = "tags"
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, d: dict, prefix: str) -> "SVNConfig":
|
||||
url = d.get("url")
|
||||
if not url:
|
||||
raise ConfigError(f"{prefix}.url: required")
|
||||
_check_type(url, str, f"{prefix}.url")
|
||||
|
||||
layout = d.get("layout", "std")
|
||||
_check_type(layout, str, f"{prefix}.layout")
|
||||
if layout not in ("std", "custom"):
|
||||
raise ConfigError(f"{prefix}.layout: must be 'std' or 'custom'")
|
||||
|
||||
trunk = d.get("trunk", "trunk")
|
||||
branches = d.get("branches", "branches")
|
||||
tags = d.get("tags", "tags")
|
||||
for name, val in [("trunk", trunk), ("branches", branches), ("tags", tags)]:
|
||||
_check_type(val, str, f"{prefix}.{name}")
|
||||
|
||||
return cls(url=url, layout=layout, trunk=trunk, branches=branches, tags=tags)
|
||||
|
||||
|
||||
@dataclass
|
||||
class GiteaConfig:
|
||||
owner: str
|
||||
repo: str
|
||||
repos_path: Optional[str] = None
|
||||
webhook_secret: Optional[str] = None
|
||||
webhook_host: str = "0.0.0.0"
|
||||
webhook_port: int = 8080
|
||||
api_url: Optional[str] = None
|
||||
api_token: Optional[str] = None
|
||||
|
||||
@property
|
||||
def repo_dir(self) -> str:
|
||||
"""Return the expected on-disk path to Gitea's bare repo."""
|
||||
if self.repos_path:
|
||||
return f"{self.repos_path.rstrip('/')}/{self.owner}/{self.repo}.git"
|
||||
# Common defaults if repos_path not configured
|
||||
for base in ("/var/lib/gitea/data/repositories",
|
||||
"/data/git/repositories",
|
||||
"/home/git/repositories"):
|
||||
candidate = f"{base}/{self.owner}/{self.repo}.git"
|
||||
import os
|
||||
if os.path.isdir(candidate):
|
||||
return candidate
|
||||
raise ConfigError(
|
||||
f"gitea.repos_path not set and no default Gitea repo found "
|
||||
f"for {self.owner}/{self.repo}"
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, d: dict, prefix: str) -> "GiteaConfig":
|
||||
owner = d.get("owner")
|
||||
if not owner:
|
||||
raise ConfigError(f"{prefix}.owner: required")
|
||||
_check_type(owner, str, f"{prefix}.owner")
|
||||
|
||||
repo = d.get("repo")
|
||||
if not repo:
|
||||
raise ConfigError(f"{prefix}.repo: required")
|
||||
_check_type(repo, str, f"{prefix}.repo")
|
||||
|
||||
repos_path = d.get("repos_path")
|
||||
_check_optional(repos_path, str, f"{prefix}.repos_path")
|
||||
|
||||
webhook_secret = d.get("webhook_secret")
|
||||
_check_optional(webhook_secret, str, f"{prefix}.webhook_secret")
|
||||
|
||||
webhook_host = d.get("webhook_host", "0.0.0.0")
|
||||
_check_type(webhook_host, str, f"{prefix}.webhook_host")
|
||||
|
||||
webhook_port = d.get("webhook_port", 8080)
|
||||
_check_type(webhook_port, int, f"{prefix}.webhook_port")
|
||||
|
||||
api_url = d.get("api_url")
|
||||
_check_optional(api_url, str, f"{prefix}.api_url")
|
||||
|
||||
api_token = d.get("api_token")
|
||||
_check_optional(api_token, str, f"{prefix}.api_token")
|
||||
|
||||
return cls(
|
||||
owner=owner, repo=repo,
|
||||
repos_path=repos_path,
|
||||
webhook_secret=webhook_secret,
|
||||
webhook_host=webhook_host,
|
||||
webhook_port=webhook_port,
|
||||
api_url=api_url, api_token=api_token,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MirrorConfig:
|
||||
id: str
|
||||
svn: SVNConfig
|
||||
gitea: GiteaConfig
|
||||
authors: Dict[str, str] = field(default_factory=dict)
|
||||
sync_interval: int = 120
|
||||
enabled: bool = True
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, d: dict) -> "MirrorConfig":
|
||||
mid = d.get("id")
|
||||
if not mid:
|
||||
raise ConfigError("mirror.id: required")
|
||||
_check_type(mid, str, "mirror.id")
|
||||
|
||||
svn_raw = d.get("svn")
|
||||
if not svn_raw:
|
||||
raise ConfigError(f"mirror '{mid}'.svn: required")
|
||||
_check_type(svn_raw, dict, f"mirror '{mid}'.svn")
|
||||
|
||||
gitea_raw = d.get("gitea")
|
||||
if not gitea_raw:
|
||||
raise ConfigError(f"mirror '{mid}'.gitea: required")
|
||||
_check_type(gitea_raw, dict, f"mirror '{mid}'.gitea")
|
||||
|
||||
svn = SVNConfig.from_dict(svn_raw, f"mirror '{mid}'.svn")
|
||||
gitea = GiteaConfig.from_dict(gitea_raw, f"mirror '{mid}'.gitea")
|
||||
|
||||
authors = d.get("authors", {})
|
||||
_check_type(authors, dict, f"mirror '{mid}'.authors")
|
||||
for k, v in authors.items():
|
||||
_check_type(k, str, f"mirror '{mid}'.authors key")
|
||||
_check_type(v, str, f"mirror '{mid}'.authors.{k}")
|
||||
|
||||
sync_interval = d.get("sync_interval", 120)
|
||||
_check_type(sync_interval, int, f"mirror '{mid}'.sync_interval")
|
||||
|
||||
enabled = d.get("enabled", True)
|
||||
_check_type(enabled, bool, f"mirror '{mid}'.enabled")
|
||||
|
||||
return cls(
|
||||
id=mid,
|
||||
svn=svn,
|
||||
gitea=gitea,
|
||||
authors=authors,
|
||||
sync_interval=sync_interval,
|
||||
enabled=enabled,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class AppConfig:
|
||||
data_dir: str = "/var/svn-mirror"
|
||||
mirrors: Dict[str, MirrorConfig] = field(default_factory=dict)
|
||||
|
||||
@classmethod
|
||||
def load(cls, path: str) -> "AppConfig":
|
||||
if not os.path.exists(path):
|
||||
raise ConfigError(f"Config file not found: {path}")
|
||||
|
||||
with open(path) as f:
|
||||
raw = yaml.safe_load(f)
|
||||
|
||||
if not raw:
|
||||
raise ConfigError("Empty config file")
|
||||
|
||||
data_dir = raw.get("data_dir", "/var/svn-mirror")
|
||||
_check_type(data_dir, str, "data_dir")
|
||||
|
||||
mirrors_raw = raw.get("mirrors", [])
|
||||
_check_type(mirrors_raw, list, "mirrors")
|
||||
|
||||
mirrors = {}
|
||||
for i, entry in enumerate(mirrors_raw):
|
||||
_check_type(entry, dict, f"mirrors[{i}]")
|
||||
mc = MirrorConfig.from_dict(entry)
|
||||
if mc.id in mirrors:
|
||||
raise ConfigError(f"Duplicate mirror id: {mc.id}")
|
||||
mirrors[mc.id] = mc
|
||||
|
||||
return cls(data_dir=data_dir, mirrors=mirrors)
|
||||
@@ -0,0 +1,142 @@
|
||||
"""SQLite mapping database for SVN↔Git revision mapping."""
|
||||
|
||||
import sqlite3
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class MappingDB:
|
||||
def __init__(self, path: Path):
|
||||
self.path = path
|
||||
self.conn = sqlite3.connect(str(path))
|
||||
self.conn.row_factory = sqlite3.Row
|
||||
self.conn.execute("PRAGMA journal_mode=WAL")
|
||||
self.conn.execute("PRAGMA synchronous=NORMAL")
|
||||
self._init_schema()
|
||||
|
||||
def _init_schema(self):
|
||||
self.conn.executescript("""
|
||||
CREATE TABLE IF NOT EXISTS schema_version (
|
||||
version INTEGER PRIMARY KEY
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS svn_to_git (
|
||||
svn_revision INTEGER NOT NULL,
|
||||
git_commit_hash TEXT NOT NULL,
|
||||
svn_branch TEXT NOT NULL,
|
||||
git_ref TEXT NOT NULL,
|
||||
synced_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
|
||||
source TEXT NOT NULL DEFAULT 'svn'
|
||||
CHECK (source IN ('svn', 'git')),
|
||||
UNIQUE(svn_revision, svn_branch)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_svn_to_git_hash
|
||||
ON svn_to_git(git_commit_hash);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_svn_to_git_svn
|
||||
ON svn_to_git(svn_revision);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sync_state (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
);
|
||||
|
||||
INSERT OR IGNORE INTO schema_version (version) VALUES (1);
|
||||
""")
|
||||
self.conn.commit()
|
||||
|
||||
def close(self):
|
||||
self.conn.close()
|
||||
|
||||
# --- Mapping CRUD ---
|
||||
|
||||
def record_mapping(
|
||||
self,
|
||||
svn_revision: int,
|
||||
git_commit_hash: str,
|
||||
svn_branch: str,
|
||||
git_ref: str,
|
||||
source: str = "svn",
|
||||
):
|
||||
self.conn.execute(
|
||||
"""INSERT OR REPLACE INTO svn_to_git
|
||||
(svn_revision, git_commit_hash, svn_branch, git_ref, source)
|
||||
VALUES (?, ?, ?, ?, ?)""",
|
||||
(svn_revision, git_commit_hash, svn_branch, git_ref, source),
|
||||
)
|
||||
self.conn.commit()
|
||||
|
||||
def get_git_hash(self, svn_revision: int, svn_branch: str) -> Optional[str]:
|
||||
row = self.conn.execute(
|
||||
"SELECT git_commit_hash FROM svn_to_git WHERE svn_revision=? AND svn_branch=?",
|
||||
(svn_revision, svn_branch),
|
||||
).fetchone()
|
||||
return row[0] if row else None
|
||||
|
||||
def get_svn_revision(self, git_commit_hash: str) -> Optional[int]:
|
||||
row = self.conn.execute(
|
||||
"SELECT svn_revision FROM svn_to_git WHERE git_commit_hash=?",
|
||||
(git_commit_hash,),
|
||||
).fetchone()
|
||||
return row[0] if row else None
|
||||
|
||||
def get_last_svn_revision(self, svn_branch: str = "trunk") -> Optional[int]:
|
||||
row = self.conn.execute(
|
||||
"SELECT MAX(svn_revision) FROM svn_to_git WHERE svn_branch=?",
|
||||
(svn_branch,),
|
||||
).fetchone()
|
||||
return row[0] if row and row[0] is not None else None
|
||||
|
||||
def mapping_exists(self, svn_revision: int, svn_branch: str) -> bool:
|
||||
row = self.conn.execute(
|
||||
"SELECT 1 FROM svn_to_git WHERE svn_revision=? AND svn_branch=?",
|
||||
(svn_revision, svn_branch),
|
||||
).fetchone()
|
||||
return row is not None
|
||||
|
||||
def get_all_mappings(self):
|
||||
return self.conn.execute(
|
||||
"SELECT svn_revision, git_commit_hash, svn_branch, git_ref, source "
|
||||
"FROM svn_to_git ORDER BY svn_revision"
|
||||
).fetchall()
|
||||
|
||||
def ref_exists(self, git_ref: str) -> bool:
|
||||
row = self.conn.execute(
|
||||
"SELECT 1 FROM svn_to_git WHERE git_ref=?", (git_ref,)
|
||||
).fetchone()
|
||||
return row is not None
|
||||
|
||||
def get_mapping_by_git_hash(self, git_hash: str) -> Optional[dict]:
|
||||
row = self.conn.execute(
|
||||
"SELECT svn_revision, svn_branch, git_ref, source "
|
||||
"FROM svn_to_git WHERE git_commit_hash=? LIMIT 1",
|
||||
(git_hash,),
|
||||
).fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
def get_max_revision_across_all_branches(self) -> Optional[int]:
|
||||
row = self.conn.execute(
|
||||
"SELECT MAX(svn_revision) FROM svn_to_git",
|
||||
).fetchone()
|
||||
return row[0] if row and row[0] is not None else None
|
||||
|
||||
def mapping_count(self) -> int:
|
||||
row = self.conn.execute("SELECT COUNT(*) FROM svn_to_git").fetchone()
|
||||
return row[0] if row else 0
|
||||
|
||||
# --- Sync State ---
|
||||
|
||||
def set_state(self, key: str, value: str):
|
||||
self.conn.execute(
|
||||
"INSERT OR REPLACE INTO sync_state (key, value) VALUES (?, ?)",
|
||||
(key, value),
|
||||
)
|
||||
self.conn.commit()
|
||||
|
||||
def get_state(self, key: str) -> Optional[str]:
|
||||
row = self.conn.execute(
|
||||
"SELECT value FROM sync_state WHERE key=?", (key,)
|
||||
).fetchone()
|
||||
return row[0] if row else None
|
||||
@@ -0,0 +1,480 @@
|
||||
"""Git→SVN sync direction.
|
||||
|
||||
Detects new Git commits (developer pushes) and translates them into SVN
|
||||
commits, maintaining the mapping DB so that SVN→Git sync skips them.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
from .config import SVNConfig
|
||||
from .mirror import Mirror
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class GitToSVNError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
# ─── Git subprocess helper (no circular import) ────────────────
|
||||
|
||||
|
||||
def _git(git_dir: str, *args: str, input: str = None, timeout: int = 120,
|
||||
check: bool = True) -> str:
|
||||
"""Run a git command in the bare canonical repo and return stdout."""
|
||||
try:
|
||||
r = subprocess.run(
|
||||
["git", "--git-dir", git_dir, *args],
|
||||
input=input,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
raise GitToSVNError("git not found on PATH")
|
||||
except subprocess.TimeoutExpired:
|
||||
raise GitToSVNError(f"git command timed out: git {' '.join(args)}")
|
||||
|
||||
if check and r.returncode != 0:
|
||||
raise GitToSVNError(
|
||||
f"git command failed (exit {r.returncode}): "
|
||||
f"git {' '.join(args)}\n{r.stderr.strip()}"
|
||||
)
|
||||
return r.stdout.strip() if r.returncode == 0 else ""
|
||||
|
||||
|
||||
# ─── Git ref → SVN path helpers ────────────────────────────────
|
||||
|
||||
|
||||
def _git_ref_to_svn_branch(git_ref: str, svn_cfg: SVNConfig) -> Optional[str]:
|
||||
"""Reverse of :func:`branch_to_git_ref`.
|
||||
|
||||
``refs/heads/master`` → ``"trunk"``
|
||||
``refs/heads/feat`` → ``"branches/feat"``
|
||||
``refs/tags/v1`` → ``"tags/v1"``
|
||||
"""
|
||||
if git_ref == "refs/heads/master":
|
||||
return svn_cfg.trunk
|
||||
if git_ref.startswith("refs/heads/"):
|
||||
branch_name = git_ref[len("refs/heads/"):]
|
||||
return f"{svn_cfg.branches}/{branch_name}"
|
||||
if git_ref.startswith("refs/tags/"):
|
||||
tag_name = git_ref[len("refs/tags/"):]
|
||||
return f"{svn_cfg.tags}/{tag_name}"
|
||||
return None
|
||||
|
||||
|
||||
def _git_ref_to_svn_url(git_ref: str, svn_url: str, svn_cfg: SVNConfig) -> Optional[str]:
|
||||
"""Return the full SVN URL for a given Git ref."""
|
||||
branch = _git_ref_to_svn_branch(git_ref, svn_cfg)
|
||||
if branch is None:
|
||||
return None
|
||||
url = svn_url.rstrip("/") + "/" + branch
|
||||
return url
|
||||
|
||||
|
||||
def _wc_path(mirror: Mirror, svn_branch: str) -> Path:
|
||||
"""Return the working-copy path for an SVN branch name.
|
||||
|
||||
Replaces ``/`` with ``_`` so branch names like ``branches/stable``
|
||||
become a flat directory name.
|
||||
"""
|
||||
sanitized = svn_branch.replace("/", "_")
|
||||
return mirror.svn_wc_dir / sanitized
|
||||
|
||||
|
||||
# ─── SVN working copy management ───────────────────────────────
|
||||
|
||||
|
||||
def _run_svn(args: list, input: bytes = None, timeout: int = 120,
|
||||
wc: Optional[Path] = None):
|
||||
"""Run an svn command. If *wc* is given, run from that directory."""
|
||||
env = {**os.environ, "LC_ALL": "C"}
|
||||
try:
|
||||
r = subprocess.run(
|
||||
["svn"] + args,
|
||||
input=input,
|
||||
capture_output=True,
|
||||
timeout=timeout,
|
||||
cwd=str(wc) if wc else None,
|
||||
env=env,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
raise GitToSVNError("svn CLI not found – is subversion installed?")
|
||||
except subprocess.TimeoutExpired:
|
||||
raise GitToSVNError(f"svn command timed out: svn {' '.join(args)}")
|
||||
|
||||
if r.returncode != 0:
|
||||
msg = (r.stderr or r.stdout or b"").decode("utf-8", errors="replace").strip()
|
||||
raise GitToSVNError(f"svn failed (exit {r.returncode}): {msg}")
|
||||
return r
|
||||
|
||||
|
||||
def _ensure_wc(mirror: Mirror, svn_branch: str):
|
||||
"""Ensure the SVN working copy for *svn_branch* exists and is at HEAD."""
|
||||
wc = _wc_path(mirror, svn_branch)
|
||||
svn_url = mirror.config.svn.url.rstrip("/") + "/" + svn_branch
|
||||
if not (wc / ".svn").exists():
|
||||
logger.info("Checking out SVN WC for %s …", svn_branch)
|
||||
wc.mkdir(parents=True, exist_ok=True)
|
||||
_run_svn(["checkout", svn_url, str(wc)], timeout=300)
|
||||
else:
|
||||
logger.debug("Updating SVN WC for %s …", svn_branch)
|
||||
_run_svn(["revert", "-R", "."], timeout=120, wc=wc)
|
||||
_run_svn(["update"], timeout=120, wc=wc)
|
||||
return wc
|
||||
|
||||
|
||||
# ─── Author mapping (reverse) ──────────────────────────────────
|
||||
|
||||
|
||||
def _parse_git_author(author_line: str) -> Tuple[str, str]:
|
||||
"""Split ``"Name <email>"`` into (name, email).
|
||||
|
||||
Returns (author_line, "") if the format doesn't match.
|
||||
"""
|
||||
if "<" in author_line and author_line.endswith(">"):
|
||||
name = author_line[:author_line.index("<")].strip()
|
||||
email = author_line[author_line.index("<") + 1:-1].strip()
|
||||
return (name, email)
|
||||
return (author_line, "")
|
||||
|
||||
|
||||
def _git_author_to_svn_username(author_name: str, author_email: str,
|
||||
authors: Dict[str, str]) -> str:
|
||||
"""Try to map a Git author (name, email) back to an SVN username.
|
||||
|
||||
The ``authors`` dict maps SVN username → ``"Name <email>"``. We
|
||||
reverse the lookup by checking both the name and the email.
|
||||
"""
|
||||
for svn_user, mapped in authors.items():
|
||||
name, email = _parse_git_author(mapped)
|
||||
if name == author_name or email == author_email:
|
||||
return svn_user
|
||||
# Fallback: try name as-is
|
||||
return author_name
|
||||
|
||||
|
||||
# ─── Git commit detection ──────────────────────────────────────
|
||||
|
||||
|
||||
def _tracked_refs(git_dir: str, svn_cfg: SVNConfig) -> List[Tuple[str, str]]:
|
||||
"""Return ``(git_ref, svn_branch)`` pairs for commits → SVN.
|
||||
|
||||
Only returns ``refs/heads/*`` — tags are handled separately by
|
||||
:func:`_detect_new_git_tags`.
|
||||
"""
|
||||
refs: List[Tuple[str, str]] = []
|
||||
out = _git(git_dir, "for-each-ref", "--format=%(refname)", "refs/heads/",
|
||||
timeout=30, check=False)
|
||||
for ref in out.splitlines():
|
||||
ref = ref.strip()
|
||||
if not ref:
|
||||
continue
|
||||
branch = _git_ref_to_svn_branch(ref, svn_cfg)
|
||||
if branch is not None:
|
||||
refs.append((ref, branch))
|
||||
return refs
|
||||
|
||||
|
||||
def detect_new_git_commits(mirror: Mirror) -> List[Tuple[str, str, str]]:
|
||||
"""Return ``[(commit_hash, svn_branch, author_name), …]`` oldest-first.
|
||||
|
||||
Walks each tracked Git ref backwards from its tip and collects
|
||||
commits that are NOT yet recorded in the mapping DB.
|
||||
"""
|
||||
git_dir = str(mirror.canonical_dir)
|
||||
svn_cfg = mirror.config.svn
|
||||
new: List[Tuple[str, str, str]] = []
|
||||
|
||||
for git_ref, svn_branch in _tracked_refs(git_dir, svn_cfg):
|
||||
out = _git(git_dir, "rev-list", "--topo-order", "--no-merges",
|
||||
git_ref, timeout=60, check=False)
|
||||
if not out:
|
||||
continue
|
||||
|
||||
commits = out.splitlines()
|
||||
unmapped: List[str] = []
|
||||
found_boundary = False
|
||||
|
||||
for ch in commits:
|
||||
if not ch:
|
||||
continue
|
||||
if mirror.db.get_svn_revision(ch) is not None:
|
||||
found_boundary = True
|
||||
break
|
||||
unmapped.append(ch)
|
||||
|
||||
if not found_boundary:
|
||||
logger.debug("No SVN ancestor found for %s – skipping", git_ref)
|
||||
continue
|
||||
|
||||
# Add oldest-first
|
||||
for ch in reversed(unmapped):
|
||||
author = _git(git_dir, "log", "--format=%aN <%aE>", "-1", ch,
|
||||
timeout=30)
|
||||
new.append((ch, svn_branch, author))
|
||||
|
||||
return new
|
||||
|
||||
|
||||
# ─── Applying a single Git commit to SVN ────────────────────────
|
||||
|
||||
|
||||
def _apply_diff_to_wc(mirror: Mirror, commit_hash: str, svn_branch: str, wc: Path):
|
||||
"""Apply the file changes from *commit_hash* to the SVN working copy."""
|
||||
git_dir = str(mirror.canonical_dir)
|
||||
|
||||
out = _git(git_dir, "diff-tree", "--no-commit-id", "-r", "--name-status",
|
||||
"-z", commit_hash, timeout=60)
|
||||
if not out:
|
||||
logger.debug("Commit %s has no file changes", commit_hash[:8])
|
||||
return
|
||||
|
||||
# Parse null-separated output: "A\0README\0M\0src/main.rs\0..."
|
||||
# Status may include a score suffix, e.g. R100, C075 – extract the letter.
|
||||
parts = out.split("\0")
|
||||
i = 0
|
||||
while i < len(parts):
|
||||
line = parts[i].strip()
|
||||
if not line:
|
||||
i += 1
|
||||
continue
|
||||
status = line[0]
|
||||
if status not in ("A", "C", "D", "M", "R", "T"):
|
||||
i += 1
|
||||
continue
|
||||
i += 1
|
||||
if i >= len(parts):
|
||||
break
|
||||
path = parts[i]
|
||||
i += 1
|
||||
if status == "D":
|
||||
_wc_delete(wc, path)
|
||||
elif status in ("A", "C", "M", "T"):
|
||||
_wc_add_or_modify(wc, git_dir, commit_hash, path)
|
||||
elif status == "R":
|
||||
if i < len(parts):
|
||||
dest = parts[i]
|
||||
i += 1
|
||||
_wc_rename(wc, path, dest)
|
||||
|
||||
|
||||
def _wc_delete(wc: Path, path: str):
|
||||
full = wc / path
|
||||
if full.exists():
|
||||
_run_svn(["delete", "--force", str(full)], wc=wc)
|
||||
logger.debug(" D %s", path)
|
||||
|
||||
|
||||
def _wc_add_or_modify(wc: Path, git_dir: str, commit_hash: str, path: str):
|
||||
full = wc / path
|
||||
parent = full.parent
|
||||
if not parent.exists():
|
||||
parent.mkdir(parents=True, exist_ok=True)
|
||||
_run_svn(["add", "--parents", str(parent)], wc=wc)
|
||||
|
||||
is_new = not full.exists()
|
||||
full.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
r = subprocess.run(
|
||||
["git", "--git-dir", git_dir, "show", f"{commit_hash}:{path}"],
|
||||
capture_output=True, timeout=60,
|
||||
)
|
||||
if r.returncode == 0:
|
||||
full.write_bytes(r.stdout)
|
||||
|
||||
if is_new:
|
||||
_run_svn(["add", "--parents", str(full)], wc=wc)
|
||||
logger.debug(" M %s", path)
|
||||
|
||||
|
||||
def _wc_rename(wc: Path, src: str, dest: str):
|
||||
src_full = wc / src
|
||||
dest_full = wc / dest
|
||||
if src_full.exists():
|
||||
dest_full.parent.mkdir(parents=True, exist_ok=True)
|
||||
_run_svn(["move", str(src_full), str(dest_full)], wc=wc)
|
||||
logger.debug(" R %s → %s", src, dest)
|
||||
|
||||
|
||||
# ─── Parsing helpers ──────────────────────────────────────────
|
||||
|
||||
|
||||
def _parse_commit_revision(result: subprocess.CompletedProcess) -> int:
|
||||
"""Parse revision from ``svn commit`` output (on either stdout or stderr)."""
|
||||
text = (result.stdout + b"\n" + result.stderr).decode("utf-8", errors="replace")
|
||||
import re
|
||||
m = re.search(r"Committed revision\s+(\d+)", text)
|
||||
if m:
|
||||
return int(m.group(1))
|
||||
raise GitToSVNError(
|
||||
f"Cannot determine committed revision from svn output:\n{text[:500]}"
|
||||
)
|
||||
|
||||
|
||||
# ─── Main entry point ──────────────────────────────────────────
|
||||
|
||||
|
||||
def _detect_new_git_tags(mirror: Mirror) -> List[Tuple[str, str, int, str, str]]:
|
||||
"""Find Git tags not yet pushed to SVN.
|
||||
|
||||
Returns ``[(tag_ref, svn_branch, source_svn_rev, author_line), …]``.
|
||||
"""
|
||||
git_dir = str(mirror.canonical_dir)
|
||||
svn_cfg = mirror.config.svn
|
||||
new: List[Tuple[str, str, int, str, str]] = []
|
||||
|
||||
out = _git(git_dir, "for-each-ref", "--format=%(refname)", "refs/tags/",
|
||||
timeout=30, check=False)
|
||||
for tag_ref in out.splitlines():
|
||||
tag_ref = tag_ref.strip()
|
||||
if not tag_ref:
|
||||
continue
|
||||
svn_branch = _git_ref_to_svn_branch(tag_ref, svn_cfg)
|
||||
if svn_branch is None:
|
||||
continue
|
||||
# Already pushed to SVN?
|
||||
if mirror.db.ref_exists(tag_ref):
|
||||
continue
|
||||
|
||||
target = _git(git_dir, "rev-parse", tag_ref, timeout=30)
|
||||
mapping = mirror.db.get_mapping_by_git_hash(target)
|
||||
if mapping is None:
|
||||
logger.debug(
|
||||
"Tag %s target commit not in mapping DB – skipping", tag_ref,
|
||||
)
|
||||
continue
|
||||
source_rev = mapping["svn_revision"]
|
||||
source_branch = mapping["svn_branch"]
|
||||
author = _git(git_dir, "log", "--format=%aN <%aE>", "-1", target,
|
||||
timeout=30)
|
||||
new.append((tag_ref, svn_branch, source_rev, source_branch, author))
|
||||
|
||||
return new
|
||||
|
||||
|
||||
def sync_git_to_svn(mirror: Mirror) -> int:
|
||||
"""Push all pending Git commits and tags to SVN (thread-safe).
|
||||
|
||||
Returns the number of SVN revisions created.
|
||||
"""
|
||||
with mirror.sync_lock():
|
||||
return _sync_git_to_svn_impl(mirror)
|
||||
|
||||
|
||||
def _sync_git_to_svn_impl(mirror: Mirror) -> int:
|
||||
git_dir = str(mirror.canonical_dir)
|
||||
svn_cfg = mirror.config.svn
|
||||
count = 0
|
||||
|
||||
# ── 1. Push new commits on branches ───────────────────────
|
||||
commits = detect_new_git_commits(mirror)
|
||||
for commit_hash, svn_branch, author_line in commits:
|
||||
wc = _ensure_wc(mirror, svn_branch)
|
||||
|
||||
logger.info(
|
||||
"Pushing Git commit %s → SVN %s", commit_hash[:8], svn_branch,
|
||||
)
|
||||
|
||||
try:
|
||||
_apply_diff_to_wc(mirror, commit_hash, svn_branch, wc)
|
||||
|
||||
message = _git(git_dir, "log", "--format=%B", "-1", commit_hash,
|
||||
timeout=30)
|
||||
|
||||
svn_username = author_line
|
||||
name, email = _parse_git_author(author_line)
|
||||
if name != author_line and email:
|
||||
svn_username = _git_author_to_svn_username(
|
||||
name, email, mirror.config.authors,
|
||||
)
|
||||
|
||||
result = _run_svn(
|
||||
["commit", "--file", "-", "--username", svn_username],
|
||||
input=message.encode("utf-8"),
|
||||
wc=wc, timeout=120,
|
||||
)
|
||||
new_rev = _parse_commit_revision(result)
|
||||
|
||||
mirror.db.record_mapping(
|
||||
new_rev, commit_hash, svn_branch,
|
||||
_git_ref_from_branch(svn_branch, svn_cfg),
|
||||
source="git",
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Pushed Git %s → SVN r%d (%s)",
|
||||
commit_hash[:8], new_rev, svn_branch,
|
||||
)
|
||||
count += 1
|
||||
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to push Git commit %s to SVN branch %s",
|
||||
commit_hash[:8], svn_branch,
|
||||
)
|
||||
raise
|
||||
|
||||
# ── 2. Push new tags ──────────────────────────────────────
|
||||
svn_url = svn_cfg.url.rstrip("/")
|
||||
for tag_ref, svn_branch, source_rev, source_branch, author_line \
|
||||
in _detect_new_git_tags(mirror):
|
||||
logger.info("Pushing Git tag %s → SVN %s", tag_ref, svn_branch)
|
||||
|
||||
try:
|
||||
# Build source URL (the SVN path being tagged)
|
||||
src_url = f"{svn_url}/{source_branch}"
|
||||
dst_url = f"{svn_url}/{svn_branch}"
|
||||
|
||||
message = _git(git_dir, "log", "--format=%B", "-1", tag_ref,
|
||||
timeout=30)
|
||||
|
||||
svn_username = author_line
|
||||
name, email = _parse_git_author(author_line)
|
||||
if name != author_line and email:
|
||||
svn_username = _git_author_to_svn_username(
|
||||
name, email, mirror.config.authors,
|
||||
)
|
||||
|
||||
result = _run_svn(
|
||||
["copy", "-r", str(source_rev), src_url, dst_url,
|
||||
"-m", message, "--username", svn_username],
|
||||
timeout=120,
|
||||
)
|
||||
new_rev = _parse_commit_revision(result)
|
||||
|
||||
# The commit hash stored for a tag is the target commit hash
|
||||
target = _git(git_dir, "rev-parse", tag_ref, timeout=30)
|
||||
mirror.db.record_mapping(
|
||||
new_rev, target, svn_branch, tag_ref, source="git",
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Pushed Git tag %s → SVN r%d (%s)", tag_ref, new_rev,
|
||||
svn_branch,
|
||||
)
|
||||
count += 1
|
||||
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to push Git tag %s to SVN", tag_ref,
|
||||
)
|
||||
raise
|
||||
|
||||
return count
|
||||
|
||||
|
||||
def _git_ref_from_branch(svn_branch: str, svn_cfg: SVNConfig) -> str:
|
||||
"""Convert SVN branch name to Git ref (mirror of :func:`branch_to_git_ref`)."""
|
||||
if svn_branch == svn_cfg.trunk:
|
||||
return "refs/heads/master"
|
||||
if svn_branch.startswith(svn_cfg.branches + "/"):
|
||||
return "refs/heads/" + svn_branch[len(svn_cfg.branches) + 1:]
|
||||
if svn_branch.startswith(svn_cfg.tags + "/"):
|
||||
return "refs/tags/" + svn_branch[len(svn_cfg.tags) + 1:]
|
||||
return "refs/heads/" + svn_branch
|
||||
@@ -0,0 +1,138 @@
|
||||
"""Gitea integration – push/pull between canonical repo and Gitea's on-disk repo."""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from .mirror import Mirror
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class GiteaError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def _get_gitea_repo_path(mirror: Mirror) -> Path:
|
||||
"""Return the filesystem path to Gitea's bare repo for this mirror."""
|
||||
try:
|
||||
return Path(mirror.config.gitea.repo_dir)
|
||||
except Exception as e:
|
||||
raise GiteaError(str(e))
|
||||
|
||||
|
||||
# ─── Git subprocess helper ─────────────────────────────────────
|
||||
|
||||
|
||||
def _git(*args: str, timeout: int = 300, check: bool = True) -> str:
|
||||
"""Run a git command (uses CWD for repo discovery)."""
|
||||
try:
|
||||
r = subprocess.run(
|
||||
["git"] + list(args),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
env={**os.environ, "LC_ALL": "C"},
|
||||
)
|
||||
except FileNotFoundError:
|
||||
raise GiteaError("git not found on PATH")
|
||||
except subprocess.TimeoutExpired:
|
||||
raise GiteaError(f"git timed out: {' '.join(args)}")
|
||||
if check and r.returncode != 0:
|
||||
raise GiteaError(
|
||||
f"git failed (exit {r.returncode}): "
|
||||
f"git {' '.join(args)}\n{r.stderr.strip()}"
|
||||
)
|
||||
return r.stdout.strip() if r.returncode == 0 else ""
|
||||
|
||||
|
||||
# ─── Push canonical → Gitea ───────────────────────────────────
|
||||
|
||||
|
||||
def push_to_gitea(mirror: Mirror) -> bool:
|
||||
"""Push all heads and tags from the canonical repo to Gitea's bare repo.
|
||||
|
||||
Returns True if anything was pushed.
|
||||
"""
|
||||
gitea_path = _get_gitea_repo_path(mirror)
|
||||
if not gitea_path.exists():
|
||||
logger.warning("Gitea repo not found at %s – skipping push", gitea_path)
|
||||
return False
|
||||
|
||||
canonical = str(mirror.canonical_dir)
|
||||
|
||||
# Check if there's anything to push by comparing refs
|
||||
out = _git("--git-dir", canonical, "for-each-ref",
|
||||
"--format=%(objectname) %(refname)",
|
||||
"refs/heads/", "refs/tags/",
|
||||
timeout=30, check=False)
|
||||
if out:
|
||||
has_new = False
|
||||
for line in out.splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
obj_hash, ref = line.split(None, 1)
|
||||
gitea_hash = _git("--git-dir", str(gitea_path), "rev-parse",
|
||||
"--verify", "--quiet", ref,
|
||||
timeout=30, check=False)
|
||||
if obj_hash != gitea_hash:
|
||||
has_new = True
|
||||
break
|
||||
if not has_new:
|
||||
logger.debug("No new commits to push to Gitea")
|
||||
return False
|
||||
|
||||
logger.info("Pushing canonical → Gitea (%s) …", gitea_path)
|
||||
# Use `git push --prune` with explicit refspecs to mirror branches and tags
|
||||
_git(
|
||||
"--git-dir", canonical, "push", "--prune",
|
||||
str(gitea_path),
|
||||
"+refs/heads/*:refs/heads/*",
|
||||
"+refs/tags/*:refs/tags/*",
|
||||
timeout=300,
|
||||
)
|
||||
logger.info("Push to Gitea complete")
|
||||
return True
|
||||
|
||||
|
||||
# ─── Fetch Gitea → canonical ──────────────────────────────────
|
||||
|
||||
|
||||
def fetch_from_gitea(mirror: Mirror, ref: Optional[str] = None) -> bool:
|
||||
"""Fetch refs from Gitea's bare repo into the canonical repo.
|
||||
|
||||
If *ref* is given (e.g. ``refs/heads/master``), only that ref is
|
||||
fetched. Otherwise all heads are fetched.
|
||||
|
||||
Returns True if anything new was fetched.
|
||||
"""
|
||||
gitea_path = _get_gitea_repo_path(mirror)
|
||||
if not gitea_path.exists():
|
||||
raise GiteaError(f"Gitea repo not found at {gitea_path}")
|
||||
|
||||
canonical = str(mirror.canonical_dir)
|
||||
|
||||
if ref:
|
||||
refspec = f"+{ref}:{ref}"
|
||||
else:
|
||||
refspec = "+refs/heads/*:refs/heads/*"
|
||||
|
||||
logger.info("Fetching %s → canonical …", gitea_path)
|
||||
_git(
|
||||
"--git-dir", canonical, "fetch", "--prune",
|
||||
str(gitea_path), refspec,
|
||||
timeout=300,
|
||||
)
|
||||
|
||||
# Check if anything changed
|
||||
out = _git("--git-dir", canonical, "rev-list", "--count",
|
||||
f"HEAD..FETCH_HEAD", timeout=30, check=False)
|
||||
if out and out.strip() != "0":
|
||||
logger.info("Fetched new commits from Gitea")
|
||||
return True
|
||||
|
||||
logger.debug("No new commits from Gitea")
|
||||
return False
|
||||
@@ -0,0 +1,426 @@
|
||||
"""Mirror lifecycle management."""
|
||||
|
||||
import fcntl
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Generator, List, Optional, Tuple
|
||||
|
||||
from .config import MirrorConfig
|
||||
from .db import MappingDB
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MirrorError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class Mirror:
|
||||
"""Manages a single SVN↔Git mirror's lifecycle on disk."""
|
||||
|
||||
def __init__(self, config: MirrorConfig, data_dir: Path):
|
||||
self.config = config
|
||||
self.base_dir = data_dir / "mirrors" / config.id
|
||||
self.canonical_dir = self.base_dir / "canonical-repo.git"
|
||||
self.svn_wc_dir = self.base_dir / "svn-wc"
|
||||
self.mapping_db_path = self.base_dir / "mapping.db"
|
||||
self.authors_path = self.base_dir / "authors.txt"
|
||||
self.state_path = self.base_dir / "state.json"
|
||||
self.lock_path = self.base_dir / "sync.lock"
|
||||
self._db: Optional[MappingDB] = None
|
||||
|
||||
@contextmanager
|
||||
def sync_lock(self) -> Generator[None, None, None]:
|
||||
"""Acquire an exclusive flock on the mirror's lock file.
|
||||
|
||||
Guarantees that only one process/thread syncs this mirror at a
|
||||
time. Releases automatically when the context exits.
|
||||
"""
|
||||
self.lock_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fd = os.open(str(self.lock_path), os.O_CREAT | os.O_RDWR, 0o644)
|
||||
try:
|
||||
fcntl.flock(fd, fcntl.LOCK_EX)
|
||||
yield
|
||||
finally:
|
||||
fcntl.flock(fd, fcntl.LOCK_UN)
|
||||
os.close(fd)
|
||||
|
||||
@property
|
||||
def db(self) -> MappingDB:
|
||||
if self._db is None:
|
||||
self._db = MappingDB(self.mapping_db_path)
|
||||
return self._db
|
||||
|
||||
@property
|
||||
def exists(self) -> bool:
|
||||
return self.base_dir.exists()
|
||||
|
||||
# ─── Lifecycle ────────────────────────────────────────────
|
||||
|
||||
def create(self) -> None:
|
||||
"""Create directory structure, init bare Git repo and mapping DB."""
|
||||
if self.exists:
|
||||
raise MirrorError(
|
||||
f"Mirror {self.config.id} already exists at {self.base_dir}"
|
||||
)
|
||||
|
||||
logger.info("Creating mirror %s at %s", self.config.id, self.base_dir)
|
||||
self.base_dir.mkdir(parents=True, exist_ok=True)
|
||||
self.svn_wc_dir.mkdir(exist_ok=True)
|
||||
|
||||
# Init bare Git repo (our canonical copy)
|
||||
self._git("init", "--bare", str(self.canonical_dir))
|
||||
|
||||
# Init mapping DB
|
||||
_ = self.db
|
||||
|
||||
# Write authors file for git svn
|
||||
self._write_authors_file()
|
||||
|
||||
# Write initial state
|
||||
self._write_state({
|
||||
"initial_import_done": False,
|
||||
"last_svn_revision": None,
|
||||
"last_git_commit": None,
|
||||
"created_at": _now_iso(),
|
||||
"last_sync_at": None,
|
||||
})
|
||||
|
||||
logger.info("Mirror %s created successfully", self.config.id)
|
||||
|
||||
def destroy(self) -> None:
|
||||
"""Remove all mirror data from disk."""
|
||||
if not self.exists:
|
||||
raise MirrorError(f"Mirror {self.config.id} does not exist")
|
||||
|
||||
logger.warning("Destroying mirror %s at %s", self.config.id, self.base_dir)
|
||||
shutil.rmtree(self.base_dir)
|
||||
logger.info("Mirror %s destroyed", self.config.id)
|
||||
|
||||
def init_import(self) -> None:
|
||||
"""Run initial SVN import via git svn clone --no-metadata."""
|
||||
if not self.exists:
|
||||
self.create()
|
||||
|
||||
state = self._read_state()
|
||||
if state.get("initial_import_done"):
|
||||
logger.info("Mirror %s already imported (skipping)", self.config.id)
|
||||
return
|
||||
|
||||
self._check_git_svn_available()
|
||||
|
||||
import_dir = self.base_dir / ".import-tmp"
|
||||
if import_dir.exists():
|
||||
shutil.rmtree(import_dir)
|
||||
|
||||
try:
|
||||
self._run_git_svn_clone(import_dir)
|
||||
self._push_import_to_bare(import_dir)
|
||||
mappings = self._read_rev_maps(import_dir)
|
||||
self._write_mappings_to_db(mappings)
|
||||
self._finalize_import_state()
|
||||
except BaseException:
|
||||
logger.exception("Initial import failed for %s", self.config.id)
|
||||
if import_dir.exists():
|
||||
shutil.rmtree(import_dir)
|
||||
raise
|
||||
else:
|
||||
shutil.rmtree(import_dir, ignore_errors=True)
|
||||
|
||||
# ─── Status ───────────────────────────────────────────────
|
||||
|
||||
def status(self) -> Dict[str, Any]:
|
||||
"""Return a snapshot of mirror state for display."""
|
||||
if not self.base_dir.exists():
|
||||
return {
|
||||
"id": self.config.id,
|
||||
"status": "not_created",
|
||||
"svn_url": self.config.svn.url,
|
||||
"enabled": self.config.enabled,
|
||||
}
|
||||
|
||||
state = self._read_state()
|
||||
rev = self.db.get_max_revision_across_all_branches() if self.mapping_db_path.exists() else None
|
||||
count = self.db.mapping_count() if self.mapping_db_path.exists() else 0
|
||||
|
||||
return {
|
||||
"id": self.config.id,
|
||||
"status": "created",
|
||||
"imported": state.get("initial_import_done", False),
|
||||
"svn_url": self.config.svn.url,
|
||||
"enabled": self.config.enabled,
|
||||
"last_svn_revision": rev,
|
||||
"mapping_count": count,
|
||||
"base_dir": str(self.base_dir),
|
||||
"created_at": state.get("created_at"),
|
||||
"last_sync_at": state.get("last_sync_at"),
|
||||
}
|
||||
|
||||
# ─── Private helpers ──────────────────────────────────────
|
||||
|
||||
def _check_git_svn_available(self):
|
||||
try:
|
||||
r = subprocess.run(
|
||||
["git", "svn", "--version"],
|
||||
capture_output=True, text=True, timeout=30,
|
||||
)
|
||||
if r.returncode != 0:
|
||||
raise MirrorError(
|
||||
"git svn not available (exit code %d): %s" % (r.returncode, r.stderr)
|
||||
)
|
||||
except FileNotFoundError:
|
||||
raise MirrorError("git svn not found – is git-svn installed?")
|
||||
except subprocess.TimeoutExpired:
|
||||
raise MirrorError("git svn --version timed out")
|
||||
|
||||
def _run_git_svn_clone(self, target: Path):
|
||||
"""Execute git svn clone --no-metadata for the initial import."""
|
||||
svn_cfg = self.config.svn
|
||||
args = ["git", "svn", "clone", "--no-metadata"]
|
||||
|
||||
if svn_cfg.layout == "std":
|
||||
args.append("--stdlayout")
|
||||
else:
|
||||
args.extend(["-T", svn_cfg.trunk])
|
||||
args.extend(["-b", svn_cfg.branches])
|
||||
args.extend(["-t", svn_cfg.tags])
|
||||
|
||||
if self.config.authors:
|
||||
args.extend(["--authors-file", str(self.authors_path)])
|
||||
|
||||
args.extend([svn_cfg.url, str(target)])
|
||||
|
||||
logger.info("Starting git svn clone (this may take a while)…")
|
||||
logger.debug("Running: %s", " ".join(str(a) for a in args))
|
||||
|
||||
process = subprocess.Popen(
|
||||
args,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
)
|
||||
|
||||
for line in process.stdout or []:
|
||||
line = line.rstrip()
|
||||
if line:
|
||||
logger.info("[git svn] %s", line)
|
||||
|
||||
ret = process.wait()
|
||||
if ret != 0:
|
||||
raise MirrorError(f"git svn clone failed with exit code {ret}")
|
||||
|
||||
logger.info("git svn clone completed")
|
||||
|
||||
def _push_import_to_bare(self, import_dir: Path):
|
||||
"""Push all refs from the temporary clone into our canonical bare repo.
|
||||
|
||||
git svn stores branches under refs/remotes/origin/*, NOT refs/heads/*.
|
||||
We need to map:
|
||||
refs/remotes/origin/trunk → refs/heads/master
|
||||
refs/remotes/origin/branches/* → refs/heads/*
|
||||
refs/remotes/origin/tags/* → refs/tags/*
|
||||
"""
|
||||
logger.info("Pushing imported refs to canonical bare repo…")
|
||||
|
||||
# -- setup remote ---------------------------------------------------
|
||||
self._git("-C", str(import_dir), "remote", "remove", "origin",
|
||||
check=False)
|
||||
self._git("-C", str(import_dir), "remote", "add", "bare",
|
||||
str(self.canonical_dir))
|
||||
|
||||
# -- push trunk → master --------------------------------------------
|
||||
self._git("-C", str(import_dir), "push", "bare",
|
||||
"refs/remotes/origin/trunk:refs/heads/master",
|
||||
"--force")
|
||||
|
||||
# -- push other branches --------------------------------------------
|
||||
# List remote refs and push each as a local branch
|
||||
r = self._git("-C", str(import_dir), "for-each-ref",
|
||||
"--format=%(refname)", "refs/remotes/origin/",
|
||||
check=True)
|
||||
for ref in r.stdout.strip().splitlines():
|
||||
ref = ref.strip()
|
||||
if not ref:
|
||||
continue
|
||||
# Determine target ref
|
||||
if ref == "refs/remotes/origin/trunk":
|
||||
continue # already pushed above
|
||||
if ref.startswith("refs/remotes/origin/tags/"):
|
||||
# git svn uses refs/remotes/origin/tags/* – convert to real tags
|
||||
tag_name = ref.removeprefix("refs/remotes/origin/tags/")
|
||||
target = f"refs/tags/{tag_name}"
|
||||
else:
|
||||
branch_name = ref.removeprefix("refs/remotes/origin/")
|
||||
target = f"refs/heads/{branch_name}"
|
||||
self._git("-C", str(import_dir), "push", "bare",
|
||||
f"{ref}:{target}", "--force")
|
||||
|
||||
# -- push any real tags git svn might have created ------------------
|
||||
self._git("-C", str(import_dir), "push", "bare", "--tags", "--force",
|
||||
check=False)
|
||||
|
||||
def _read_rev_maps(self, import_dir: Path) -> List[Tuple[int, str, str, str]]:
|
||||
"""
|
||||
Parse .rev_map.* files from the git svn temp clone.
|
||||
|
||||
Returns list of (svn_revision, git_hash, svn_branch, git_ref).
|
||||
"""
|
||||
svn_dir = import_dir / ".git" / "svn"
|
||||
if not svn_dir.exists():
|
||||
logger.warning("No .git/svn directory found – cannot build mapping")
|
||||
return []
|
||||
|
||||
mappings: List[Tuple[int, str, str, str]] = []
|
||||
for rev_map in sorted(svn_dir.rglob(".rev_map.*")):
|
||||
if not rev_map.is_file():
|
||||
continue
|
||||
|
||||
# Relative path from .git/svn/ gives us the ref, e.g.:
|
||||
# .git/svn/refs/remotes/origin/trunk/.rev_map.<UUID>
|
||||
# → ref = refs/remotes/origin/trunk
|
||||
ref_path = rev_map.parent.relative_to(svn_dir)
|
||||
git_ref = str(ref_path).replace("\\", "/")
|
||||
svn_branch = self._ref_to_svn_branch(git_ref)
|
||||
|
||||
data = rev_map.read_bytes()
|
||||
size = 24 # 4 bytes revision + 20 bytes SHA-1
|
||||
zero_hash = "0000000000000000000000000000000000000000"
|
||||
entries = 0
|
||||
for i in range(0, len(data) - size + 1, size):
|
||||
rev_raw = data[i:i+4]
|
||||
sha_raw = data[i+4:i+24]
|
||||
if len(rev_raw) < 4 or len(sha_raw) < 20:
|
||||
break
|
||||
rev = int.from_bytes(rev_raw, "big")
|
||||
sha = sha_raw.hex()
|
||||
# git svn sometimes pads the file with a zero-hash record;
|
||||
# skip those.
|
||||
if sha == zero_hash:
|
||||
logger.debug(
|
||||
"Skipping zero-hash rev_map entry at offset %d", i
|
||||
)
|
||||
continue
|
||||
mappings.append((rev, sha, svn_branch, git_ref))
|
||||
entries += 1
|
||||
|
||||
logger.debug(
|
||||
"Read %d entries from %s (branch: %s)",
|
||||
entries, rev_map.name, svn_branch,
|
||||
)
|
||||
|
||||
return mappings
|
||||
|
||||
def _ref_to_svn_branch(self, git_ref: str) -> str:
|
||||
"""Convert a git svn remote ref path to an SVN branch name.
|
||||
|
||||
Input examples:
|
||||
refs/remotes/origin/trunk → trunk
|
||||
refs/remotes/origin/branches/feat → branches/feat
|
||||
refs/remotes/origin/tags/v1.0 → tags/v1.0
|
||||
"""
|
||||
prefix = "refs/remotes/origin/"
|
||||
if git_ref.startswith(prefix):
|
||||
return git_ref[len(prefix):]
|
||||
return git_ref
|
||||
|
||||
def _write_mappings_to_db(self, mappings: List[Tuple[int, str, str, str]]):
|
||||
"""Bulk-insert parsed mappings into SQLite."""
|
||||
logger.info("Writing %d mapping entries to DB…", len(mappings))
|
||||
for rev, sha, branch, ref in mappings:
|
||||
self.db.record_mapping(rev, sha, branch, ref, source="svn")
|
||||
logger.info("Mapping DB now has %d entries", self.db.mapping_count())
|
||||
|
||||
def _finalize_import_state(self):
|
||||
"""Update state.json + DB after successful import."""
|
||||
last_rev = self.db.get_max_revision_across_all_branches()
|
||||
|
||||
self._write_state({
|
||||
"initial_import_done": True,
|
||||
"last_svn_revision": last_rev,
|
||||
"last_git_commit": None,
|
||||
"created_at": self._read_state().get("created_at", _now_iso()),
|
||||
"last_sync_at": _now_iso(),
|
||||
})
|
||||
|
||||
self.db.set_state("last_svn_revision", str(last_rev))
|
||||
self.db.set_state("last_git_commit", "")
|
||||
logger.info(
|
||||
"Initial import complete for %s (last SVN rev: %s)",
|
||||
self.config.id, last_rev,
|
||||
)
|
||||
|
||||
# ─── File I/O ─────────────────────────────────────────────
|
||||
|
||||
def _write_authors_file(self):
|
||||
"""Write authors.txt for git svn including ALL SVN authors.
|
||||
|
||||
Scans the SVN log for all unique authors, then merges with
|
||||
configured mappings. Any author without a configured mapping
|
||||
defaults to ``user <user@svn>`` so that git svn does not abort.
|
||||
"""
|
||||
all_authors = dict(self.config.authors)
|
||||
|
||||
# Discover all SVN authors from the remote repo
|
||||
try:
|
||||
r = subprocess.run(
|
||||
["svn", "log", "-q", "--xml", self.config.svn.url],
|
||||
capture_output=True, text=True, check=True, timeout=300,
|
||||
)
|
||||
import xml.etree.ElementTree as ET
|
||||
root = ET.fromstring(r.stdout)
|
||||
for entry in root.findall(".//logentry"):
|
||||
el = entry.find("author")
|
||||
if el is not None and el.text:
|
||||
svn_user = el.text.strip()
|
||||
if svn_user and svn_user not in all_authors:
|
||||
all_authors[svn_user] = f"{svn_user} <{svn_user}@svn>"
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Could not discover SVN authors (svn log failed); "
|
||||
"using configured authors only"
|
||||
)
|
||||
|
||||
with open(self.authors_path, "w") as f:
|
||||
for svn_user in sorted(all_authors):
|
||||
f.write(f"{svn_user} = {all_authors[svn_user]}\n")
|
||||
|
||||
def _read_state(self) -> Dict[str, Any]:
|
||||
if not self.state_path.exists():
|
||||
return {}
|
||||
with open(self.state_path) as f:
|
||||
return json.load(f)
|
||||
|
||||
def _write_state(self, state: Dict[str, Any]):
|
||||
with open(self.state_path, "w") as f:
|
||||
json.dump(state, f, indent=2, default=str)
|
||||
|
||||
# ─── Git subprocess helper ────────────────────────────────
|
||||
|
||||
@staticmethod
|
||||
def _git(*args: str, check: bool = True) -> subprocess.CompletedProcess:
|
||||
try:
|
||||
r = subprocess.run(
|
||||
["git", *args],
|
||||
capture_output=True, text=True, timeout=600,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
raise MirrorError("git not found on PATH")
|
||||
except subprocess.TimeoutExpired:
|
||||
raise MirrorError("git command timed out: git " + " ".join(args))
|
||||
|
||||
if check and r.returncode != 0:
|
||||
raise MirrorError(
|
||||
f"git command failed (exit {r.returncode}): "
|
||||
f"git {' '.join(args)}\n{r.stderr.strip()}"
|
||||
)
|
||||
return r
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
from datetime import datetime, timezone
|
||||
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
@@ -0,0 +1,415 @@
|
||||
"""SVN↔Git reconciliation for diverged mirrors.
|
||||
|
||||
Handles the case where both SVN and Git received commits independently
|
||||
while the sync daemon was not running.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from .mirror import Mirror
|
||||
from .svn import (
|
||||
get_latest_revision as svn_get_latest_revision,
|
||||
get_log_range as svn_get_log_range,
|
||||
)
|
||||
from .sync import parse_svn_path
|
||||
from .git_to_svn import (
|
||||
_apply_diff_to_wc,
|
||||
_run_svn as _gts_run_svn,
|
||||
_ensure_wc,
|
||||
_parse_commit_revision,
|
||||
_git_ref_to_svn_branch,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ReconcileError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
# ─── Git subprocess helper ────────────────────────────────
|
||||
|
||||
|
||||
def _git(git_dir: str, *args: str, input: str = None, timeout: int = 120,
|
||||
check: bool = True) -> str:
|
||||
try:
|
||||
r = subprocess.run(
|
||||
["git", "--git-dir", git_dir, *args],
|
||||
input=input, capture_output=True, text=True, timeout=timeout,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
raise ReconcileError("git not found on PATH")
|
||||
except subprocess.TimeoutExpired:
|
||||
raise ReconcileError(f"git command timed out: git {' '.join(args)}")
|
||||
if check and r.returncode != 0:
|
||||
raise ReconcileError(
|
||||
f"git command failed (exit {r.returncode}): "
|
||||
f"git {' '.join(args)}\n{r.stderr.strip()}"
|
||||
)
|
||||
return r.stdout.strip() if r.returncode == 0 else ""
|
||||
|
||||
|
||||
# ─── Log helpers for clean output ─────────────────────────
|
||||
|
||||
|
||||
def _log_bold(msg: str):
|
||||
print(f"\n=== {msg} ===")
|
||||
|
||||
|
||||
def _log_info(msg: str):
|
||||
print(f" {msg}")
|
||||
|
||||
|
||||
def _log_ok(msg: str):
|
||||
print(f" ✓ {msg}")
|
||||
|
||||
|
||||
def _log_warn(msg: str):
|
||||
print(f" ⚠ {msg}")
|
||||
|
||||
|
||||
def _log_error(msg: str):
|
||||
print(f" ✗ {msg}")
|
||||
|
||||
|
||||
# ─── Assessment ────────────────────────────────────────────
|
||||
|
||||
|
||||
def _get_log_summary(git_dir: str, commit_hash: str) -> Dict[str, str]:
|
||||
"""Return short author + date + message for a commit."""
|
||||
out = _git(git_dir, "log", "--format=%an <%ae>|%ai|%s", "-1", commit_hash,
|
||||
timeout=30)
|
||||
parts = out.split("|", 2)
|
||||
return {
|
||||
"hash": commit_hash,
|
||||
"abbrev": commit_hash[:8],
|
||||
"author": parts[0] if len(parts) > 0 else "?",
|
||||
"date": parts[1] if len(parts) > 1 else "?",
|
||||
"message": parts[2] if len(parts) > 2 else "?",
|
||||
}
|
||||
|
||||
|
||||
def _find_boundary_and_dev_commits(
|
||||
git_dir: str, mirror: Mirror, ref: str = "refs/heads/master",
|
||||
) -> Tuple[Optional[str], List[str]]:
|
||||
"""Walk *ref* backwards to find the last mapped commit (boundary).
|
||||
|
||||
Returns (boundary_hash, [developer_commit_hashes…]).
|
||||
Developer commits are returned oldest-first.
|
||||
"""
|
||||
out = _git(git_dir, "rev-list", "--topo-order", ref, timeout=120,
|
||||
check=False)
|
||||
if not out:
|
||||
return None, []
|
||||
|
||||
all_commits = out.splitlines()
|
||||
db = mirror.db
|
||||
|
||||
# First pass: find unmapped commits between HEAD and the first mapped ancestor
|
||||
unmapped: List[str] = []
|
||||
boundary = None
|
||||
|
||||
for ch in all_commits:
|
||||
if db.get_svn_revision(ch) is not None:
|
||||
boundary = ch
|
||||
break
|
||||
unmapped.append(ch)
|
||||
|
||||
if boundary is None:
|
||||
return None, []
|
||||
|
||||
unmapped.reverse()
|
||||
return boundary, unmapped
|
||||
|
||||
|
||||
def _collect_changed_files(
|
||||
git_dir: str, commits: List[str],
|
||||
) -> Dict[str, List[str]]:
|
||||
"""Return {file_path: [commit_hash…]} for all files changed in *commits*."""
|
||||
files: Dict[str, List[str]] = {}
|
||||
for ch in commits:
|
||||
out = _git(git_dir, "diff-tree", "--no-commit-id", "-r", "--name-only",
|
||||
ch, timeout=30, check=False)
|
||||
for f in out.splitlines():
|
||||
f = f.strip()
|
||||
if f:
|
||||
files.setdefault(f, []).append(ch)
|
||||
return files
|
||||
|
||||
|
||||
def assess(mirror: Mirror, ref: str = "refs/heads/master") -> Dict[str, Any]:
|
||||
"""Analyze divergence and return a structured report.
|
||||
|
||||
Only analyses *ref* (default: refs/heads/master → SVN trunk).
|
||||
"""
|
||||
git_dir = str(mirror.canonical_dir)
|
||||
db = mirror.db
|
||||
svn_url = mirror.config.svn.url
|
||||
|
||||
# ── 1. Git side ───────────────────────────────────────
|
||||
git_head = _git(git_dir, "rev-parse", "--verify", "--quiet", ref,
|
||||
timeout=30, check=False)
|
||||
if not git_head:
|
||||
raise ReconcileError(f"Ref {ref} not found in canonical repo")
|
||||
|
||||
boundary, dev_commits = _find_boundary_and_dev_commits(git_dir, mirror, ref)
|
||||
|
||||
dev_summaries: List[Dict[str, str]] = []
|
||||
dev_changed_files: Dict[str, List[str]] = {}
|
||||
if dev_commits:
|
||||
for ch in dev_commits:
|
||||
dev_summaries.append(_get_log_summary(git_dir, ch))
|
||||
dev_changed_files = _collect_changed_files(git_dir, dev_commits)
|
||||
|
||||
# ── 2. SVN side ───────────────────────────────────────
|
||||
last_svn_rev_str = db.get_state("last_svn_revision")
|
||||
last_svn_rev = int(last_svn_rev_str) if last_svn_rev_str else 0
|
||||
latest_svn_rev = svn_get_latest_revision(svn_url)
|
||||
|
||||
pending_svn: List[Dict[str, Any]] = []
|
||||
svn_changed_files: Dict[str, List[int]] = {}
|
||||
if last_svn_rev < latest_svn_rev:
|
||||
try:
|
||||
entries = svn_get_log_range(svn_url, last_svn_rev + 1, latest_svn_rev)
|
||||
except Exception:
|
||||
entries = []
|
||||
for entry in entries:
|
||||
pending_svn.append({
|
||||
"revision": entry.revision,
|
||||
"author": entry.author,
|
||||
"message": entry.message.strip() or "(no message)",
|
||||
})
|
||||
for change in entry.paths:
|
||||
_, rel = parse_svn_path(change.path, mirror.config.svn)
|
||||
if rel:
|
||||
svn_changed_files.setdefault(rel, []).append(entry.revision)
|
||||
|
||||
# ── 3. Cross-reference ────────────────────────────────
|
||||
dev_file_set = set(dev_changed_files.keys())
|
||||
svn_file_set = set(svn_changed_files.keys())
|
||||
|
||||
conflicting = sorted(dev_file_set & svn_file_set)
|
||||
dev_only = sorted(dev_file_set - svn_file_set)
|
||||
svn_only = sorted(svn_file_set - dev_file_set)
|
||||
|
||||
boundary_svn_rev = db.get_svn_revision(boundary) if boundary else None
|
||||
|
||||
return {
|
||||
"mirror_id": mirror.config.id,
|
||||
"ref": ref,
|
||||
"git_head": git_head,
|
||||
"git_head_abbrev": git_head[:8] if git_head else "?",
|
||||
"boundary": boundary,
|
||||
"boundary_abbrev": boundary[:8] if boundary else "?",
|
||||
"boundary_svn_revision": boundary_svn_rev,
|
||||
"developer_commits": dev_summaries,
|
||||
"developer_count": len(dev_summaries),
|
||||
"pending_svn_revisions": pending_svn,
|
||||
"svn_pending_count": len(pending_svn),
|
||||
"last_svn_revision": last_svn_rev,
|
||||
"latest_svn_revision": latest_svn_rev,
|
||||
"dev_only_files": dev_only,
|
||||
"svn_only_files": svn_only,
|
||||
"conflicting_files": conflicting,
|
||||
"has_conflicts": len(conflicting) > 0,
|
||||
"has_divergence": len(dev_summaries) > 0 or len(pending_svn) > 0,
|
||||
}
|
||||
|
||||
|
||||
def print_assessment(report: Dict[str, Any]):
|
||||
"""Pretty-print an assessment report to stdout."""
|
||||
_log_bold(f"Reconciliation Assessment for mirror \"{report['mirror_id']}\"")
|
||||
_log_info(f"Ref: {report['ref']}")
|
||||
_log_info(f"Git HEAD: {report['git_head_abbrev']}")
|
||||
_log_info(f"")
|
||||
|
||||
boundary = report['boundary']
|
||||
boundary_rev = report['boundary_svn_revision']
|
||||
if boundary:
|
||||
_log_info(f"Last synced commit: {report['boundary_abbrev']}")
|
||||
_log_info(f" → SVN revision: r{boundary_rev}")
|
||||
else:
|
||||
_log_warn("No synced commit found on this ref")
|
||||
_log_info("")
|
||||
|
||||
# SVN pending
|
||||
pending = report['pending_svn_revisions']
|
||||
if pending:
|
||||
_log_info(f"Pending SVN revisions: {report['svn_pending_count']} "
|
||||
f"(r{report['last_svn_revision'] + 1}–"
|
||||
f"r{report['latest_svn_revision']})")
|
||||
for svn in pending:
|
||||
msg = svn['message'][:72]
|
||||
_log_info(f" r{svn['revision']}: {svn['author']:20s} \"{msg}\"")
|
||||
else:
|
||||
_log_info("Pending SVN revisions: none")
|
||||
_log_info("")
|
||||
|
||||
# Git pending
|
||||
dev = report['developer_commits']
|
||||
if dev:
|
||||
_log_info(f"Pending Git commits: {report['developer_count']}")
|
||||
for gc in dev:
|
||||
msg = gc['message'][:72]
|
||||
_log_info(f" {gc['abbrev']}: {gc['author']:30s} \"{msg}\"")
|
||||
else:
|
||||
_log_info("Pending Git commits: none")
|
||||
_log_info("")
|
||||
|
||||
# File analysis
|
||||
_log_bold("File change analysis")
|
||||
if report['conflicting_files']:
|
||||
_log_warn(f"Changed on BOTH sides (CONFLICT): "
|
||||
f"{len(report['conflicting_files'])} file(s)")
|
||||
for f in report['conflicting_files']:
|
||||
_log_info(f" {f}")
|
||||
else:
|
||||
_log_ok("No conflicting files")
|
||||
_log_info("")
|
||||
|
||||
if report['dev_only_files']:
|
||||
_log_info(f"Changed only on Git side: "
|
||||
f"{len(report['dev_only_files'])} file(s)")
|
||||
for f in report['dev_only_files'][:20]:
|
||||
_log_info(f" {f}")
|
||||
if len(report['dev_only_files']) > 20:
|
||||
_log_info(f" … and {len(report['dev_only_files']) - 20} more")
|
||||
if report['svn_only_files']:
|
||||
_log_info(f"Changed only on SVN side: "
|
||||
f"{len(report['svn_only_files'])} file(s)")
|
||||
for f in report['svn_only_files'][:20]:
|
||||
_log_info(f" {f}")
|
||||
if len(report['svn_only_files']) > 20:
|
||||
_log_info(f" … and {len(report['svn_only_files']) - 20} more")
|
||||
_log_info("")
|
||||
|
||||
if not report['has_divergence']:
|
||||
_log_ok("Mirror is already in sync — no reconciliation needed")
|
||||
else:
|
||||
_log_info("Strategy: git-wins (Git content takes precedence on conflicts)")
|
||||
if report['conflicting_files']:
|
||||
_log_info(f" → {len(report['conflicting_files'])} conflicting "
|
||||
f"file(s) will use Git version")
|
||||
_log_info("")
|
||||
_log_info("Run without --dry-run to apply.")
|
||||
|
||||
|
||||
# ─── Reconciliation ────────────────────────────────────────
|
||||
|
||||
|
||||
def reconcile(mirror: Mirror, strategy: str = "git-wins",
|
||||
dry_run: bool = False) -> int:
|
||||
"""Reconcile a diverged SVN↔Git mirror.
|
||||
|
||||
Strategy options:
|
||||
- ``git-wins`` (default): Git HEAD content takes precedence on
|
||||
conflicting files. SVN-only changes are preserved.
|
||||
|
||||
Returns the SVN revision number of the reconciliation commit,
|
||||
or 0 if nothing was needed.
|
||||
"""
|
||||
git_dir = str(mirror.canonical_dir)
|
||||
svn_cfg = mirror.config.svn
|
||||
db = mirror.db
|
||||
ref = "refs/heads/master"
|
||||
svn_branch = svn_cfg.trunk
|
||||
|
||||
with mirror.sync_lock():
|
||||
# ── 1. Assess ─────────────────────────────────────
|
||||
report = assess(mirror, ref=ref)
|
||||
|
||||
if dry_run:
|
||||
print_assessment(report)
|
||||
return 0
|
||||
|
||||
if not report['has_divergence']:
|
||||
_log_ok("Mirror is already in sync")
|
||||
return 0
|
||||
|
||||
boundary = report['boundary']
|
||||
dev_commits_hashes = [c['hash'] for c in report['developer_commits']]
|
||||
pending_revs = [r['revision'] for r in report['pending_svn_revisions']]
|
||||
|
||||
# ── 2. Backup mapping DB ──────────────────────────
|
||||
backup_path = Path(str(db.path) + f".reconcile-backup-{_now_ts()}")
|
||||
shutil.copy2(db.path, backup_path)
|
||||
_log_info(f"Mapping DB backed up to {backup_path.name}")
|
||||
|
||||
# ── 3. Ensure SVN WC at HEAD ──────────────────────
|
||||
_log_info(f"Checking out SVN {svn_branch} @ HEAD …")
|
||||
try:
|
||||
wc = _ensure_wc(mirror, svn_branch)
|
||||
except Exception as e:
|
||||
raise ReconcileError(f"Failed to checkout SVN WC: {e}")
|
||||
|
||||
# ── 4. Apply each developer commit to the WC ──────
|
||||
conflicts_seen = []
|
||||
for ch in dev_commits_hashes:
|
||||
summary = _get_log_summary(git_dir, ch)
|
||||
_log_info(f"Applying commit {summary['abbrev']}: "
|
||||
f"{summary['message'][:60]}…")
|
||||
|
||||
if strategy == "git-wins":
|
||||
try:
|
||||
_apply_diff_to_wc(mirror, ch, svn_branch, wc)
|
||||
except Exception as e:
|
||||
_log_warn(f"Failed to apply {ch[:8]}: {e}")
|
||||
_log_warn(" Skipping — manual intervention may be needed")
|
||||
else:
|
||||
raise ReconcileError(f"Unknown strategy: {strategy}")
|
||||
|
||||
# ── 5. SVN commit ─────────────────────────────────
|
||||
message = (
|
||||
f"[reconcile] Merge Git changes since SVN r{report['boundary_svn_revision']}\n"
|
||||
f"\n"
|
||||
f"Reconciliation of {len(dev_commits_hashes)} Git commit(s) and "
|
||||
f"{len(pending_revs)} SVN revision(s)\n"
|
||||
f"\n"
|
||||
f"Git commits applied:\n"
|
||||
)
|
||||
for gc in report['developer_commits']:
|
||||
message += f" {gc['abbrev']} {gc['message'][:72]}\n"
|
||||
|
||||
_log_info("Committing reconciled WC to SVN …")
|
||||
try:
|
||||
svn_author = "reconcile"
|
||||
result = _gts_run_svn(
|
||||
["commit", "--file", "-", "--username", svn_author],
|
||||
input=message.encode("utf-8"),
|
||||
wc=wc, timeout=120,
|
||||
)
|
||||
new_rev = _parse_commit_revision(result)
|
||||
except Exception as e:
|
||||
raise ReconcileError(f"SVN commit failed: {e}")
|
||||
|
||||
_log_ok(f"Created SVN r{new_rev} (reconciliation commit)")
|
||||
|
||||
# ── 6. Record mapping ─────────────────────────────
|
||||
git_head = report['git_head']
|
||||
db.record_mapping(new_rev, git_head, svn_branch, ref, source="git")
|
||||
_log_ok(f"Mapping recorded: r{new_rev} ↔ {git_head[:8]} ({ref})")
|
||||
|
||||
# Leave last_svn_revision unchanged — sync_all will
|
||||
# process pre-reconciliation SVN revisions naturally
|
||||
# and skip r{new_rev} because it's already mapped.
|
||||
|
||||
# ── 7. Summary ────────────────────────────────────
|
||||
_log_bold("Reconciliation complete")
|
||||
_log_info(f" SVN revision: r{new_rev}")
|
||||
_log_info(f" Git HEAD: {git_head[:8]}")
|
||||
if report['conflicting_files']:
|
||||
_log_warn(f" Conflicts resolved via '{strategy}' strategy for "
|
||||
f"{len(report['conflicting_files'])} file(s)")
|
||||
_log_info("")
|
||||
_log_info("Run `sync` to bring the Git mirror in line with SVN.")
|
||||
|
||||
return new_rev
|
||||
|
||||
|
||||
def _now_ts() -> str:
|
||||
from datetime import datetime
|
||||
return datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
@@ -0,0 +1,173 @@
|
||||
"""SVN repository remote interface (via svn CLI)."""
|
||||
|
||||
import subprocess
|
||||
import xml.etree.ElementTree as ET
|
||||
from dataclasses import dataclass, field
|
||||
from typing import List, Optional
|
||||
|
||||
|
||||
class SVNError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass
|
||||
class SVNPathChange:
|
||||
action: str # 'A', 'M', 'D', 'R'
|
||||
kind: str # 'file', 'dir'
|
||||
path: str # absolute SVN path, e.g. /trunk/src/main.c
|
||||
copyfrom_path: Optional[str] = None
|
||||
copyfrom_rev: Optional[int] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class SVNLogEntry:
|
||||
revision: int
|
||||
author: str
|
||||
date: str
|
||||
message: str
|
||||
paths: List[SVNPathChange] = field(default_factory=list)
|
||||
|
||||
|
||||
def _run_svn(args: list, input: bytes = None, timeout: int = 120) -> subprocess.CompletedProcess:
|
||||
try:
|
||||
r = subprocess.run(
|
||||
["svn"] + args,
|
||||
input=input,
|
||||
capture_output=True,
|
||||
timeout=timeout,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
raise SVNError("svn CLI not found – is subversion installed?")
|
||||
except subprocess.TimeoutExpired:
|
||||
raise SVNError(f"svn command timed out: svn {' '.join(args)}")
|
||||
|
||||
if r.returncode != 0:
|
||||
msg = (r.stderr or r.stdout or b"").decode("utf-8", errors="replace").strip()
|
||||
raise SVNError(f"svn failed (exit {r.returncode}): {msg}")
|
||||
|
||||
return r
|
||||
|
||||
|
||||
def get_latest_revision(url: str) -> int:
|
||||
"""Return the HEAD revision number of an SVN repository."""
|
||||
r = _run_svn(["info", "--xml", url], timeout=60)
|
||||
root = ET.fromstring(r.stdout)
|
||||
entry = root.find(".//entry")
|
||||
if entry is None:
|
||||
raise SVNError("Could not find <entry> in svn info output")
|
||||
rev = entry.get("revision")
|
||||
if rev is None:
|
||||
raise SVNError("No revision attribute in svn info entry")
|
||||
return int(rev)
|
||||
|
||||
|
||||
def get_log(url: str, revision: int) -> SVNLogEntry:
|
||||
"""Fetch log entry with changed paths for a single SVN revision."""
|
||||
r = _run_svn(
|
||||
["log", "-r", str(revision), "--verbose", "--xml", url],
|
||||
timeout=120,
|
||||
)
|
||||
root = ET.fromstring(r.stdout)
|
||||
logentry = root.find("logentry")
|
||||
|
||||
if logentry is None:
|
||||
raise SVNError(f"Revision {revision} not found")
|
||||
|
||||
author_el = logentry.find("author")
|
||||
date_el = logentry.find("date")
|
||||
msg_el = logentry.find("msg")
|
||||
|
||||
author = author_el.text.strip() if author_el is not None and author_el.text else "unknown"
|
||||
date = date_el.text.strip() if date_el is not None and date_el.text else ""
|
||||
msg = msg_el.text.strip() if msg_el is not None and msg_el.text else ""
|
||||
|
||||
paths = []
|
||||
paths_el = logentry.find("paths")
|
||||
if paths_el is not None:
|
||||
for path_el in paths_el.findall("path"):
|
||||
action = path_el.get("action", "M")
|
||||
kind = path_el.get("kind", "file")
|
||||
text = (path_el.text or "").strip()
|
||||
cf_path = path_el.get("copyfrom-path")
|
||||
cf_rev_str = path_el.get("copyfrom-rev")
|
||||
|
||||
paths.append(SVNPathChange(
|
||||
action=action,
|
||||
kind=kind,
|
||||
path=text,
|
||||
copyfrom_path=cf_path,
|
||||
copyfrom_rev=int(cf_rev_str) if cf_rev_str else None,
|
||||
))
|
||||
|
||||
return SVNLogEntry(
|
||||
revision=revision,
|
||||
author=author,
|
||||
date=date,
|
||||
message=msg,
|
||||
paths=paths,
|
||||
)
|
||||
|
||||
|
||||
def get_log_range(url: str, start_rev: int, end_rev: int) -> List[SVNLogEntry]:
|
||||
"""Fetch log entries for a range of SVN revisions in one call.
|
||||
|
||||
Returns entries from *start_rev* to *end_rev* (inclusive), oldest first.
|
||||
"""
|
||||
r = _run_svn(
|
||||
["log", "-r", f"{start_rev}:{end_rev}", "--verbose", "--xml", url],
|
||||
timeout=300,
|
||||
)
|
||||
root = ET.fromstring(r.stdout)
|
||||
entries: List[SVNLogEntry] = []
|
||||
# svn log returns newest first; reverse to get oldest first
|
||||
for logentry in reversed(root.findall("logentry")):
|
||||
rev_str = logentry.get("revision")
|
||||
if rev_str is None:
|
||||
continue
|
||||
revision = int(rev_str)
|
||||
|
||||
author_el = logentry.find("author")
|
||||
date_el = logentry.find("date")
|
||||
msg_el = logentry.find("msg")
|
||||
|
||||
author = author_el.text.strip() if author_el is not None and author_el.text else "unknown"
|
||||
date = date_el.text.strip() if date_el is not None and date_el.text else ""
|
||||
msg = msg_el.text.strip() if msg_el is not None and msg_el.text else ""
|
||||
|
||||
paths = []
|
||||
paths_el = logentry.find("paths")
|
||||
if paths_el is not None:
|
||||
for path_el in paths_el.findall("path"):
|
||||
action = path_el.get("action", "M")
|
||||
kind = path_el.get("kind", "file")
|
||||
text = (path_el.text or "").strip()
|
||||
cf_path = path_el.get("copyfrom-path")
|
||||
cf_rev_str = path_el.get("copyfrom-rev")
|
||||
|
||||
paths.append(SVNPathChange(
|
||||
action=action,
|
||||
kind=kind,
|
||||
path=text,
|
||||
copyfrom_path=cf_path,
|
||||
copyfrom_rev=int(cf_rev_str) if cf_rev_str else None,
|
||||
))
|
||||
|
||||
entries.append(SVNLogEntry(
|
||||
revision=revision,
|
||||
author=author,
|
||||
date=date,
|
||||
message=msg,
|
||||
paths=paths,
|
||||
))
|
||||
|
||||
return entries
|
||||
|
||||
|
||||
def get_file(url: str, path: str, revision: int) -> bytes:
|
||||
"""Return the full content of a file at a given SVN revision.
|
||||
|
||||
`path` is the absolute SVN path, e.g. /trunk/src/main.c.
|
||||
"""
|
||||
full_url = url.rstrip("/") + path
|
||||
r = _run_svn(["cat", "-r", str(revision), full_url], timeout=60)
|
||||
return r.stdout
|
||||
@@ -0,0 +1,495 @@
|
||||
"""SVN→Git synchronization engine.
|
||||
|
||||
Builds Git commits from SVN revisions using `git ls-tree`, `git hash-object`,
|
||||
`git mktree`, and `git commit-tree` via subprocess. No pygit2 required.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
from .config import MirrorConfig
|
||||
from .mirror import Mirror
|
||||
from .gitea import push_to_gitea, GiteaError
|
||||
from .git_to_svn import sync_git_to_svn
|
||||
from .svn import (
|
||||
SVNPathChange,
|
||||
get_file as svn_get_file,
|
||||
get_latest_revision as svn_get_latest_revision,
|
||||
get_log as svn_get_log,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Well-known Git empty tree hash (4b825dc642cb6eb9a060e54bf899d153036d1e3c).
|
||||
# Exists in every Git repository implicitly.
|
||||
_EMPTY_TREE = "4b825dc642cb6eb9a060e54bf899d153036d1e3c"
|
||||
|
||||
|
||||
class SyncError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
# ─── SVN path parsing ────────────────────────────────────────
|
||||
|
||||
|
||||
def parse_svn_path(full_path: str, svn_cfg) -> Tuple[Optional[str], str]:
|
||||
"""Split an absolute SVN path into (branch, relative_path).
|
||||
|
||||
Standard layout examples:
|
||||
/trunk/src/main.c → ("trunk", "src/main.c")
|
||||
/branches/feat/x.c → ("branches/feat", "x.c")
|
||||
/tags/v1.0/README → ("tags/v1.0", "README")
|
||||
"""
|
||||
path = full_path.lstrip("/")
|
||||
|
||||
t = svn_cfg.trunk
|
||||
b = svn_cfg.branches
|
||||
tg = svn_cfg.tags
|
||||
|
||||
if svn_cfg.layout == "std":
|
||||
if path == t:
|
||||
return (t, "")
|
||||
if path.startswith(t + "/"):
|
||||
return (t, path[len(t) + 1:])
|
||||
|
||||
for prefix in (b, tg):
|
||||
if path.startswith(prefix + "/"):
|
||||
parts = path.split("/", 2)
|
||||
branch = f"{parts[0]}/{parts[1]}"
|
||||
rel = parts[2] if len(parts) > 2 else ""
|
||||
return (branch, rel)
|
||||
|
||||
return (None, path)
|
||||
|
||||
|
||||
def group_changes_by_branch(
|
||||
changes: List[SVNPathChange], svn_cfg,
|
||||
) -> Dict[str, List[Tuple]]:
|
||||
"""Group a list of SVN path changes by affected branch.
|
||||
|
||||
Returns {branch: [(action, kind, rel_path, full_svn_path, ...)]}
|
||||
"""
|
||||
groups: Dict[str, List[Tuple]] = {}
|
||||
for c in changes:
|
||||
branch, rel = parse_svn_path(c.path, svn_cfg)
|
||||
if branch is None:
|
||||
logger.warning("Cannot determine branch for path %s", c.path)
|
||||
continue
|
||||
groups.setdefault(branch, []).append((
|
||||
c.action, c.kind, rel, c.path,
|
||||
c.copyfrom_path, c.copyfrom_rev,
|
||||
))
|
||||
return groups
|
||||
|
||||
|
||||
# ─── Ref / author helpers ────────────────────────────────────
|
||||
|
||||
|
||||
def branch_to_git_ref(branch: str, svn_cfg) -> str:
|
||||
"""Map SVN branch name to Git ref.
|
||||
|
||||
trunk → refs/heads/master
|
||||
branches/X → refs/heads/X
|
||||
tags/X → refs/tags/X
|
||||
"""
|
||||
if branch == svn_cfg.trunk:
|
||||
return "refs/heads/master"
|
||||
if branch.startswith(svn_cfg.tags + "/") and len(branch) > len(svn_cfg.tags) + 1:
|
||||
return "refs/tags/" + branch[len(svn_cfg.tags) + 1:]
|
||||
if branch.startswith(svn_cfg.branches + "/") and len(branch) > len(svn_cfg.branches) + 1:
|
||||
return "refs/heads/" + branch[len(svn_cfg.branches) + 1:]
|
||||
return "refs/heads/" + branch
|
||||
|
||||
|
||||
def map_author(svn_username: str, authors: Dict[str, str]) -> Tuple[str, str]:
|
||||
"""Map SVN username to Git (name, email)."""
|
||||
mapped = authors.get(svn_username)
|
||||
if mapped:
|
||||
# Format: "John Doe <john@example.com>"
|
||||
if "<" in mapped and mapped.endswith(">"):
|
||||
name = mapped[:mapped.index("<")].strip()
|
||||
email = mapped[mapped.index("<") + 1:-1].strip()
|
||||
return (name, email)
|
||||
return (mapped, f"{svn_username}@local")
|
||||
return (svn_username, f"{svn_username}@local")
|
||||
|
||||
|
||||
# ─── Git tree building ────────────────────────────────────────
|
||||
|
||||
|
||||
def _git(git_dir: str, *args: str, input: str = None, timeout: int = 120,
|
||||
check: bool = True) -> str:
|
||||
"""Run a git command in the bare canonical repo and return stdout."""
|
||||
try:
|
||||
r = subprocess.run(
|
||||
["git", "--git-dir", git_dir, *args],
|
||||
input=input,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
raise SyncError("git not found on PATH")
|
||||
except subprocess.TimeoutExpired:
|
||||
raise SyncError(f"git command timed out: git {' '.join(args)}")
|
||||
|
||||
if check and r.returncode != 0:
|
||||
raise SyncError(
|
||||
f"git command failed (exit {r.returncode}): "
|
||||
f"git {' '.join(args)}\n{r.stderr.strip()}"
|
||||
)
|
||||
return r.stdout.strip() if r.returncode == 0 else ""
|
||||
|
||||
|
||||
def _git_bytes(git_dir: str, *args: str, input: bytes = None,
|
||||
timeout: int = 120) -> str:
|
||||
"""Like _git but accepts bytes input (for binary file content).
|
||||
|
||||
``text=True`` is omitted so subprocess passes raw bytes on stdin.
|
||||
"""
|
||||
try:
|
||||
r = subprocess.run(
|
||||
["git", "--git-dir", git_dir, *args],
|
||||
input=input,
|
||||
capture_output=True,
|
||||
timeout=timeout,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
raise SyncError("git not found on PATH")
|
||||
except subprocess.TimeoutExpired:
|
||||
raise SyncError(f"git command timed out: git {' '.join(args)}")
|
||||
|
||||
if r.returncode != 0:
|
||||
raise SyncError(
|
||||
f"git command failed (exit {r.returncode}): "
|
||||
f"git {' '.join(args)}\n{r.stderr.strip()}"
|
||||
)
|
||||
return r.stdout.strip().decode("utf-8", errors="replace")
|
||||
|
||||
|
||||
def _build_tree_from_changes(
|
||||
git_dir: str,
|
||||
parent_hash: Optional[str],
|
||||
changes: List[Tuple],
|
||||
svn_url: str,
|
||||
revision: int,
|
||||
) -> str:
|
||||
"""Build a new Git tree by applying SVN changes to the parent tree.
|
||||
|
||||
Strategy:
|
||||
1. Get flat file listing from parent tree via `git ls-tree -r`.
|
||||
2. Apply add/modify/delete operations to the in-memory dict.
|
||||
3. Write new tree via `git mktree`.
|
||||
|
||||
This avoids the complexity of nested TreeBuilders and handles
|
||||
directory deletions simply (all files under the path are removed).
|
||||
"""
|
||||
# ── 1. Read parent tree into flat dict ─────────────────────
|
||||
entries: Dict[str, Tuple[str, str]] = {} # path → (mode, oid)
|
||||
if parent_hash:
|
||||
out = _git(git_dir, "ls-tree", "-r", parent_hash, timeout=60)
|
||||
for line in out.splitlines():
|
||||
if not line:
|
||||
continue
|
||||
meta, path = line.split("\t", 1)
|
||||
mode, obj_type, oid = meta.split(None, 2)
|
||||
entries[path] = (mode, oid)
|
||||
|
||||
# ── 2. Apply changes ──────────────────────────────────────
|
||||
for action, kind, rel_path, full_path, cf_path, cf_rev in changes:
|
||||
if action in ("A", "M") and kind == "file":
|
||||
content = svn_get_file(svn_url, full_path, revision)
|
||||
blob_oid = _git_bytes(git_dir, "hash-object", "-w", "--stdin",
|
||||
input=content, timeout=60)
|
||||
entries[rel_path] = ("100644", blob_oid)
|
||||
|
||||
elif action == "D" and kind == "file":
|
||||
entries.pop(rel_path, None)
|
||||
|
||||
elif action == "D" and kind == "dir":
|
||||
prefix = rel_path.rstrip("/") + "/"
|
||||
to_del = [k for k in entries if k == rel_path or k.startswith(prefix)]
|
||||
for k in to_del:
|
||||
del entries[k]
|
||||
|
||||
elif action == "R" and kind == "file":
|
||||
entries.pop(rel_path, None)
|
||||
content = svn_get_file(svn_url, full_path, revision)
|
||||
blob_oid = _git_bytes(git_dir, "hash-object", "-w", "--stdin",
|
||||
input=content, timeout=60)
|
||||
entries[rel_path] = ("100644", blob_oid)
|
||||
|
||||
# ── 3. Write new tree via recursive mktree ─────────────────
|
||||
return _tree_from_flat_entries(git_dir, entries)
|
||||
|
||||
|
||||
def _tree_from_flat_entries(git_dir: str,
|
||||
entries: Dict[str, Tuple[str, str]]) -> str:
|
||||
"""Build a Git tree from a flat ``{path: (mode, oid)}`` dict.
|
||||
|
||||
``git mktree`` only accepts entries for a *single* level (no slashes
|
||||
in paths), so we group entries by their first path component and
|
||||
recurse into sub-trees.
|
||||
"""
|
||||
if not entries:
|
||||
return _EMPTY_TREE
|
||||
|
||||
lines: List[str] = []
|
||||
children: Dict[str, Dict[str, Tuple[str, str]]] = {}
|
||||
|
||||
for path, (mode, oid) in sorted(entries.items()):
|
||||
idx = path.find("/")
|
||||
if idx == -1:
|
||||
lines.append(f"{mode} blob {oid}\t{path}")
|
||||
else:
|
||||
head, tail = path[:idx], path[idx + 1:]
|
||||
children.setdefault(head, {})[tail] = (mode, oid)
|
||||
|
||||
for dir_name in sorted(children):
|
||||
subtree = _tree_from_flat_entries(git_dir, children[dir_name])
|
||||
if subtree != _EMPTY_TREE:
|
||||
lines.append(f"040000 tree {subtree}\t{dir_name}")
|
||||
|
||||
if not lines:
|
||||
return _EMPTY_TREE
|
||||
|
||||
input_str = "\n".join(lines)
|
||||
return _git(git_dir, "mktree", input=input_str, timeout=60)
|
||||
|
||||
|
||||
def _find_copy_source(cfg, db, changes) -> Optional[str]:
|
||||
"""If *changes* includes an ``svn copy``, return the source Git commit hash.
|
||||
|
||||
When SVN creates a branch or tag via ``svn copy src dst`` the change
|
||||
is recorded as ``(A, dir, …, copyfrom_path, copyfrom_rev)``. We
|
||||
look up the source commit in the mapping DB so the new branch starts
|
||||
with the correct tree.
|
||||
|
||||
Falls back to the latest mapping for the source branch when the exact
|
||||
``(cf_rev, src_branch)`` pair is not found (common when the copyfrom
|
||||
revision is a global revision that didn't touch the source branch).
|
||||
"""
|
||||
for c in changes:
|
||||
if len(c) < 6:
|
||||
continue
|
||||
action, kind, _rel, _full, cf_path, cf_rev = c[:6]
|
||||
if action == "A" and kind == "dir" and cf_path and cf_rev:
|
||||
src_branch, _ = parse_svn_path(cf_path, cfg.svn)
|
||||
if not src_branch:
|
||||
continue
|
||||
|
||||
# 1. Try exact match first
|
||||
src_hash = db.get_git_hash(cf_rev, src_branch)
|
||||
if src_hash:
|
||||
logger.debug(
|
||||
"Copy source for %s@%d → %s (%s)",
|
||||
cf_path, cf_rev, src_hash[:8], src_branch,
|
||||
)
|
||||
return src_hash
|
||||
|
||||
# 2. Fallback: latest mapping for the source branch <= cf_rev
|
||||
row = db.conn.execute(
|
||||
"SELECT git_commit_hash FROM svn_to_git "
|
||||
"WHERE svn_branch=? AND svn_revision<=? "
|
||||
"ORDER BY svn_revision DESC LIMIT 1",
|
||||
(src_branch, cf_rev),
|
||||
).fetchone()
|
||||
if row:
|
||||
logger.debug(
|
||||
"Copy source for %s@%d → %s (%s, fallback)",
|
||||
cf_path, cf_rev, row[0][:8], src_branch,
|
||||
)
|
||||
return row[0]
|
||||
|
||||
return None
|
||||
|
||||
|
||||
# ─── Per-revision sync ────────────────────────────────────────
|
||||
|
||||
|
||||
def sync_svn_revision(mirror: Mirror, revision: int) -> bool:
|
||||
"""Translate a single SVN revision into Git commit(s).
|
||||
|
||||
Returns True if at least one Git commit was created.
|
||||
"""
|
||||
cfg = mirror.config
|
||||
git_dir = str(mirror.canonical_dir)
|
||||
|
||||
# Fetch SVN metadata
|
||||
log_entry = svn_get_log(cfg.svn.url, revision)
|
||||
branch_changes = group_changes_by_branch(log_entry.paths, cfg.svn)
|
||||
|
||||
if not branch_changes:
|
||||
logger.debug("Rev %d: no branch changes to sync", revision)
|
||||
return False
|
||||
|
||||
any_commit = False
|
||||
|
||||
for branch, changes in branch_changes.items():
|
||||
git_ref = branch_to_git_ref(branch, cfg.svn)
|
||||
|
||||
if mirror.db.mapping_exists(revision, branch):
|
||||
logger.debug("Rev %d/%s already mapped – skipping", revision, branch)
|
||||
continue
|
||||
|
||||
# Find parent commit for this branch
|
||||
parent_hash_str = _git(git_dir, "rev-parse", "--verify", "--quiet",
|
||||
git_ref, timeout=30, check=False)
|
||||
parent_hash = parent_hash_str if parent_hash_str else None
|
||||
|
||||
# For a new branch/tag created via svn copy, use the source tree
|
||||
if parent_hash is None:
|
||||
parent_hash = _find_copy_source(cfg, mirror.db, changes)
|
||||
|
||||
# Build new tree
|
||||
new_tree = _build_tree_from_changes(
|
||||
git_dir, parent_hash, changes, cfg.svn.url, revision,
|
||||
)
|
||||
|
||||
# Map author
|
||||
author_name, author_email = map_author(log_entry.author, cfg.authors)
|
||||
author_line = f"{author_name} <{author_email}>"
|
||||
|
||||
# Create commit via commit-tree
|
||||
c_args = ["commit-tree", new_tree]
|
||||
if parent_hash:
|
||||
c_args += ["-p", parent_hash]
|
||||
|
||||
commit_env = {
|
||||
"GIT_AUTHOR_NAME": author_name,
|
||||
"GIT_AUTHOR_EMAIL": author_email,
|
||||
"GIT_COMMITTER_NAME": author_name,
|
||||
"GIT_COMMITTER_EMAIL": author_email,
|
||||
}
|
||||
|
||||
commit_hash = _git_env(git_dir, commit_env, *c_args,
|
||||
input=log_entry.message, timeout=120)
|
||||
|
||||
# Update ref
|
||||
_git(git_dir, "update-ref", git_ref, commit_hash, timeout=30)
|
||||
|
||||
# Record mapping
|
||||
mirror.db.record_mapping(revision, commit_hash, branch, git_ref, "svn")
|
||||
any_commit = True
|
||||
|
||||
logger.info(
|
||||
"Synced SVN r%d → %s (%s / %s)",
|
||||
revision, commit_hash[:8], branch, git_ref,
|
||||
)
|
||||
|
||||
return any_commit
|
||||
|
||||
|
||||
def _git_env(git_dir: str, env: Dict[str, str], *args: str,
|
||||
input: str = None, timeout: int = 120) -> str:
|
||||
"""Like _git but with additional environment variables (for commit authorship)."""
|
||||
full_env = {**os.environ, "GIT_DIR": git_dir, **env}
|
||||
try:
|
||||
r = subprocess.run(
|
||||
["git", *args],
|
||||
input=input,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
env=full_env,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
raise SyncError("git not found on PATH")
|
||||
except subprocess.TimeoutExpired:
|
||||
raise SyncError(f"git command timed out: git {' '.join(args)}")
|
||||
|
||||
if r.returncode != 0:
|
||||
raise SyncError(
|
||||
f"git command failed (exit {r.returncode}): "
|
||||
f"git {' '.join(args)}\n{r.stderr.strip()}"
|
||||
)
|
||||
return r.stdout.strip()
|
||||
|
||||
|
||||
# ─── Full sync cycle ──────────────────────────────────────────
|
||||
|
||||
|
||||
def sync_all(mirror: Mirror) -> int:
|
||||
"""Sync all pending SVN revisions for a single mirror (thread-safe).
|
||||
|
||||
Returns the number of revisions processed.
|
||||
"""
|
||||
with mirror.sync_lock():
|
||||
return _sync_all_impl(mirror)
|
||||
|
||||
|
||||
def _sync_all_impl(mirror: Mirror) -> int:
|
||||
state_rev = mirror.db.get_state("last_svn_revision")
|
||||
trunk_name = mirror.config.svn.trunk
|
||||
last_rev = int(state_rev) if state_rev else (mirror.db.get_last_svn_revision(trunk_name) or 0)
|
||||
latest_rev = svn_get_latest_revision(mirror.config.svn.url)
|
||||
|
||||
if latest_rev <= last_rev:
|
||||
logger.debug("Mirror %s is up to date (r%d)", mirror.config.id, last_rev)
|
||||
return 0
|
||||
|
||||
logger.info(
|
||||
"Syncing %s SVN r%d → r%d",
|
||||
mirror.config.id, last_rev + 1, latest_rev,
|
||||
)
|
||||
|
||||
count = 0
|
||||
for rev in range(last_rev + 1, latest_rev + 1):
|
||||
try:
|
||||
if sync_svn_revision(mirror, rev):
|
||||
count += 1
|
||||
except Exception:
|
||||
logger.exception("Failed to sync SVN r%d for %s", rev, mirror.config.id)
|
||||
raise
|
||||
|
||||
if count:
|
||||
mirror.db.set_state("last_svn_revision", str(latest_rev))
|
||||
logger.info("Synced %d revision(s) for %s", count, mirror.config.id)
|
||||
return count
|
||||
|
||||
|
||||
# ─── Continuous daemon loop ───────────────────────────────────
|
||||
|
||||
|
||||
def run_daemon_loop(mirrors: Dict[str, Mirror]) -> None:
|
||||
"""Run a continuous sync loop, checking each mirror on its own interval."""
|
||||
last_sync: Dict[str, float] = {mid: 0.0 for mid in mirrors}
|
||||
|
||||
logger.info("Starting sync daemon with %d mirror(s)", len(mirrors))
|
||||
try:
|
||||
while True:
|
||||
now = time.time()
|
||||
for mid, m in mirrors.items():
|
||||
interval = m.config.sync_interval
|
||||
if now - last_sync[mid] < interval:
|
||||
continue
|
||||
|
||||
if not m.base_dir.exists():
|
||||
logger.debug("Mirror %s not yet created – skipping", mid)
|
||||
continue
|
||||
|
||||
state = m._read_state()
|
||||
if not state.get("initial_import_done"):
|
||||
logger.debug("Mirror %s not yet imported – skipping", mid)
|
||||
continue
|
||||
|
||||
logger.debug("Checking mirror %s…", mid)
|
||||
try:
|
||||
svn_count = sync_all(m)
|
||||
if svn_count:
|
||||
push_to_gitea(m)
|
||||
git_count = sync_git_to_svn(m)
|
||||
if svn_count or git_count:
|
||||
last_sync[mid] = now
|
||||
except (GiteaError) as ge:
|
||||
logger.warning("Gitea push failed for %s: %s", mid, ge)
|
||||
except Exception:
|
||||
logger.exception("Sync cycle failed for %s", mid)
|
||||
|
||||
time.sleep(5)
|
||||
except KeyboardInterrupt:
|
||||
logger.info("Daemon stopped by user")
|
||||
@@ -0,0 +1,144 @@
|
||||
"""Gitea webhook receiver – HTTP server that triggers Git→SVN sync on push."""
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import logging
|
||||
from http.server import HTTPServer, BaseHTTPRequestHandler
|
||||
from typing import Dict, Optional
|
||||
|
||||
from .config import MirrorConfig
|
||||
from .git_to_svn import sync_git_to_svn
|
||||
from .gitea import fetch_from_gitea
|
||||
from .mirror import Mirror
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class _Handler(BaseHTTPRequestHandler):
|
||||
"""HTTP request handler for Gitea webhooks."""
|
||||
|
||||
# Set by the caller before starting the server
|
||||
mirrors: Dict[str, Mirror] = {}
|
||||
lookup: Dict[str, str] = {} # "owner/repo" → mirror_id
|
||||
secrets: Dict[str, Optional[str]] = {} # mirror_id → secret (or None)
|
||||
|
||||
def log_request(self, code="-", size="-"):
|
||||
logger.debug("Webhook: %s %s → %s", self.command, self.path, code)
|
||||
|
||||
def _send_json(self, code: int, body: dict):
|
||||
self.send_response(code)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.end_headers()
|
||||
self.wfile.write(json.dumps(body).encode())
|
||||
|
||||
def _verify_signature(self, body: bytes, secret: Optional[str]) -> bool:
|
||||
"""Verify X-Gitea-Signature HMAC-SHA256 if a secret is configured."""
|
||||
if secret is None:
|
||||
return True # no secret configured – trust all
|
||||
sig = self.headers.get("X-Gitea-Signature", "")
|
||||
expected = hmac.new(secret.encode(), body, hashlib.sha256).hexdigest()
|
||||
return hmac.compare_digest(sig, expected)
|
||||
|
||||
def do_GET(self):
|
||||
if self.path == "/health":
|
||||
self._send_json(200, {"status": "ok"})
|
||||
else:
|
||||
self._send_json(404, {"error": "not found"})
|
||||
|
||||
def do_POST(self):
|
||||
if self.path != "/webhook":
|
||||
self._send_json(404, {"error": "not found"})
|
||||
return
|
||||
|
||||
length = int(self.headers.get("Content-Length", 0))
|
||||
body = self.rfile.read(length)
|
||||
|
||||
# Determine which mirror this webhook is for
|
||||
payload = json.loads(body)
|
||||
owner = (
|
||||
payload.get("repository", {}).get("owner", {}).get("login")
|
||||
or payload.get("repository", {}).get("owner", {}).get("username")
|
||||
)
|
||||
repo = payload.get("repository", {}).get("name")
|
||||
if not owner or not repo:
|
||||
self._send_json(400, {"error": "missing repository owner/name"})
|
||||
return
|
||||
|
||||
key = f"{owner}/{repo}"
|
||||
mirror_id = self.lookup.get(key)
|
||||
if mirror_id is None:
|
||||
logger.warning("No mirror configured for %s", key)
|
||||
self._send_json(404, {"error": f"no mirror for {key}"})
|
||||
return
|
||||
|
||||
# Verify signature
|
||||
secret = self.secrets.get(mirror_id)
|
||||
if not self._verify_signature(body, secret):
|
||||
logger.warning("Invalid webhook signature for %s", mirror_id)
|
||||
self._send_json(403, {"error": "invalid signature"})
|
||||
return
|
||||
|
||||
ref = payload.get("ref", "")
|
||||
logger.info(
|
||||
"Webhook: %s pushed to %s (%s)",
|
||||
payload.get("pusher", {}).get("login", "?"), key, ref,
|
||||
)
|
||||
|
||||
mirror = self.mirrors.get(mirror_id)
|
||||
if mirror is None:
|
||||
self._send_json(500, {"error": f"mirror {mirror_id} not loaded"})
|
||||
return
|
||||
|
||||
try:
|
||||
fetch_from_gitea(mirror, ref=ref)
|
||||
count = sync_git_to_svn(mirror)
|
||||
self._send_json(200, {
|
||||
"status": "ok",
|
||||
"mirror": mirror_id,
|
||||
"git_to_svn_count": count,
|
||||
})
|
||||
except Exception as e:
|
||||
logger.exception("Webhook processing failed for %s", mirror_id)
|
||||
self._send_json(500, {"error": str(e)})
|
||||
|
||||
do_PUT = do_POST # tolerate PUT for convenience
|
||||
|
||||
|
||||
def _build_lookup(mirrors: Dict[str, Mirror]) -> Dict[str, str]:
|
||||
"""Build ``owner/repo → mirror_id`` lookup dict."""
|
||||
lookup: Dict[str, str] = {}
|
||||
for mid, m in mirrors.items():
|
||||
cfg = m.config.gitea
|
||||
key = f"{cfg.owner}/{cfg.repo}"
|
||||
lookup[key] = mid
|
||||
return lookup
|
||||
|
||||
|
||||
def build_handler_class(mirrors: Dict[str, Mirror]) -> type:
|
||||
"""Return a handler class pre-configured with mirror references."""
|
||||
lookup = _build_lookup(mirrors)
|
||||
secrets = {
|
||||
mid: m.config.gitea.webhook_secret
|
||||
for mid, m in mirrors.items()
|
||||
}
|
||||
return type("ConfiguredHandler", (_Handler,), {
|
||||
"mirrors": mirrors,
|
||||
"lookup": lookup,
|
||||
"secrets": secrets,
|
||||
})
|
||||
|
||||
|
||||
def run_webhook_server(mirrors: Dict[str, Mirror], host: str = "0.0.0.0",
|
||||
port: int = 8080):
|
||||
"""Start the webhook HTTP server (blocks until KeyboardInterrupt)."""
|
||||
handler_cls = build_handler_class(mirrors)
|
||||
server = HTTPServer((host, port), handler_cls)
|
||||
logger.info("Webhook server listening on %s:%d", host, port)
|
||||
logger.info(" POST /webhook – Gitea push events")
|
||||
logger.info(" GET /health – health check")
|
||||
try:
|
||||
server.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
logger.info("Webhook server stopped")
|
||||
server.server_close()
|
||||
Reference in New Issue
Block a user