uberwald
45fb8c40d0
Server-side sidecar daemon. Uses own SQLite mapping table (not git svn) so Git commit hashes never change. Pure Python, zero framework deps. Includes: - SVN→Git sync engine with nested-directory tree reconstruction - Git→SVN push with author mapping and working-copy management - Branch/tag sync (svn copy → git branch/tag and reverse) - Gitea integration (on-disk push/fetch, webhook receiver) - Reconciliation tool for divergence recovery (assess + auto-fix) - Full end-to-end regression test (bash, 34 checks) - systemd unit and deployment docs Only external dependency: PyYAML (config parsing).
416 lines
15 KiB
Python
416 lines
15 KiB
Python
"""SVN↔Git reconciliation for diverged mirrors.
|
||
|
||
Handles the case where both SVN and Git received commits independently
|
||
while the sync daemon was not running.
|
||
"""
|
||
|
||
import logging
|
||
import shutil
|
||
import subprocess
|
||
from pathlib import Path
|
||
from typing import Any, Dict, List, Optional, Tuple
|
||
|
||
from .mirror import Mirror
|
||
from .svn import (
|
||
get_latest_revision as svn_get_latest_revision,
|
||
get_log_range as svn_get_log_range,
|
||
)
|
||
from .sync import parse_svn_path
|
||
from .git_to_svn import (
|
||
_apply_diff_to_wc,
|
||
_run_svn as _gts_run_svn,
|
||
_ensure_wc,
|
||
_parse_commit_revision,
|
||
_git_ref_to_svn_branch,
|
||
)
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
class ReconcileError(Exception):
|
||
pass
|
||
|
||
|
||
# ─── Git subprocess helper ────────────────────────────────
|
||
|
||
|
||
def _git(git_dir: str, *args: str, input: str = None, timeout: int = 120,
|
||
check: bool = True) -> str:
|
||
try:
|
||
r = subprocess.run(
|
||
["git", "--git-dir", git_dir, *args],
|
||
input=input, capture_output=True, text=True, timeout=timeout,
|
||
)
|
||
except FileNotFoundError:
|
||
raise ReconcileError("git not found on PATH")
|
||
except subprocess.TimeoutExpired:
|
||
raise ReconcileError(f"git command timed out: git {' '.join(args)}")
|
||
if check and r.returncode != 0:
|
||
raise ReconcileError(
|
||
f"git command failed (exit {r.returncode}): "
|
||
f"git {' '.join(args)}\n{r.stderr.strip()}"
|
||
)
|
||
return r.stdout.strip() if r.returncode == 0 else ""
|
||
|
||
|
||
# ─── Log helpers for clean output ─────────────────────────
|
||
|
||
|
||
def _log_bold(msg: str):
|
||
print(f"\n=== {msg} ===")
|
||
|
||
|
||
def _log_info(msg: str):
|
||
print(f" {msg}")
|
||
|
||
|
||
def _log_ok(msg: str):
|
||
print(f" ✓ {msg}")
|
||
|
||
|
||
def _log_warn(msg: str):
|
||
print(f" ⚠ {msg}")
|
||
|
||
|
||
def _log_error(msg: str):
|
||
print(f" ✗ {msg}")
|
||
|
||
|
||
# ─── Assessment ────────────────────────────────────────────
|
||
|
||
|
||
def _get_log_summary(git_dir: str, commit_hash: str) -> Dict[str, str]:
|
||
"""Return short author + date + message for a commit."""
|
||
out = _git(git_dir, "log", "--format=%an <%ae>|%ai|%s", "-1", commit_hash,
|
||
timeout=30)
|
||
parts = out.split("|", 2)
|
||
return {
|
||
"hash": commit_hash,
|
||
"abbrev": commit_hash[:8],
|
||
"author": parts[0] if len(parts) > 0 else "?",
|
||
"date": parts[1] if len(parts) > 1 else "?",
|
||
"message": parts[2] if len(parts) > 2 else "?",
|
||
}
|
||
|
||
|
||
def _find_boundary_and_dev_commits(
|
||
git_dir: str, mirror: Mirror, ref: str = "refs/heads/master",
|
||
) -> Tuple[Optional[str], List[str]]:
|
||
"""Walk *ref* backwards to find the last mapped commit (boundary).
|
||
|
||
Returns (boundary_hash, [developer_commit_hashes…]).
|
||
Developer commits are returned oldest-first.
|
||
"""
|
||
out = _git(git_dir, "rev-list", "--topo-order", ref, timeout=120,
|
||
check=False)
|
||
if not out:
|
||
return None, []
|
||
|
||
all_commits = out.splitlines()
|
||
db = mirror.db
|
||
|
||
# First pass: find unmapped commits between HEAD and the first mapped ancestor
|
||
unmapped: List[str] = []
|
||
boundary = None
|
||
|
||
for ch in all_commits:
|
||
if db.get_svn_revision(ch) is not None:
|
||
boundary = ch
|
||
break
|
||
unmapped.append(ch)
|
||
|
||
if boundary is None:
|
||
return None, []
|
||
|
||
unmapped.reverse()
|
||
return boundary, unmapped
|
||
|
||
|
||
def _collect_changed_files(
|
||
git_dir: str, commits: List[str],
|
||
) -> Dict[str, List[str]]:
|
||
"""Return {file_path: [commit_hash…]} for all files changed in *commits*."""
|
||
files: Dict[str, List[str]] = {}
|
||
for ch in commits:
|
||
out = _git(git_dir, "diff-tree", "--no-commit-id", "-r", "--name-only",
|
||
ch, timeout=30, check=False)
|
||
for f in out.splitlines():
|
||
f = f.strip()
|
||
if f:
|
||
files.setdefault(f, []).append(ch)
|
||
return files
|
||
|
||
|
||
def assess(mirror: Mirror, ref: str = "refs/heads/master") -> Dict[str, Any]:
|
||
"""Analyze divergence and return a structured report.
|
||
|
||
Only analyses *ref* (default: refs/heads/master → SVN trunk).
|
||
"""
|
||
git_dir = str(mirror.canonical_dir)
|
||
db = mirror.db
|
||
svn_url = mirror.config.svn.url
|
||
|
||
# ── 1. Git side ───────────────────────────────────────
|
||
git_head = _git(git_dir, "rev-parse", "--verify", "--quiet", ref,
|
||
timeout=30, check=False)
|
||
if not git_head:
|
||
raise ReconcileError(f"Ref {ref} not found in canonical repo")
|
||
|
||
boundary, dev_commits = _find_boundary_and_dev_commits(git_dir, mirror, ref)
|
||
|
||
dev_summaries: List[Dict[str, str]] = []
|
||
dev_changed_files: Dict[str, List[str]] = {}
|
||
if dev_commits:
|
||
for ch in dev_commits:
|
||
dev_summaries.append(_get_log_summary(git_dir, ch))
|
||
dev_changed_files = _collect_changed_files(git_dir, dev_commits)
|
||
|
||
# ── 2. SVN side ───────────────────────────────────────
|
||
last_svn_rev_str = db.get_state("last_svn_revision")
|
||
last_svn_rev = int(last_svn_rev_str) if last_svn_rev_str else 0
|
||
latest_svn_rev = svn_get_latest_revision(svn_url)
|
||
|
||
pending_svn: List[Dict[str, Any]] = []
|
||
svn_changed_files: Dict[str, List[int]] = {}
|
||
if last_svn_rev < latest_svn_rev:
|
||
try:
|
||
entries = svn_get_log_range(svn_url, last_svn_rev + 1, latest_svn_rev)
|
||
except Exception:
|
||
entries = []
|
||
for entry in entries:
|
||
pending_svn.append({
|
||
"revision": entry.revision,
|
||
"author": entry.author,
|
||
"message": entry.message.strip() or "(no message)",
|
||
})
|
||
for change in entry.paths:
|
||
_, rel = parse_svn_path(change.path, mirror.config.svn)
|
||
if rel:
|
||
svn_changed_files.setdefault(rel, []).append(entry.revision)
|
||
|
||
# ── 3. Cross-reference ────────────────────────────────
|
||
dev_file_set = set(dev_changed_files.keys())
|
||
svn_file_set = set(svn_changed_files.keys())
|
||
|
||
conflicting = sorted(dev_file_set & svn_file_set)
|
||
dev_only = sorted(dev_file_set - svn_file_set)
|
||
svn_only = sorted(svn_file_set - dev_file_set)
|
||
|
||
boundary_svn_rev = db.get_svn_revision(boundary) if boundary else None
|
||
|
||
return {
|
||
"mirror_id": mirror.config.id,
|
||
"ref": ref,
|
||
"git_head": git_head,
|
||
"git_head_abbrev": git_head[:8] if git_head else "?",
|
||
"boundary": boundary,
|
||
"boundary_abbrev": boundary[:8] if boundary else "?",
|
||
"boundary_svn_revision": boundary_svn_rev,
|
||
"developer_commits": dev_summaries,
|
||
"developer_count": len(dev_summaries),
|
||
"pending_svn_revisions": pending_svn,
|
||
"svn_pending_count": len(pending_svn),
|
||
"last_svn_revision": last_svn_rev,
|
||
"latest_svn_revision": latest_svn_rev,
|
||
"dev_only_files": dev_only,
|
||
"svn_only_files": svn_only,
|
||
"conflicting_files": conflicting,
|
||
"has_conflicts": len(conflicting) > 0,
|
||
"has_divergence": len(dev_summaries) > 0 or len(pending_svn) > 0,
|
||
}
|
||
|
||
|
||
def print_assessment(report: Dict[str, Any]):
|
||
"""Pretty-print an assessment report to stdout."""
|
||
_log_bold(f"Reconciliation Assessment for mirror \"{report['mirror_id']}\"")
|
||
_log_info(f"Ref: {report['ref']}")
|
||
_log_info(f"Git HEAD: {report['git_head_abbrev']}")
|
||
_log_info(f"")
|
||
|
||
boundary = report['boundary']
|
||
boundary_rev = report['boundary_svn_revision']
|
||
if boundary:
|
||
_log_info(f"Last synced commit: {report['boundary_abbrev']}")
|
||
_log_info(f" → SVN revision: r{boundary_rev}")
|
||
else:
|
||
_log_warn("No synced commit found on this ref")
|
||
_log_info("")
|
||
|
||
# SVN pending
|
||
pending = report['pending_svn_revisions']
|
||
if pending:
|
||
_log_info(f"Pending SVN revisions: {report['svn_pending_count']} "
|
||
f"(r{report['last_svn_revision'] + 1}–"
|
||
f"r{report['latest_svn_revision']})")
|
||
for svn in pending:
|
||
msg = svn['message'][:72]
|
||
_log_info(f" r{svn['revision']}: {svn['author']:20s} \"{msg}\"")
|
||
else:
|
||
_log_info("Pending SVN revisions: none")
|
||
_log_info("")
|
||
|
||
# Git pending
|
||
dev = report['developer_commits']
|
||
if dev:
|
||
_log_info(f"Pending Git commits: {report['developer_count']}")
|
||
for gc in dev:
|
||
msg = gc['message'][:72]
|
||
_log_info(f" {gc['abbrev']}: {gc['author']:30s} \"{msg}\"")
|
||
else:
|
||
_log_info("Pending Git commits: none")
|
||
_log_info("")
|
||
|
||
# File analysis
|
||
_log_bold("File change analysis")
|
||
if report['conflicting_files']:
|
||
_log_warn(f"Changed on BOTH sides (CONFLICT): "
|
||
f"{len(report['conflicting_files'])} file(s)")
|
||
for f in report['conflicting_files']:
|
||
_log_info(f" {f}")
|
||
else:
|
||
_log_ok("No conflicting files")
|
||
_log_info("")
|
||
|
||
if report['dev_only_files']:
|
||
_log_info(f"Changed only on Git side: "
|
||
f"{len(report['dev_only_files'])} file(s)")
|
||
for f in report['dev_only_files'][:20]:
|
||
_log_info(f" {f}")
|
||
if len(report['dev_only_files']) > 20:
|
||
_log_info(f" … and {len(report['dev_only_files']) - 20} more")
|
||
if report['svn_only_files']:
|
||
_log_info(f"Changed only on SVN side: "
|
||
f"{len(report['svn_only_files'])} file(s)")
|
||
for f in report['svn_only_files'][:20]:
|
||
_log_info(f" {f}")
|
||
if len(report['svn_only_files']) > 20:
|
||
_log_info(f" … and {len(report['svn_only_files']) - 20} more")
|
||
_log_info("")
|
||
|
||
if not report['has_divergence']:
|
||
_log_ok("Mirror is already in sync — no reconciliation needed")
|
||
else:
|
||
_log_info("Strategy: git-wins (Git content takes precedence on conflicts)")
|
||
if report['conflicting_files']:
|
||
_log_info(f" → {len(report['conflicting_files'])} conflicting "
|
||
f"file(s) will use Git version")
|
||
_log_info("")
|
||
_log_info("Run without --dry-run to apply.")
|
||
|
||
|
||
# ─── Reconciliation ────────────────────────────────────────
|
||
|
||
|
||
def reconcile(mirror: Mirror, strategy: str = "git-wins",
|
||
dry_run: bool = False) -> int:
|
||
"""Reconcile a diverged SVN↔Git mirror.
|
||
|
||
Strategy options:
|
||
- ``git-wins`` (default): Git HEAD content takes precedence on
|
||
conflicting files. SVN-only changes are preserved.
|
||
|
||
Returns the SVN revision number of the reconciliation commit,
|
||
or 0 if nothing was needed.
|
||
"""
|
||
git_dir = str(mirror.canonical_dir)
|
||
svn_cfg = mirror.config.svn
|
||
db = mirror.db
|
||
ref = "refs/heads/master"
|
||
svn_branch = svn_cfg.trunk
|
||
|
||
with mirror.sync_lock():
|
||
# ── 1. Assess ─────────────────────────────────────
|
||
report = assess(mirror, ref=ref)
|
||
|
||
if dry_run:
|
||
print_assessment(report)
|
||
return 0
|
||
|
||
if not report['has_divergence']:
|
||
_log_ok("Mirror is already in sync")
|
||
return 0
|
||
|
||
boundary = report['boundary']
|
||
dev_commits_hashes = [c['hash'] for c in report['developer_commits']]
|
||
pending_revs = [r['revision'] for r in report['pending_svn_revisions']]
|
||
|
||
# ── 2. Backup mapping DB ──────────────────────────
|
||
backup_path = Path(str(db.path) + f".reconcile-backup-{_now_ts()}")
|
||
shutil.copy2(db.path, backup_path)
|
||
_log_info(f"Mapping DB backed up to {backup_path.name}")
|
||
|
||
# ── 3. Ensure SVN WC at HEAD ──────────────────────
|
||
_log_info(f"Checking out SVN {svn_branch} @ HEAD …")
|
||
try:
|
||
wc = _ensure_wc(mirror, svn_branch)
|
||
except Exception as e:
|
||
raise ReconcileError(f"Failed to checkout SVN WC: {e}")
|
||
|
||
# ── 4. Apply each developer commit to the WC ──────
|
||
conflicts_seen = []
|
||
for ch in dev_commits_hashes:
|
||
summary = _get_log_summary(git_dir, ch)
|
||
_log_info(f"Applying commit {summary['abbrev']}: "
|
||
f"{summary['message'][:60]}…")
|
||
|
||
if strategy == "git-wins":
|
||
try:
|
||
_apply_diff_to_wc(mirror, ch, svn_branch, wc)
|
||
except Exception as e:
|
||
_log_warn(f"Failed to apply {ch[:8]}: {e}")
|
||
_log_warn(" Skipping — manual intervention may be needed")
|
||
else:
|
||
raise ReconcileError(f"Unknown strategy: {strategy}")
|
||
|
||
# ── 5. SVN commit ─────────────────────────────────
|
||
message = (
|
||
f"[reconcile] Merge Git changes since SVN r{report['boundary_svn_revision']}\n"
|
||
f"\n"
|
||
f"Reconciliation of {len(dev_commits_hashes)} Git commit(s) and "
|
||
f"{len(pending_revs)} SVN revision(s)\n"
|
||
f"\n"
|
||
f"Git commits applied:\n"
|
||
)
|
||
for gc in report['developer_commits']:
|
||
message += f" {gc['abbrev']} {gc['message'][:72]}\n"
|
||
|
||
_log_info("Committing reconciled WC to SVN …")
|
||
try:
|
||
svn_author = "reconcile"
|
||
result = _gts_run_svn(
|
||
["commit", "--file", "-", "--username", svn_author],
|
||
input=message.encode("utf-8"),
|
||
wc=wc, timeout=120,
|
||
)
|
||
new_rev = _parse_commit_revision(result)
|
||
except Exception as e:
|
||
raise ReconcileError(f"SVN commit failed: {e}")
|
||
|
||
_log_ok(f"Created SVN r{new_rev} (reconciliation commit)")
|
||
|
||
# ── 6. Record mapping ─────────────────────────────
|
||
git_head = report['git_head']
|
||
db.record_mapping(new_rev, git_head, svn_branch, ref, source="git")
|
||
_log_ok(f"Mapping recorded: r{new_rev} ↔ {git_head[:8]} ({ref})")
|
||
|
||
# Leave last_svn_revision unchanged — sync_all will
|
||
# process pre-reconciliation SVN revisions naturally
|
||
# and skip r{new_rev} because it's already mapped.
|
||
|
||
# ── 7. Summary ────────────────────────────────────
|
||
_log_bold("Reconciliation complete")
|
||
_log_info(f" SVN revision: r{new_rev}")
|
||
_log_info(f" Git HEAD: {git_head[:8]}")
|
||
if report['conflicting_files']:
|
||
_log_warn(f" Conflicts resolved via '{strategy}' strategy for "
|
||
f"{len(report['conflicting_files'])} file(s)")
|
||
_log_info("")
|
||
_log_info("Run `sync` to bring the Git mirror in line with SVN.")
|
||
|
||
return new_rev
|
||
|
||
|
||
def _now_ts() -> str:
|
||
from datetime import datetime
|
||
return datetime.now().strftime("%Y%m%d_%H%M%S")
|