feat: initial import — SVN↔Git bi-directional mirror for Gitea
Server-side sidecar daemon. Uses own SQLite mapping table (not git svn) so Git commit hashes never change. Pure Python, zero framework deps. Includes: - SVN→Git sync engine with nested-directory tree reconstruction - Git→SVN push with author mapping and working-copy management - Branch/tag sync (svn copy → git branch/tag and reverse) - Gitea integration (on-disk push/fetch, webhook receiver) - Reconciliation tool for divergence recovery (assess + auto-fix) - Full end-to-end regression test (bash, 34 checks) - systemd unit and deployment docs Only external dependency: PyYAML (config parsing).
This commit is contained in:
@@ -0,0 +1,415 @@
|
||||
"""SVN↔Git reconciliation for diverged mirrors.
|
||||
|
||||
Handles the case where both SVN and Git received commits independently
|
||||
while the sync daemon was not running.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from .mirror import Mirror
|
||||
from .svn import (
|
||||
get_latest_revision as svn_get_latest_revision,
|
||||
get_log_range as svn_get_log_range,
|
||||
)
|
||||
from .sync import parse_svn_path
|
||||
from .git_to_svn import (
|
||||
_apply_diff_to_wc,
|
||||
_run_svn as _gts_run_svn,
|
||||
_ensure_wc,
|
||||
_parse_commit_revision,
|
||||
_git_ref_to_svn_branch,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ReconcileError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
# ─── Git subprocess helper ────────────────────────────────
|
||||
|
||||
|
||||
def _git(git_dir: str, *args: str, input: str = None, timeout: int = 120,
|
||||
check: bool = True) -> str:
|
||||
try:
|
||||
r = subprocess.run(
|
||||
["git", "--git-dir", git_dir, *args],
|
||||
input=input, capture_output=True, text=True, timeout=timeout,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
raise ReconcileError("git not found on PATH")
|
||||
except subprocess.TimeoutExpired:
|
||||
raise ReconcileError(f"git command timed out: git {' '.join(args)}")
|
||||
if check and r.returncode != 0:
|
||||
raise ReconcileError(
|
||||
f"git command failed (exit {r.returncode}): "
|
||||
f"git {' '.join(args)}\n{r.stderr.strip()}"
|
||||
)
|
||||
return r.stdout.strip() if r.returncode == 0 else ""
|
||||
|
||||
|
||||
# ─── Log helpers for clean output ─────────────────────────
|
||||
|
||||
|
||||
def _log_bold(msg: str):
|
||||
print(f"\n=== {msg} ===")
|
||||
|
||||
|
||||
def _log_info(msg: str):
|
||||
print(f" {msg}")
|
||||
|
||||
|
||||
def _log_ok(msg: str):
|
||||
print(f" ✓ {msg}")
|
||||
|
||||
|
||||
def _log_warn(msg: str):
|
||||
print(f" ⚠ {msg}")
|
||||
|
||||
|
||||
def _log_error(msg: str):
|
||||
print(f" ✗ {msg}")
|
||||
|
||||
|
||||
# ─── Assessment ────────────────────────────────────────────
|
||||
|
||||
|
||||
def _get_log_summary(git_dir: str, commit_hash: str) -> Dict[str, str]:
|
||||
"""Return short author + date + message for a commit."""
|
||||
out = _git(git_dir, "log", "--format=%an <%ae>|%ai|%s", "-1", commit_hash,
|
||||
timeout=30)
|
||||
parts = out.split("|", 2)
|
||||
return {
|
||||
"hash": commit_hash,
|
||||
"abbrev": commit_hash[:8],
|
||||
"author": parts[0] if len(parts) > 0 else "?",
|
||||
"date": parts[1] if len(parts) > 1 else "?",
|
||||
"message": parts[2] if len(parts) > 2 else "?",
|
||||
}
|
||||
|
||||
|
||||
def _find_boundary_and_dev_commits(
|
||||
git_dir: str, mirror: Mirror, ref: str = "refs/heads/master",
|
||||
) -> Tuple[Optional[str], List[str]]:
|
||||
"""Walk *ref* backwards to find the last mapped commit (boundary).
|
||||
|
||||
Returns (boundary_hash, [developer_commit_hashes…]).
|
||||
Developer commits are returned oldest-first.
|
||||
"""
|
||||
out = _git(git_dir, "rev-list", "--topo-order", ref, timeout=120,
|
||||
check=False)
|
||||
if not out:
|
||||
return None, []
|
||||
|
||||
all_commits = out.splitlines()
|
||||
db = mirror.db
|
||||
|
||||
# First pass: find unmapped commits between HEAD and the first mapped ancestor
|
||||
unmapped: List[str] = []
|
||||
boundary = None
|
||||
|
||||
for ch in all_commits:
|
||||
if db.get_svn_revision(ch) is not None:
|
||||
boundary = ch
|
||||
break
|
||||
unmapped.append(ch)
|
||||
|
||||
if boundary is None:
|
||||
return None, []
|
||||
|
||||
unmapped.reverse()
|
||||
return boundary, unmapped
|
||||
|
||||
|
||||
def _collect_changed_files(
|
||||
git_dir: str, commits: List[str],
|
||||
) -> Dict[str, List[str]]:
|
||||
"""Return {file_path: [commit_hash…]} for all files changed in *commits*."""
|
||||
files: Dict[str, List[str]] = {}
|
||||
for ch in commits:
|
||||
out = _git(git_dir, "diff-tree", "--no-commit-id", "-r", "--name-only",
|
||||
ch, timeout=30, check=False)
|
||||
for f in out.splitlines():
|
||||
f = f.strip()
|
||||
if f:
|
||||
files.setdefault(f, []).append(ch)
|
||||
return files
|
||||
|
||||
|
||||
def assess(mirror: Mirror, ref: str = "refs/heads/master") -> Dict[str, Any]:
|
||||
"""Analyze divergence and return a structured report.
|
||||
|
||||
Only analyses *ref* (default: refs/heads/master → SVN trunk).
|
||||
"""
|
||||
git_dir = str(mirror.canonical_dir)
|
||||
db = mirror.db
|
||||
svn_url = mirror.config.svn.url
|
||||
|
||||
# ── 1. Git side ───────────────────────────────────────
|
||||
git_head = _git(git_dir, "rev-parse", "--verify", "--quiet", ref,
|
||||
timeout=30, check=False)
|
||||
if not git_head:
|
||||
raise ReconcileError(f"Ref {ref} not found in canonical repo")
|
||||
|
||||
boundary, dev_commits = _find_boundary_and_dev_commits(git_dir, mirror, ref)
|
||||
|
||||
dev_summaries: List[Dict[str, str]] = []
|
||||
dev_changed_files: Dict[str, List[str]] = {}
|
||||
if dev_commits:
|
||||
for ch in dev_commits:
|
||||
dev_summaries.append(_get_log_summary(git_dir, ch))
|
||||
dev_changed_files = _collect_changed_files(git_dir, dev_commits)
|
||||
|
||||
# ── 2. SVN side ───────────────────────────────────────
|
||||
last_svn_rev_str = db.get_state("last_svn_revision")
|
||||
last_svn_rev = int(last_svn_rev_str) if last_svn_rev_str else 0
|
||||
latest_svn_rev = svn_get_latest_revision(svn_url)
|
||||
|
||||
pending_svn: List[Dict[str, Any]] = []
|
||||
svn_changed_files: Dict[str, List[int]] = {}
|
||||
if last_svn_rev < latest_svn_rev:
|
||||
try:
|
||||
entries = svn_get_log_range(svn_url, last_svn_rev + 1, latest_svn_rev)
|
||||
except Exception:
|
||||
entries = []
|
||||
for entry in entries:
|
||||
pending_svn.append({
|
||||
"revision": entry.revision,
|
||||
"author": entry.author,
|
||||
"message": entry.message.strip() or "(no message)",
|
||||
})
|
||||
for change in entry.paths:
|
||||
_, rel = parse_svn_path(change.path, mirror.config.svn)
|
||||
if rel:
|
||||
svn_changed_files.setdefault(rel, []).append(entry.revision)
|
||||
|
||||
# ── 3. Cross-reference ────────────────────────────────
|
||||
dev_file_set = set(dev_changed_files.keys())
|
||||
svn_file_set = set(svn_changed_files.keys())
|
||||
|
||||
conflicting = sorted(dev_file_set & svn_file_set)
|
||||
dev_only = sorted(dev_file_set - svn_file_set)
|
||||
svn_only = sorted(svn_file_set - dev_file_set)
|
||||
|
||||
boundary_svn_rev = db.get_svn_revision(boundary) if boundary else None
|
||||
|
||||
return {
|
||||
"mirror_id": mirror.config.id,
|
||||
"ref": ref,
|
||||
"git_head": git_head,
|
||||
"git_head_abbrev": git_head[:8] if git_head else "?",
|
||||
"boundary": boundary,
|
||||
"boundary_abbrev": boundary[:8] if boundary else "?",
|
||||
"boundary_svn_revision": boundary_svn_rev,
|
||||
"developer_commits": dev_summaries,
|
||||
"developer_count": len(dev_summaries),
|
||||
"pending_svn_revisions": pending_svn,
|
||||
"svn_pending_count": len(pending_svn),
|
||||
"last_svn_revision": last_svn_rev,
|
||||
"latest_svn_revision": latest_svn_rev,
|
||||
"dev_only_files": dev_only,
|
||||
"svn_only_files": svn_only,
|
||||
"conflicting_files": conflicting,
|
||||
"has_conflicts": len(conflicting) > 0,
|
||||
"has_divergence": len(dev_summaries) > 0 or len(pending_svn) > 0,
|
||||
}
|
||||
|
||||
|
||||
def print_assessment(report: Dict[str, Any]):
|
||||
"""Pretty-print an assessment report to stdout."""
|
||||
_log_bold(f"Reconciliation Assessment for mirror \"{report['mirror_id']}\"")
|
||||
_log_info(f"Ref: {report['ref']}")
|
||||
_log_info(f"Git HEAD: {report['git_head_abbrev']}")
|
||||
_log_info(f"")
|
||||
|
||||
boundary = report['boundary']
|
||||
boundary_rev = report['boundary_svn_revision']
|
||||
if boundary:
|
||||
_log_info(f"Last synced commit: {report['boundary_abbrev']}")
|
||||
_log_info(f" → SVN revision: r{boundary_rev}")
|
||||
else:
|
||||
_log_warn("No synced commit found on this ref")
|
||||
_log_info("")
|
||||
|
||||
# SVN pending
|
||||
pending = report['pending_svn_revisions']
|
||||
if pending:
|
||||
_log_info(f"Pending SVN revisions: {report['svn_pending_count']} "
|
||||
f"(r{report['last_svn_revision'] + 1}–"
|
||||
f"r{report['latest_svn_revision']})")
|
||||
for svn in pending:
|
||||
msg = svn['message'][:72]
|
||||
_log_info(f" r{svn['revision']}: {svn['author']:20s} \"{msg}\"")
|
||||
else:
|
||||
_log_info("Pending SVN revisions: none")
|
||||
_log_info("")
|
||||
|
||||
# Git pending
|
||||
dev = report['developer_commits']
|
||||
if dev:
|
||||
_log_info(f"Pending Git commits: {report['developer_count']}")
|
||||
for gc in dev:
|
||||
msg = gc['message'][:72]
|
||||
_log_info(f" {gc['abbrev']}: {gc['author']:30s} \"{msg}\"")
|
||||
else:
|
||||
_log_info("Pending Git commits: none")
|
||||
_log_info("")
|
||||
|
||||
# File analysis
|
||||
_log_bold("File change analysis")
|
||||
if report['conflicting_files']:
|
||||
_log_warn(f"Changed on BOTH sides (CONFLICT): "
|
||||
f"{len(report['conflicting_files'])} file(s)")
|
||||
for f in report['conflicting_files']:
|
||||
_log_info(f" {f}")
|
||||
else:
|
||||
_log_ok("No conflicting files")
|
||||
_log_info("")
|
||||
|
||||
if report['dev_only_files']:
|
||||
_log_info(f"Changed only on Git side: "
|
||||
f"{len(report['dev_only_files'])} file(s)")
|
||||
for f in report['dev_only_files'][:20]:
|
||||
_log_info(f" {f}")
|
||||
if len(report['dev_only_files']) > 20:
|
||||
_log_info(f" … and {len(report['dev_only_files']) - 20} more")
|
||||
if report['svn_only_files']:
|
||||
_log_info(f"Changed only on SVN side: "
|
||||
f"{len(report['svn_only_files'])} file(s)")
|
||||
for f in report['svn_only_files'][:20]:
|
||||
_log_info(f" {f}")
|
||||
if len(report['svn_only_files']) > 20:
|
||||
_log_info(f" … and {len(report['svn_only_files']) - 20} more")
|
||||
_log_info("")
|
||||
|
||||
if not report['has_divergence']:
|
||||
_log_ok("Mirror is already in sync — no reconciliation needed")
|
||||
else:
|
||||
_log_info("Strategy: git-wins (Git content takes precedence on conflicts)")
|
||||
if report['conflicting_files']:
|
||||
_log_info(f" → {len(report['conflicting_files'])} conflicting "
|
||||
f"file(s) will use Git version")
|
||||
_log_info("")
|
||||
_log_info("Run without --dry-run to apply.")
|
||||
|
||||
|
||||
# ─── Reconciliation ────────────────────────────────────────
|
||||
|
||||
|
||||
def reconcile(mirror: Mirror, strategy: str = "git-wins",
|
||||
dry_run: bool = False) -> int:
|
||||
"""Reconcile a diverged SVN↔Git mirror.
|
||||
|
||||
Strategy options:
|
||||
- ``git-wins`` (default): Git HEAD content takes precedence on
|
||||
conflicting files. SVN-only changes are preserved.
|
||||
|
||||
Returns the SVN revision number of the reconciliation commit,
|
||||
or 0 if nothing was needed.
|
||||
"""
|
||||
git_dir = str(mirror.canonical_dir)
|
||||
svn_cfg = mirror.config.svn
|
||||
db = mirror.db
|
||||
ref = "refs/heads/master"
|
||||
svn_branch = svn_cfg.trunk
|
||||
|
||||
with mirror.sync_lock():
|
||||
# ── 1. Assess ─────────────────────────────────────
|
||||
report = assess(mirror, ref=ref)
|
||||
|
||||
if dry_run:
|
||||
print_assessment(report)
|
||||
return 0
|
||||
|
||||
if not report['has_divergence']:
|
||||
_log_ok("Mirror is already in sync")
|
||||
return 0
|
||||
|
||||
boundary = report['boundary']
|
||||
dev_commits_hashes = [c['hash'] for c in report['developer_commits']]
|
||||
pending_revs = [r['revision'] for r in report['pending_svn_revisions']]
|
||||
|
||||
# ── 2. Backup mapping DB ──────────────────────────
|
||||
backup_path = Path(str(db.path) + f".reconcile-backup-{_now_ts()}")
|
||||
shutil.copy2(db.path, backup_path)
|
||||
_log_info(f"Mapping DB backed up to {backup_path.name}")
|
||||
|
||||
# ── 3. Ensure SVN WC at HEAD ──────────────────────
|
||||
_log_info(f"Checking out SVN {svn_branch} @ HEAD …")
|
||||
try:
|
||||
wc = _ensure_wc(mirror, svn_branch)
|
||||
except Exception as e:
|
||||
raise ReconcileError(f"Failed to checkout SVN WC: {e}")
|
||||
|
||||
# ── 4. Apply each developer commit to the WC ──────
|
||||
conflicts_seen = []
|
||||
for ch in dev_commits_hashes:
|
||||
summary = _get_log_summary(git_dir, ch)
|
||||
_log_info(f"Applying commit {summary['abbrev']}: "
|
||||
f"{summary['message'][:60]}…")
|
||||
|
||||
if strategy == "git-wins":
|
||||
try:
|
||||
_apply_diff_to_wc(mirror, ch, svn_branch, wc)
|
||||
except Exception as e:
|
||||
_log_warn(f"Failed to apply {ch[:8]}: {e}")
|
||||
_log_warn(" Skipping — manual intervention may be needed")
|
||||
else:
|
||||
raise ReconcileError(f"Unknown strategy: {strategy}")
|
||||
|
||||
# ── 5. SVN commit ─────────────────────────────────
|
||||
message = (
|
||||
f"[reconcile] Merge Git changes since SVN r{report['boundary_svn_revision']}\n"
|
||||
f"\n"
|
||||
f"Reconciliation of {len(dev_commits_hashes)} Git commit(s) and "
|
||||
f"{len(pending_revs)} SVN revision(s)\n"
|
||||
f"\n"
|
||||
f"Git commits applied:\n"
|
||||
)
|
||||
for gc in report['developer_commits']:
|
||||
message += f" {gc['abbrev']} {gc['message'][:72]}\n"
|
||||
|
||||
_log_info("Committing reconciled WC to SVN …")
|
||||
try:
|
||||
svn_author = "reconcile"
|
||||
result = _gts_run_svn(
|
||||
["commit", "--file", "-", "--username", svn_author],
|
||||
input=message.encode("utf-8"),
|
||||
wc=wc, timeout=120,
|
||||
)
|
||||
new_rev = _parse_commit_revision(result)
|
||||
except Exception as e:
|
||||
raise ReconcileError(f"SVN commit failed: {e}")
|
||||
|
||||
_log_ok(f"Created SVN r{new_rev} (reconciliation commit)")
|
||||
|
||||
# ── 6. Record mapping ─────────────────────────────
|
||||
git_head = report['git_head']
|
||||
db.record_mapping(new_rev, git_head, svn_branch, ref, source="git")
|
||||
_log_ok(f"Mapping recorded: r{new_rev} ↔ {git_head[:8]} ({ref})")
|
||||
|
||||
# Leave last_svn_revision unchanged — sync_all will
|
||||
# process pre-reconciliation SVN revisions naturally
|
||||
# and skip r{new_rev} because it's already mapped.
|
||||
|
||||
# ── 7. Summary ────────────────────────────────────
|
||||
_log_bold("Reconciliation complete")
|
||||
_log_info(f" SVN revision: r{new_rev}")
|
||||
_log_info(f" Git HEAD: {git_head[:8]}")
|
||||
if report['conflicting_files']:
|
||||
_log_warn(f" Conflicts resolved via '{strategy}' strategy for "
|
||||
f"{len(report['conflicting_files'])} file(s)")
|
||||
_log_info("")
|
||||
_log_info("Run `sync` to bring the Git mirror in line with SVN.")
|
||||
|
||||
return new_rev
|
||||
|
||||
|
||||
def _now_ts() -> str:
|
||||
from datetime import datetime
|
||||
return datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
Reference in New Issue
Block a user