feat: thread SVN auth (username/password) through all svn call sites
This commit is contained in:
@@ -11,6 +11,8 @@ mirrors:
|
|||||||
svn:
|
svn:
|
||||||
url: https://svn.example.com/svn/myproject
|
url: https://svn.example.com/svn/myproject
|
||||||
layout: std # "std" or "custom"
|
layout: std # "std" or "custom"
|
||||||
|
username: # SVN login (omit if anonymous)
|
||||||
|
password: # SVN password
|
||||||
# For non-standard layout:
|
# For non-standard layout:
|
||||||
# layout: custom
|
# layout: custom
|
||||||
# trunk: trunk
|
# trunk: trunk
|
||||||
|
|||||||
+15
-1
@@ -28,6 +28,14 @@ class SVNConfig:
|
|||||||
trunk: str = "trunk"
|
trunk: str = "trunk"
|
||||||
branches: str = "branches"
|
branches: str = "branches"
|
||||||
tags: str = "tags"
|
tags: str = "tags"
|
||||||
|
username: Optional[str] = None
|
||||||
|
password: Optional[str] = None
|
||||||
|
|
||||||
|
def auth_args(self) -> list:
|
||||||
|
"""Return ``['--username', u, '--password', p]`` if configured, else []."""
|
||||||
|
if self.username:
|
||||||
|
return ["--username", self.username, "--password", self.password or ""]
|
||||||
|
return []
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_dict(cls, d: dict, prefix: str) -> "SVNConfig":
|
def from_dict(cls, d: dict, prefix: str) -> "SVNConfig":
|
||||||
@@ -47,7 +55,13 @@ class SVNConfig:
|
|||||||
for name, val in [("trunk", trunk), ("branches", branches), ("tags", tags)]:
|
for name, val in [("trunk", trunk), ("branches", branches), ("tags", tags)]:
|
||||||
_check_type(val, str, f"{prefix}.{name}")
|
_check_type(val, str, f"{prefix}.{name}")
|
||||||
|
|
||||||
return cls(url=url, layout=layout, trunk=trunk, branches=branches, tags=tags)
|
username = d.get("username")
|
||||||
|
_check_optional(username, str, f"{prefix}.username")
|
||||||
|
password = d.get("password")
|
||||||
|
_check_optional(password, str, f"{prefix}.password")
|
||||||
|
|
||||||
|
return cls(url=url, layout=layout, trunk=trunk, branches=branches,
|
||||||
|
tags=tags, username=username, password=password)
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
|
|||||||
@@ -91,8 +91,11 @@ def _wc_path(mirror: Mirror, svn_branch: str) -> Path:
|
|||||||
|
|
||||||
|
|
||||||
def _run_svn(args: list, input: bytes = None, timeout: int = 120,
|
def _run_svn(args: list, input: bytes = None, timeout: int = 120,
|
||||||
wc: Optional[Path] = None):
|
wc: Optional[Path] = None,
|
||||||
|
auth_args: Optional[list] = None):
|
||||||
"""Run an svn command. If *wc* is given, run from that directory."""
|
"""Run an svn command. If *wc* is given, run from that directory."""
|
||||||
|
if auth_args:
|
||||||
|
args = auth_args + args
|
||||||
env = {**os.environ, "LC_ALL": "C"}
|
env = {**os.environ, "LC_ALL": "C"}
|
||||||
try:
|
try:
|
||||||
r = subprocess.run(
|
r = subprocess.run(
|
||||||
@@ -116,12 +119,14 @@ def _run_svn(args: list, input: bytes = None, timeout: int = 120,
|
|||||||
|
|
||||||
def _ensure_wc(mirror: Mirror, svn_branch: str):
|
def _ensure_wc(mirror: Mirror, svn_branch: str):
|
||||||
"""Ensure the SVN working copy for *svn_branch* exists and is at HEAD."""
|
"""Ensure the SVN working copy for *svn_branch* exists and is at HEAD."""
|
||||||
|
auth_args = mirror.config.svn.auth_args()
|
||||||
wc = _wc_path(mirror, svn_branch)
|
wc = _wc_path(mirror, svn_branch)
|
||||||
svn_url = mirror.config.svn.url.rstrip("/") + "/" + svn_branch
|
svn_url = mirror.config.svn.url.rstrip("/") + "/" + svn_branch
|
||||||
if not (wc / ".svn").exists():
|
if not (wc / ".svn").exists():
|
||||||
logger.info("Checking out SVN WC for %s …", svn_branch)
|
logger.info("Checking out SVN WC for %s …", svn_branch)
|
||||||
wc.mkdir(parents=True, exist_ok=True)
|
wc.mkdir(parents=True, exist_ok=True)
|
||||||
_run_svn(["checkout", svn_url, str(wc)], timeout=300)
|
_run_svn(["checkout", svn_url, str(wc)], timeout=300,
|
||||||
|
auth_args=auth_args)
|
||||||
else:
|
else:
|
||||||
logger.debug("Updating SVN WC for %s …", svn_branch)
|
logger.debug("Updating SVN WC for %s …", svn_branch)
|
||||||
_run_svn(["revert", "-R", "."], timeout=120, wc=wc)
|
_run_svn(["revert", "-R", "."], timeout=120, wc=wc)
|
||||||
|
|||||||
@@ -190,6 +190,9 @@ class Mirror:
|
|||||||
args.extend(["-b", svn_cfg.branches])
|
args.extend(["-b", svn_cfg.branches])
|
||||||
args.extend(["-t", svn_cfg.tags])
|
args.extend(["-t", svn_cfg.tags])
|
||||||
|
|
||||||
|
if svn_cfg.username:
|
||||||
|
args.extend(["--username", svn_cfg.username])
|
||||||
|
|
||||||
if self.config.authors:
|
if self.config.authors:
|
||||||
args.extend(["--authors-file", str(self.authors_path)])
|
args.extend(["--authors-file", str(self.authors_path)])
|
||||||
|
|
||||||
|
|||||||
@@ -146,6 +146,7 @@ def assess(mirror: Mirror, ref: str = "refs/heads/master") -> Dict[str, Any]:
|
|||||||
|
|
||||||
Only analyses *ref* (default: refs/heads/master → SVN trunk).
|
Only analyses *ref* (default: refs/heads/master → SVN trunk).
|
||||||
"""
|
"""
|
||||||
|
auth_args = mirror.config.svn.auth_args()
|
||||||
git_dir = str(mirror.canonical_dir)
|
git_dir = str(mirror.canonical_dir)
|
||||||
db = mirror.db
|
db = mirror.db
|
||||||
svn_url = mirror.config.svn.url
|
svn_url = mirror.config.svn.url
|
||||||
@@ -168,13 +169,14 @@ def assess(mirror: Mirror, ref: str = "refs/heads/master") -> Dict[str, Any]:
|
|||||||
# ── 2. SVN side ───────────────────────────────────────
|
# ── 2. SVN side ───────────────────────────────────────
|
||||||
last_svn_rev_str = db.get_state("last_svn_revision")
|
last_svn_rev_str = db.get_state("last_svn_revision")
|
||||||
last_svn_rev = int(last_svn_rev_str) if last_svn_rev_str else 0
|
last_svn_rev = int(last_svn_rev_str) if last_svn_rev_str else 0
|
||||||
latest_svn_rev = svn_get_latest_revision(svn_url)
|
latest_svn_rev = svn_get_latest_revision(svn_url, auth_args=auth_args)
|
||||||
|
|
||||||
pending_svn: List[Dict[str, Any]] = []
|
pending_svn: List[Dict[str, Any]] = []
|
||||||
svn_changed_files: Dict[str, List[int]] = {}
|
svn_changed_files: Dict[str, List[int]] = {}
|
||||||
if last_svn_rev < latest_svn_rev:
|
if last_svn_rev < latest_svn_rev:
|
||||||
try:
|
try:
|
||||||
entries = svn_get_log_range(svn_url, last_svn_rev + 1, latest_svn_rev)
|
entries = svn_get_log_range(svn_url, last_svn_rev + 1, latest_svn_rev,
|
||||||
|
auth_args=auth_args)
|
||||||
except Exception:
|
except Exception:
|
||||||
entries = []
|
entries = []
|
||||||
for entry in entries:
|
for entry in entries:
|
||||||
|
|||||||
+15
-9
@@ -28,7 +28,10 @@ class SVNLogEntry:
|
|||||||
paths: List[SVNPathChange] = field(default_factory=list)
|
paths: List[SVNPathChange] = field(default_factory=list)
|
||||||
|
|
||||||
|
|
||||||
def _run_svn(args: list, input: bytes = None, timeout: int = 120) -> subprocess.CompletedProcess:
|
def _run_svn(args: list, input: bytes = None, timeout: int = 120,
|
||||||
|
auth_args: Optional[list] = None) -> subprocess.CompletedProcess:
|
||||||
|
if auth_args:
|
||||||
|
args = auth_args + args
|
||||||
try:
|
try:
|
||||||
r = subprocess.run(
|
r = subprocess.run(
|
||||||
["svn"] + args,
|
["svn"] + args,
|
||||||
@@ -48,9 +51,9 @@ def _run_svn(args: list, input: bytes = None, timeout: int = 120) -> subprocess.
|
|||||||
return r
|
return r
|
||||||
|
|
||||||
|
|
||||||
def get_latest_revision(url: str) -> int:
|
def get_latest_revision(url: str, auth_args: Optional[list] = None) -> int:
|
||||||
"""Return the HEAD revision number of an SVN repository."""
|
"""Return the HEAD revision number of an SVN repository."""
|
||||||
r = _run_svn(["info", "--xml", url], timeout=60)
|
r = _run_svn(["info", "--xml", url], timeout=60, auth_args=auth_args)
|
||||||
root = ET.fromstring(r.stdout)
|
root = ET.fromstring(r.stdout)
|
||||||
entry = root.find(".//entry")
|
entry = root.find(".//entry")
|
||||||
if entry is None:
|
if entry is None:
|
||||||
@@ -61,11 +64,11 @@ def get_latest_revision(url: str) -> int:
|
|||||||
return int(rev)
|
return int(rev)
|
||||||
|
|
||||||
|
|
||||||
def get_log(url: str, revision: int) -> SVNLogEntry:
|
def get_log(url: str, revision: int, auth_args: Optional[list] = None) -> SVNLogEntry:
|
||||||
"""Fetch log entry with changed paths for a single SVN revision."""
|
"""Fetch log entry with changed paths for a single SVN revision."""
|
||||||
r = _run_svn(
|
r = _run_svn(
|
||||||
["log", "-r", str(revision), "--verbose", "--xml", url],
|
["log", "-r", str(revision), "--verbose", "--xml", url],
|
||||||
timeout=120,
|
timeout=120, auth_args=auth_args,
|
||||||
)
|
)
|
||||||
root = ET.fromstring(r.stdout)
|
root = ET.fromstring(r.stdout)
|
||||||
logentry = root.find("logentry")
|
logentry = root.find("logentry")
|
||||||
@@ -108,14 +111,15 @@ def get_log(url: str, revision: int) -> SVNLogEntry:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def get_log_range(url: str, start_rev: int, end_rev: int) -> List[SVNLogEntry]:
|
def get_log_range(url: str, start_rev: int, end_rev: int,
|
||||||
|
auth_args: Optional[list] = None) -> List[SVNLogEntry]:
|
||||||
"""Fetch log entries for a range of SVN revisions in one call.
|
"""Fetch log entries for a range of SVN revisions in one call.
|
||||||
|
|
||||||
Returns entries from *start_rev* to *end_rev* (inclusive), oldest first.
|
Returns entries from *start_rev* to *end_rev* (inclusive), oldest first.
|
||||||
"""
|
"""
|
||||||
r = _run_svn(
|
r = _run_svn(
|
||||||
["log", "-r", f"{start_rev}:{end_rev}", "--verbose", "--xml", url],
|
["log", "-r", f"{start_rev}:{end_rev}", "--verbose", "--xml", url],
|
||||||
timeout=300,
|
timeout=300, auth_args=auth_args,
|
||||||
)
|
)
|
||||||
root = ET.fromstring(r.stdout)
|
root = ET.fromstring(r.stdout)
|
||||||
entries: List[SVNLogEntry] = []
|
entries: List[SVNLogEntry] = []
|
||||||
@@ -163,11 +167,13 @@ def get_log_range(url: str, start_rev: int, end_rev: int) -> List[SVNLogEntry]:
|
|||||||
return entries
|
return entries
|
||||||
|
|
||||||
|
|
||||||
def get_file(url: str, path: str, revision: int) -> bytes:
|
def get_file(url: str, path: str, revision: int,
|
||||||
|
auth_args: Optional[list] = None) -> bytes:
|
||||||
"""Return the full content of a file at a given SVN revision.
|
"""Return the full content of a file at a given SVN revision.
|
||||||
|
|
||||||
`path` is the absolute SVN path, e.g. /trunk/src/main.c.
|
`path` is the absolute SVN path, e.g. /trunk/src/main.c.
|
||||||
"""
|
"""
|
||||||
full_url = url.rstrip("/") + path
|
full_url = url.rstrip("/") + path
|
||||||
r = _run_svn(["cat", "-r", str(revision), full_url], timeout=60)
|
r = _run_svn(["cat", "-r", str(revision), full_url], timeout=60,
|
||||||
|
auth_args=auth_args)
|
||||||
return r.stdout
|
return r.stdout
|
||||||
|
|||||||
+10
-6
@@ -177,6 +177,7 @@ def _build_tree_from_changes(
|
|||||||
changes: List[Tuple],
|
changes: List[Tuple],
|
||||||
svn_url: str,
|
svn_url: str,
|
||||||
revision: int,
|
revision: int,
|
||||||
|
auth_args: Optional[List[str]] = None,
|
||||||
) -> str:
|
) -> str:
|
||||||
"""Build a new Git tree by applying SVN changes to the parent tree.
|
"""Build a new Git tree by applying SVN changes to the parent tree.
|
||||||
|
|
||||||
@@ -202,7 +203,7 @@ def _build_tree_from_changes(
|
|||||||
# ── 2. Apply changes ──────────────────────────────────────
|
# ── 2. Apply changes ──────────────────────────────────────
|
||||||
for action, kind, rel_path, full_path, cf_path, cf_rev in changes:
|
for action, kind, rel_path, full_path, cf_path, cf_rev in changes:
|
||||||
if action in ("A", "M") and kind == "file":
|
if action in ("A", "M") and kind == "file":
|
||||||
content = svn_get_file(svn_url, full_path, revision)
|
content = svn_get_file(svn_url, full_path, revision, auth_args=auth_args)
|
||||||
blob_oid = _git_bytes(git_dir, "hash-object", "-w", "--stdin",
|
blob_oid = _git_bytes(git_dir, "hash-object", "-w", "--stdin",
|
||||||
input=content, timeout=60)
|
input=content, timeout=60)
|
||||||
entries[rel_path] = ("100644", blob_oid)
|
entries[rel_path] = ("100644", blob_oid)
|
||||||
@@ -218,7 +219,7 @@ def _build_tree_from_changes(
|
|||||||
|
|
||||||
elif action == "R" and kind == "file":
|
elif action == "R" and kind == "file":
|
||||||
entries.pop(rel_path, None)
|
entries.pop(rel_path, None)
|
||||||
content = svn_get_file(svn_url, full_path, revision)
|
content = svn_get_file(svn_url, full_path, revision, auth_args=auth_args)
|
||||||
blob_oid = _git_bytes(git_dir, "hash-object", "-w", "--stdin",
|
blob_oid = _git_bytes(git_dir, "hash-object", "-w", "--stdin",
|
||||||
input=content, timeout=60)
|
input=content, timeout=60)
|
||||||
entries[rel_path] = ("100644", blob_oid)
|
entries[rel_path] = ("100644", blob_oid)
|
||||||
@@ -311,7 +312,8 @@ def _find_copy_source(cfg, db, changes) -> Optional[str]:
|
|||||||
# ─── Per-revision sync ────────────────────────────────────────
|
# ─── Per-revision sync ────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
def sync_svn_revision(mirror: Mirror, revision: int) -> bool:
|
def sync_svn_revision(mirror: Mirror, revision: int,
|
||||||
|
auth_args: Optional[List[str]] = None) -> bool:
|
||||||
"""Translate a single SVN revision into Git commit(s).
|
"""Translate a single SVN revision into Git commit(s).
|
||||||
|
|
||||||
Returns True if at least one Git commit was created.
|
Returns True if at least one Git commit was created.
|
||||||
@@ -320,7 +322,7 @@ def sync_svn_revision(mirror: Mirror, revision: int) -> bool:
|
|||||||
git_dir = str(mirror.canonical_dir)
|
git_dir = str(mirror.canonical_dir)
|
||||||
|
|
||||||
# Fetch SVN metadata
|
# Fetch SVN metadata
|
||||||
log_entry = svn_get_log(cfg.svn.url, revision)
|
log_entry = svn_get_log(cfg.svn.url, revision, auth_args=auth_args)
|
||||||
branch_changes = group_changes_by_branch(log_entry.paths, cfg.svn)
|
branch_changes = group_changes_by_branch(log_entry.paths, cfg.svn)
|
||||||
|
|
||||||
if not branch_changes:
|
if not branch_changes:
|
||||||
@@ -348,6 +350,7 @@ def sync_svn_revision(mirror: Mirror, revision: int) -> bool:
|
|||||||
# Build new tree
|
# Build new tree
|
||||||
new_tree = _build_tree_from_changes(
|
new_tree = _build_tree_from_changes(
|
||||||
git_dir, parent_hash, changes, cfg.svn.url, revision,
|
git_dir, parent_hash, changes, cfg.svn.url, revision,
|
||||||
|
auth_args=auth_args,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Map author
|
# Map author
|
||||||
@@ -423,10 +426,11 @@ def sync_all(mirror: Mirror) -> int:
|
|||||||
|
|
||||||
|
|
||||||
def _sync_all_impl(mirror: Mirror) -> int:
|
def _sync_all_impl(mirror: Mirror) -> int:
|
||||||
|
auth_args = mirror.config.svn.auth_args()
|
||||||
state_rev = mirror.db.get_state("last_svn_revision")
|
state_rev = mirror.db.get_state("last_svn_revision")
|
||||||
trunk_name = mirror.config.svn.trunk
|
trunk_name = mirror.config.svn.trunk
|
||||||
last_rev = int(state_rev) if state_rev else (mirror.db.get_last_svn_revision(trunk_name) or 0)
|
last_rev = int(state_rev) if state_rev else (mirror.db.get_last_svn_revision(trunk_name) or 0)
|
||||||
latest_rev = svn_get_latest_revision(mirror.config.svn.url)
|
latest_rev = svn_get_latest_revision(mirror.config.svn.url, auth_args=auth_args)
|
||||||
|
|
||||||
if latest_rev <= last_rev:
|
if latest_rev <= last_rev:
|
||||||
logger.debug("Mirror %s is up to date (r%d)", mirror.config.id, last_rev)
|
logger.debug("Mirror %s is up to date (r%d)", mirror.config.id, last_rev)
|
||||||
@@ -440,7 +444,7 @@ def _sync_all_impl(mirror: Mirror) -> int:
|
|||||||
count = 0
|
count = 0
|
||||||
for rev in range(last_rev + 1, latest_rev + 1):
|
for rev in range(last_rev + 1, latest_rev + 1):
|
||||||
try:
|
try:
|
||||||
if sync_svn_revision(mirror, rev):
|
if sync_svn_revision(mirror, rev, auth_args=auth_args):
|
||||||
count += 1
|
count += 1
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.exception("Failed to sync SVN r%d for %s", rev, mirror.config.id)
|
logger.exception("Failed to sync SVN r%d for %s", rev, mirror.config.id)
|
||||||
|
|||||||
Reference in New Issue
Block a user