From 90db42833330dcf3f59244c22e7e44f9e9206767 Mon Sep 17 00:00:00 2001 From: yujiezhang-ops Date: Fri, 11 Sep 2026 09:45:39 +0800 Subject: [PATCH] fix: allow slow Gitee OTA uploads --- scripts/sync_gitee_ota_release.py | 132 ++++++++++++++++++++----- scripts/test_sync_gitee_ota_release.py | 64 +++++++++++- 2 files changed, 169 insertions(+), 27 deletions(-) diff --git a/scripts/sync_gitee_ota_release.py b/scripts/sync_gitee_ota_release.py index 726037b..8a406f7 100644 --- a/scripts/sync_gitee_ota_release.py +++ b/scripts/sync_gitee_ota_release.py @@ -15,6 +15,7 @@ import re import secrets import tempfile +import time import urllib.error import urllib.parse import urllib.request @@ -24,6 +25,14 @@ OTA_ASSET_RE = re.compile(r"^ota-BootAgent-(?:darwin|linux|windows)-(?:amd64|arm64)\.zip$") GITHUB_API = "https://api.github.com" GITEE_API = "https://gitee.com/api/v5" +# Gitee answers intermittently: BootAgent's own updater notes roughly a third of +# consecutive SHA256SUMS fetches returning 403. The action this script replaced +# retried three times, so transient failures must not fail the release sync. +ATTEMPTS = 3 +RETRY_DELAY_SECONDS = 5 +# Server-side and rate-limit responses are worth another attempt; a 401/404 is a +# configuration error that will answer the same way every time. +RETRYABLE_STATUSES = frozenset({403, 408, 429, 500, 502, 503, 504}) def wanted_asset(name: str) -> bool: @@ -37,13 +46,42 @@ def selected_assets(release: dict[str, object]) -> list[dict[str, object]]: return [asset for asset in assets if isinstance(asset, dict) and wanted_asset(str(asset.get("name", "")))] -def request_json(request: urllib.request.Request) -> dict[str, object]: +class RequestFailed(RuntimeError): + """An HTTP request that failed, carrying the status so retry can judge it.""" + + def __init__(self, message: str, status: int | None) -> None: + super().__init__(message) + self.status = status + + def retryable(self) -> bool: + return self.status is None or self.status in RETRYABLE_STATUSES + + +def with_retry(description: str, action): + """Run action, retrying the failures Gitee produces intermittently.""" + for attempt in range(1, ATTEMPTS + 1): + try: + return action() + except RequestFailed as error: + if attempt == ATTEMPTS or not error.retryable(): + raise + print(f"{description} failed (attempt {attempt}/{ATTEMPTS}), retrying: {error}") + time.sleep(RETRY_DELAY_SECONDS * attempt) + raise AssertionError("unreachable") + + +def request_json(request: urllib.request.Request, timeout: float = 60) -> dict[str, object]: try: - with urllib.request.urlopen(request, timeout=60) as response: + with urllib.request.urlopen(request, timeout=timeout) as response: payload = response.read() except urllib.error.HTTPError as error: body = error.read().decode("utf-8", errors="replace") - raise RuntimeError(f"{request.method} {request.full_url} failed ({error.code}): {body[:500]}") from error + raise RequestFailed( + f"{request.method} {request.full_url} failed ({error.code}): {body[:500]}", error.code + ) from error + except (OSError, urllib.error.URLError) as error: + # A dropped connection has no status; it is the transient case retry exists for. + raise RequestFailed(f"{request.method} {request.full_url} failed: {error}", None) from error result = json.loads(payload) if payload else {} if not isinstance(result, dict): raise RuntimeError(f"{request.method} {request.full_url} returned a non-object response") @@ -55,16 +93,19 @@ def github_release(owner: str, repo: str, tag: str, token: str) -> dict[str, obj if token: headers["Authorization"] = f"Bearer {token}" url = f"{GITHUB_API}/repos/{owner}/{repo}/releases/tags/{urllib.parse.quote(tag)}" - return request_json(urllib.request.Request(url, headers=headers)) + return with_retry( + "read GitHub release", lambda: request_json(urllib.request.Request(url, headers=headers)) + ) def gitee_release(owner: str, repo: str, tag: str, token: str) -> dict[str, object] | None: url = f"{GITEE_API}/repos/{owner}/{repo}/releases/tags/{urllib.parse.quote(tag)}" request = urllib.request.Request(url, headers={"Authorization": f"token {token}", "Accept": "application/json"}) try: - return request_json(request) - except RuntimeError as error: - if " failed (404):" in str(error): + return with_retry("read Gitee release", lambda: request_json(request)) + except RequestFailed as error: + # No release for this tag yet, which is the normal first-sync case. + if error.status == 404: return None raise @@ -82,7 +123,10 @@ def create_gitee_release( } ).encode() url = f"{GITEE_API}/repos/{owner}/{repo}/releases" - return request_json(urllib.request.Request(url, data=values, method="POST")) + return with_retry( + "create Gitee release", + lambda: request_json(urllib.request.Request(url, data=values, method="POST")), + ) def download_asset(asset: dict[str, object], directory: Path, token: str) -> Path: @@ -93,17 +137,28 @@ def download_asset(asset: dict[str, object], directory: Path, token: str) -> Pat headers["Authorization"] = f"Bearer {token}" target = directory / name request = urllib.request.Request(url, headers=headers) - try: - with urllib.request.urlopen(request, timeout=120) as response, target.open("wb") as output: - while chunk := response.read(1024 * 1024): - output.write(chunk) - except (OSError, urllib.error.URLError) as error: - target.unlink(missing_ok=True) - raise RuntimeError(f"failed to download GitHub asset {name}: {error}") from error + + def fetch() -> None: + try: + with urllib.request.urlopen(request, timeout=120) as response, target.open("wb") as output: + while chunk := response.read(1024 * 1024): + output.write(chunk) + except urllib.error.HTTPError as error: + target.unlink(missing_ok=True) + raise RequestFailed(f"failed to download GitHub asset {name}: {error}", error.code) from error + except (OSError, urllib.error.URLError) as error: + # A partial file must not survive: the next attempt reopens it with + # "wb", but a failure that exhausts the retries would otherwise leave + # a truncated asset behind for the upload step to send. + target.unlink(missing_ok=True) + raise RequestFailed(f"failed to download GitHub asset {name}: {error}", None) from error + + with_retry(f"download {name}", fetch) return target -def multipart_file(field: str, path: Path, token: str) -> tuple[bytes, str]: +def multipart_envelope(field: str, path: Path, token: str) -> tuple[bytes, bytes, str]: + """Return the multipart prefix and suffix that wrap the file's bytes.""" boundary = f"bootagent-{secrets.token_hex(16)}" content_type = mimetypes.guess_type(path.name)[0] or "application/octet-stream" prefix = ( @@ -112,20 +167,45 @@ def multipart_file(field: str, path: Path, token: str) -> tuple[bytes, str]: f"Content-Type: {content_type}\r\n\r\n" ).encode() suffix = f"\r\n--{boundary}--\r\n".encode() - return prefix + path.read_bytes() + suffix, boundary + return prefix, suffix, boundary def upload_asset(owner: str, repo: str, token: str, release_id: object, path: Path) -> None: - body, boundary = multipart_file("file", path, token) + """Attach one file to a Gitee release, streaming it rather than buffering it. + + The body is assembled on disk and handed to urllib as an open file, with an + explicit Content-Length: urllib cannot size a file object itself, and without + the header it would fall back to chunked encoding, which Gitee's endpoint + does not accept. An OTA zip is over 100 MB, so reading it into memory to + build the request costs more than the request itself. + """ + prefix, suffix, boundary = multipart_envelope("file", path, token) url = f"{GITEE_API}/repos/{owner}/{repo}/releases/{release_id}/attach_files" - request_json( - urllib.request.Request( - url, - data=body, - method="POST", - headers={"Content-Type": f"multipart/form-data; boundary={boundary}"}, - ) - ) + with tempfile.NamedTemporaryFile(prefix="bootagent-upload-", suffix=".multipart") as body: + body.write(prefix) + with path.open("rb") as source: + while chunk := source.read(1024 * 1024): + body.write(chunk) + body.write(suffix) + length = body.tell() + + def send() -> dict[str, object]: + # Rewound per attempt: a retry has to resend from the start. + body.seek(0) + return request_json( + urllib.request.Request( + url, + data=body, + method="POST", + headers={ + "Content-Type": f"multipart/form-data; boundary={boundary}", + "Content-Length": str(length), + }, + ), + timeout=30 * 60, + ) + + with_retry(f"upload {path.name}", send) def main() -> int: diff --git a/scripts/test_sync_gitee_ota_release.py b/scripts/test_sync_gitee_ota_release.py index 31c5b21..7701ecd 100644 --- a/scripts/test_sync_gitee_ota_release.py +++ b/scripts/test_sync_gitee_ota_release.py @@ -1,6 +1,13 @@ import unittest +from unittest import mock -from scripts.sync_gitee_ota_release import selected_assets, wanted_asset +from scripts.sync_gitee_ota_release import ( + ATTEMPTS, + RequestFailed, + selected_assets, + wanted_asset, + with_retry, +) class SyncGiteeOTAReleaseTests(unittest.TestCase): @@ -25,5 +32,60 @@ def test_rejects_similarly_named_files(self) -> None: self.assertFalse(wanted_asset("BootAgent-linux-amd64.deb")) +class RetryTests(unittest.TestCase): + """Gitee returns 403 intermittently, so a transient failure must not fail the sync.""" + + def test_retries_a_transient_status_then_succeeds(self) -> None: + attempts = [] + + def action() -> str: + attempts.append(None) + if len(attempts) < 3: + raise RequestFailed("Gitee said no", 403) + return "uploaded" + + with mock.patch("scripts.sync_gitee_ota_release.time.sleep"): + self.assertEqual(with_retry("upload", action), "uploaded") + self.assertEqual(len(attempts), 3) + + def test_gives_up_after_the_attempt_limit(self) -> None: + attempts = [] + + def action() -> None: + attempts.append(None) + raise RequestFailed("Gitee is down", 502) + + with mock.patch("scripts.sync_gitee_ota_release.time.sleep"): + with self.assertRaises(RequestFailed): + with_retry("upload", action) + self.assertEqual(len(attempts), ATTEMPTS) + + def test_does_not_retry_a_configuration_error(self) -> None: + """A bad token answers the same way every time; retrying only delays the failure.""" + attempts = [] + + def action() -> None: + attempts.append(None) + raise RequestFailed("unauthorized", 401) + + with mock.patch("scripts.sync_gitee_ota_release.time.sleep"): + with self.assertRaises(RequestFailed): + with_retry("upload", action) + self.assertEqual(len(attempts), 1) + + def test_retries_a_connection_drop_that_carries_no_status(self) -> None: + attempts = [] + + def action() -> str: + attempts.append(None) + if len(attempts) < 2: + raise RequestFailed("connection reset", None) + return "done" + + with mock.patch("scripts.sync_gitee_ota_release.time.sleep"): + self.assertEqual(with_retry("download", action), "done") + self.assertEqual(len(attempts), 2) + + if __name__ == "__main__": unittest.main()