fix: allow slow Gitee OTA uploads

scripts/sync_gitee_ota_release.py

@@ -15,6 +15,7 @@
 import re
 import secrets
 import tempfile
+import time
 import urllib.error
 import urllib.parse
 import urllib.request
@@ -24,6 +25,14 @@
 OTA_ASSET_RE = re.compile(r"^ota-BootAgent-(?:darwin|linux|windows)-(?:amd64|arm64)\.zip$")
 GITHUB_API = "https://api.github.com"
 GITEE_API = "https://gitee.com/api/v5"
+# Gitee answers intermittently: BootAgent's own updater notes roughly a third of
+# consecutive SHA256SUMS fetches returning 403. The action this script replaced
+# retried three times, so transient failures must not fail the release sync.
+ATTEMPTS = 3
+RETRY_DELAY_SECONDS = 5
+# Server-side and rate-limit responses are worth another attempt; a 401/404 is a
+# configuration error that will answer the same way every time.
+RETRYABLE_STATUSES = frozenset({403, 408, 429, 500, 502, 503, 504})
 
 
 def wanted_asset(name: str) -> bool:
@@ -37,13 +46,42 @@ def selected_assets(release: dict[str, object]) -> list[dict[str, object]]:
     return [asset for asset in assets if isinstance(asset, dict) and wanted_asset(str(asset.get("name", "")))]
 
 
-def request_json(request: urllib.request.Request) -> dict[str, object]:
+class RequestFailed(RuntimeError):
+    """An HTTP request that failed, carrying the status so retry can judge it."""
+
+    def __init__(self, message: str, status: int | None) -> None:
+        super().__init__(message)
+        self.status = status
+
+    def retryable(self) -> bool:
+        return self.status is None or self.status in RETRYABLE_STATUSES
+
+
+def with_retry(description: str, action):
+    """Run action, retrying the failures Gitee produces intermittently."""
+    for attempt in range(1, ATTEMPTS + 1):
+        try:
+            return action()
+        except RequestFailed as error:
+            if attempt == ATTEMPTS or not error.retryable():
+                raise
+            print(f"{description} failed (attempt {attempt}/{ATTEMPTS}), retrying: {error}")
+            time.sleep(RETRY_DELAY_SECONDS * attempt)
+    raise AssertionError("unreachable")
+
+
+def request_json(request: urllib.request.Request, timeout: float = 60) -> dict[str, object]:
     try:
-        with urllib.request.urlopen(request, timeout=60) as response:
+        with urllib.request.urlopen(request, timeout=timeout) as response:
             payload = response.read()
     except urllib.error.HTTPError as error:
         body = error.read().decode("utf-8", errors="replace")
-        raise RuntimeError(f"{request.method} {request.full_url} failed ({error.code}): {body[:500]}") from error
+        raise RequestFailed(
+            f"{request.method} {request.full_url} failed ({error.code}): {body[:500]}", error.code
+        ) from error
+    except (OSError, urllib.error.URLError) as error:
+        # A dropped connection has no status; it is the transient case retry exists for.
+        raise RequestFailed(f"{request.method} {request.full_url} failed: {error}", None) from error
     result = json.loads(payload) if payload else {}
     if not isinstance(result, dict):
         raise RuntimeError(f"{request.method} {request.full_url} returned a non-object response")
@@ -55,16 +93,19 @@ def github_release(owner: str, repo: str, tag: str, token: str) -> dict[str, obj
     if token:
         headers["Authorization"] = f"Bearer {token}"
     url = f"{GITHUB_API}/repos/{owner}/{repo}/releases/tags/{urllib.parse.quote(tag)}"
-    return request_json(urllib.request.Request(url, headers=headers))
+    return with_retry(
+        "read GitHub release", lambda: request_json(urllib.request.Request(url, headers=headers))
+    )
 
 
 def gitee_release(owner: str, repo: str, tag: str, token: str) -> dict[str, object] | None:
     url = f"{GITEE_API}/repos/{owner}/{repo}/releases/tags/{urllib.parse.quote(tag)}"
     request = urllib.request.Request(url, headers={"Authorization": f"token {token}", "Accept": "application/json"})
     try:
-        return request_json(request)
-    except RuntimeError as error:
-        if " failed (404):" in str(error):
+        return with_retry("read Gitee release", lambda: request_json(request))
+    except RequestFailed as error:
+        # No release for this tag yet, which is the normal first-sync case.
+        if error.status == 404:
             return None
         raise
 
@@ -82,7 +123,10 @@ def create_gitee_release(
         }
     ).encode()
     url = f"{GITEE_API}/repos/{owner}/{repo}/releases"
-    return request_json(urllib.request.Request(url, data=values, method="POST"))
+    return with_retry(
+        "create Gitee release",
+        lambda: request_json(urllib.request.Request(url, data=values, method="POST")),
+    )
 
 
 def download_asset(asset: dict[str, object], directory: Path, token: str) -> Path:
@@ -93,17 +137,28 @@ def download_asset(asset: dict[str, object], directory: Path, token: str) -> Pat
         headers["Authorization"] = f"Bearer {token}"
     target = directory / name
     request = urllib.request.Request(url, headers=headers)
-    try:
-        with urllib.request.urlopen(request, timeout=120) as response, target.open("wb") as output:
-            while chunk := response.read(1024 * 1024):
-                output.write(chunk)
-    except (OSError, urllib.error.URLError) as error:
-        target.unlink(missing_ok=True)
-        raise RuntimeError(f"failed to download GitHub asset {name}: {error}") from error
+
+    def fetch() -> None:
+        try:
+            with urllib.request.urlopen(request, timeout=120) as response, target.open("wb") as output:
+                while chunk := response.read(1024 * 1024):
+                    output.write(chunk)
+        except urllib.error.HTTPError as error:
+            target.unlink(missing_ok=True)
+            raise RequestFailed(f"failed to download GitHub asset {name}: {error}", error.code) from error
+        except (OSError, urllib.error.URLError) as error:
+            # A partial file must not survive: the next attempt reopens it with
+            # "wb", but a failure that exhausts the retries would otherwise leave
+            # a truncated asset behind for the upload step to send.
+            target.unlink(missing_ok=True)
+            raise RequestFailed(f"failed to download GitHub asset {name}: {error}", None) from error
+
+    with_retry(f"download {name}", fetch)
     return target
 
 
-def multipart_file(field: str, path: Path, token: str) -> tuple[bytes, str]:
+def multipart_envelope(field: str, path: Path, token: str) -> tuple[bytes, bytes, str]:
+    """Return the multipart prefix and suffix that wrap the file's bytes."""
     boundary = f"bootagent-{secrets.token_hex(16)}"
     content_type = mimetypes.guess_type(path.name)[0] or "application/octet-stream"
     prefix = (
@@ -112,20 +167,45 @@ def multipart_file(field: str, path: Path, token: str) -> tuple[bytes, str]:
         f"Content-Type: {content_type}\r\n\r\n"
     ).encode()
     suffix = f"\r\n--{boundary}--\r\n".encode()
-    return prefix + path.read_bytes() + suffix, boundary
+    return prefix, suffix, boundary
 
 
 def upload_asset(owner: str, repo: str, token: str, release_id: object, path: Path) -> None:
-    body, boundary = multipart_file("file", path, token)
+    """Attach one file to a Gitee release, streaming it rather than buffering it.
+
+    The body is assembled on disk and handed to urllib as an open file, with an
+    explicit Content-Length: urllib cannot size a file object itself, and without
+    the header it would fall back to chunked encoding, which Gitee's endpoint
+    does not accept. An OTA zip is over 100 MB, so reading it into memory to
+    build the request costs more than the request itself.
+    """
+    prefix, suffix, boundary = multipart_envelope("file", path, token)
     url = f"{GITEE_API}/repos/{owner}/{repo}/releases/{release_id}/attach_files"
-    request_json(
-        urllib.request.Request(
-            url,
-            data=body,
-            method="POST",
-            headers={"Content-Type": f"multipart/form-data; boundary={boundary}"},
-        )
-    )
+    with tempfile.NamedTemporaryFile(prefix="bootagent-upload-", suffix=".multipart") as body:
+        body.write(prefix)
+        with path.open("rb") as source:
+            while chunk := source.read(1024 * 1024):
+                body.write(chunk)
+        body.write(suffix)
+        length = body.tell()
+
+        def send() -> dict[str, object]:
+            # Rewound per attempt: a retry has to resend from the start.
+            body.seek(0)
+            return request_json(
+                urllib.request.Request(
+                    url,
+                    data=body,
+                    method="POST",
+                    headers={
+                        "Content-Type": f"multipart/form-data; boundary={boundary}",
+                        "Content-Length": str(length),
+                    },
+                ),
+                timeout=30 * 60,
+            )
+
+        with_retry(f"upload {path.name}", send)
 
 
 def main() -> int:

scripts/test_sync_gitee_ota_release.py

@@ -1,6 +1,13 @@
 import unittest
+from unittest import mock
 
-from scripts.sync_gitee_ota_release import selected_assets, wanted_asset
+from scripts.sync_gitee_ota_release import (
+    ATTEMPTS,
+    RequestFailed,
+    selected_assets,
+    wanted_asset,
+    with_retry,
+)
 
 
 class SyncGiteeOTAReleaseTests(unittest.TestCase):
@@ -25,5 +32,60 @@ def test_rejects_similarly_named_files(self) -> None:
         self.assertFalse(wanted_asset("BootAgent-linux-amd64.deb"))
 
 
+class RetryTests(unittest.TestCase):
+    """Gitee returns 403 intermittently, so a transient failure must not fail the sync."""
+
+    def test_retries_a_transient_status_then_succeeds(self) -> None:
+        attempts = []
+
+        def action() -> str:
+            attempts.append(None)
+            if len(attempts) < 3:
+                raise RequestFailed("Gitee said no", 403)
+            return "uploaded"
+
+        with mock.patch("scripts.sync_gitee_ota_release.time.sleep"):
+            self.assertEqual(with_retry("upload", action), "uploaded")
+        self.assertEqual(len(attempts), 3)
+
+    def test_gives_up_after_the_attempt_limit(self) -> None:
+        attempts = []
+
+        def action() -> None:
+            attempts.append(None)
+            raise RequestFailed("Gitee is down", 502)
+
+        with mock.patch("scripts.sync_gitee_ota_release.time.sleep"):
+            with self.assertRaises(RequestFailed):
+                with_retry("upload", action)
+        self.assertEqual(len(attempts), ATTEMPTS)
+
+    def test_does_not_retry_a_configuration_error(self) -> None:
+        """A bad token answers the same way every time; retrying only delays the failure."""
+        attempts = []
+
+        def action() -> None:
+            attempts.append(None)
+            raise RequestFailed("unauthorized", 401)
+
+        with mock.patch("scripts.sync_gitee_ota_release.time.sleep"):
+            with self.assertRaises(RequestFailed):
+                with_retry("upload", action)
+        self.assertEqual(len(attempts), 1)
+
+    def test_retries_a_connection_drop_that_carries_no_status(self) -> None:
+        attempts = []
+
+        def action() -> str:
+            attempts.append(None)
+            if len(attempts) < 2:
+                raise RequestFailed("connection reset", None)
+            return "done"
+
+        with mock.patch("scripts.sync_gitee_ota_release.time.sleep"):
+            self.assertEqual(with_retry("download", action), "done")
+        self.assertEqual(len(attempts), 2)
+
+
 if __name__ == "__main__":
     unittest.main()