From 58a541c489a18455831a3263b1c37cff677b63f5 Mon Sep 17 00:00:00 2001 From: Omkar Chandorkar Date: Sat, 12 Sep 2026 09:51:41 +0530 Subject: [PATCH 1/7] bot: Apply ordered delta OTA chains Signed-off-by: Omkar Chandorkar --- dumpyarabot/aria2_manager.py | 23 +- dumpyarabot/arq_config.py | 28 ++ dumpyarabot/arq_jobs.py | 193 ++++++---- dumpyarabot/firmware_downloader.py | 144 ++++--- dumpyarabot/firmware_extractor.py | 581 ++++++++++++++++++++++++++++- dumpyarabot/handlers.py | 88 +++-- dumpyarabot/message_formatting.py | 55 ++- dumpyarabot/message_queue.py | 20 +- dumpyarabot/mockup_handlers.py | 41 +- dumpyarabot/moderated_handlers.py | 200 ++++++---- dumpyarabot/privacy.py | 78 ++++ dumpyarabot/process_utils.py | 4 +- dumpyarabot/schemas.py | 5 +- dumpyarabot/url_utils.py | 21 +- pyproject.toml | 1 + uv.lock | 2 + 16 files changed, 1213 insertions(+), 271 deletions(-) create mode 100644 dumpyarabot/privacy.py diff --git a/dumpyarabot/aria2_manager.py b/dumpyarabot/aria2_manager.py index ace2947..9e1d656 100644 --- a/dumpyarabot/aria2_manager.py +++ b/dumpyarabot/aria2_manager.py @@ -5,13 +5,17 @@ import socket import subprocess from collections import deque +from collections.abc import AsyncIterator from dataclasses import dataclass from pathlib import Path -from collections.abc import AsyncIterator import aria2p from rich.console import Console -from dumpyarabot.process_utils import _register_process_for_current_job, _unregister_process_for_current_job + +from dumpyarabot.process_utils import ( + _register_process_for_current_job, + _unregister_process_for_current_job, +) console = Console() @@ -103,10 +107,17 @@ def _find_free_port() -> int: class Aria2Manager: """Manages an aria2c daemon and provides RPC-based download with progress tracking.""" - def __init__(self, download_dir: str, split: int = 16, max_connection_per_server: int = 16): + def __init__( + self, + download_dir: str, + split: int = 16, + max_connection_per_server: int = 16, + log_download_names: bool = True, + ): self.download_dir = Path(download_dir) self.split = split self.max_connection_per_server = max_connection_per_server + self.log_download_names = log_download_names self._process: asyncio.subprocess.Process | None = None self._stderr_task: asyncio.Task[None] | None = None self._stderr_lines: deque[str] = deque(maxlen=50) @@ -255,7 +266,8 @@ async def download( raise RuntimeError(f"Failed to add download for: {url}") gid = download.gid - console.print(f"[blue]Download added (gid={gid}): {url}[/blue]") + if self.log_download_names: + console.print(f"[blue]Download added (gid={gid}): {url}[/blue]") elapsed = 0.0 try: @@ -283,7 +295,8 @@ async def download( yield progress if progress.is_complete: - console.print(f"[green]Download complete: {file_name}[/green]") + if self.log_download_names: + console.print(f"[green]Download complete: {file_name}[/green]") return if progress.is_error: diff --git a/dumpyarabot/arq_config.py b/dumpyarabot/arq_config.py index 70442ce..12da9e0 100644 --- a/dumpyarabot/arq_config.py +++ b/dumpyarabot/arq_config.py @@ -5,6 +5,7 @@ """ import json +import logging import os import shutil import signal as _signal @@ -29,11 +30,38 @@ from rich.console import Console from dumpyarabot.config import settings +from dumpyarabot.privacy import redact_urls from dumpyarabot.schemas import JobCancelResult console = Console() +class _RedactArqArguments(logging.Filter): + """Prevent persisted firmware URLs from appearing in ARQ argument logs.""" + + def filter(self, record: logging.LogRecord) -> bool: + if isinstance(record.args, tuple): + record.args = tuple( + redact_urls(str(value), private=True) + if isinstance(value, (str, Exception)) + else value + for value in record.args + ) + elif isinstance(record.args, dict): + record.args = { + key: redact_urls(str(value), private=True) + if isinstance(value, (str, Exception)) + else value + for key, value in record.args.items() + } + if isinstance(record.msg, str): + record.msg = redact_urls(record.msg, private=True) + return True + + +logging.getLogger("arq.worker").addFilter(_RedactArqArguments()) + + def get_redis_settings(): """Parse Redis URL and return connection settings.""" parsed = urlparse(settings.REDIS_URL) diff --git a/dumpyarabot/arq_jobs.py b/dumpyarabot/arq_jobs.py index b3e269c..14dfc52 100644 --- a/dumpyarabot/arq_jobs.py +++ b/dumpyarabot/arq_jobs.py @@ -6,19 +6,23 @@ import asyncio import re +import shutil import tempfile import traceback from datetime import datetime, timezone from pathlib import Path from typing import Any, Dict, Optional -from urllib.parse import urlsplit, urlunsplit from rich.console import Console from dumpyarabot.aria2_manager import DownloadProgress from dumpyarabot.config import settings from dumpyarabot.firmware_downloader import FirmwareDownloader -from dumpyarabot.firmware_extractor import FirmwareExtractor +from dumpyarabot.firmware_extractor import ( + FirmwareExtractor, + JobCancelledError, + NativeExtractionCancelled, +) from dumpyarabot.gitlab_manager import ( GITLAB_BASE_URL, BranchAlreadyExistsError, @@ -30,6 +34,12 @@ format_download_progress, ) from dumpyarabot.message_queue import message_queue +from dumpyarabot.privacy import ( + is_private_job, + redact_for_job, + redact_urls, + sanitize_url, +) from dumpyarabot.process_utils import reset_current_job_id, set_current_job_id from dumpyarabot.property_extractor import PropertyExtractor from dumpyarabot.schemas import DumpJob @@ -43,22 +53,18 @@ re.compile(r'(token[=:]\s*)\S+', re.IGNORECASE), re.compile(r'(password[=:]\s*)\S+', re.IGNORECASE), ] -_URL_PATTERN = re.compile(r'https?://[^\s<>"\']+', re.IGNORECASE) -def _sanitize_traceback(tb_str: str) -> str: +def _sanitize_traceback(tb_str: str, *, private: bool = False) -> str: """Remove sensitive tokens and credentials from traceback strings.""" for pattern in _SENSITIVE_PATTERNS: tb_str = pattern.sub(r'\1[REDACTED]', tb_str) - return _URL_PATTERN.sub( - lambda match: _sanitize_url_for_log(match.group(0).rstrip(".,;:!?)\"]}'")), - tb_str, - ) + return redact_urls(tb_str, private=private) -def _sanitize_text(value: Any) -> str: +def _sanitize_text(value: Any, *, private: bool = False) -> str: """Sanitize arbitrary log text.""" - return _sanitize_traceback(str(value)) + return _sanitize_traceback(str(value), private=private) def _derive_last_successful_step(progress_history: list[Dict[str, Any]], failed_step: Optional[str] = None) -> Optional[str]: @@ -79,27 +85,7 @@ def _derive_last_successful_step(progress_history: list[Dict[str, Any]], failed_ def _sanitize_url_for_log(url_value: Any) -> str: """Redact credentials and query parameters from logged URLs.""" - url = str(url_value or "unknown") - try: - parts = urlsplit(url) - except ValueError: - return url - - try: - hostname = parts.hostname or "" - port = parts.port - username = parts.username - except ValueError: - return url - - netloc = hostname - if port: - netloc = f"{netloc}:{port}" - if username: - netloc = f"[REDACTED]@{netloc}" - - sanitized = urlunsplit((parts.scheme, netloc, parts.path, "", "")) - return sanitized or url + return sanitize_url(url_value) class PeriodicTimerUpdate: @@ -253,8 +239,12 @@ def _build_failure_log_text(job_data: Dict[str, Any]) -> str: lines.append(f"Job ID: {job_data.get('job_id', 'unknown')}") lines.append(f"Worker: {job_data.get('worker_id', 'unknown')}") - url = (job_data.get("dump_args") or {}).get("url", "unknown") - lines.append(f"URL: {_sanitize_url_for_log(url)}") + private = is_private_job(job_data) + if private: + lines.append("URL: [hidden for private dump]") + else: + url = (job_data.get("dump_args") or {}).get("url", "unknown") + lines.append(f"URL: {_sanitize_url_for_log(url)}") metadata = job_data.get("metadata") or {} lines.append(f"Started: {metadata.get('start_time', 'unknown')}") @@ -268,19 +258,19 @@ def _build_failure_log_text(job_data: Dict[str, Any]) -> str: pct_display = f"{float(entry.get('percentage', 0) or 0):.0f}%" except (TypeError, ValueError): pct_display = "?%" - lines.append(f"[{ts}] ({pct_display}) {_sanitize_text(msg)}") + lines.append(f"[{ts}] ({pct_display}) {redact_for_job(_sanitize_text(msg), job_data)}") error_ctx = metadata.get("error_context") or {} if error_ctx: lines.append("\n=== ERROR CONTEXT ===") - lines.append(f"Failed at: {_sanitize_text(error_ctx.get('current_step', 'unknown'))}") + lines.append(f"Failed at: {redact_for_job(_sanitize_text(error_ctx.get('current_step', 'unknown')), job_data)}") if error_ctx.get('last_successful_step'): - lines.append(f"Last successful: {_sanitize_text(error_ctx['last_successful_step'])}") - lines.append(f"Error message: {_sanitize_text(error_ctx.get('message', 'unknown'))}") + lines.append(f"Last successful: {redact_for_job(_sanitize_text(error_ctx['last_successful_step']), job_data)}") + lines.append(f"Error message: {redact_for_job(_sanitize_text(error_ctx.get('message', 'unknown')), job_data)}") tb = error_ctx.get("traceback") if tb: lines.append("\n=== TRACEBACK (sanitized) ===") - lines.append(_sanitize_traceback(tb)) + lines.append(redact_for_job(_sanitize_traceback(tb), job_data)) return "\n".join(lines) @@ -388,11 +378,19 @@ async def _send_failure_notification( caption="Failure log", ) except Exception as log_err: - console.print(f"[yellow]Could not queue failure log file: {log_err}[/yellow]") + console.print( + f"[yellow]Could not queue failure log file: " + f"{redact_urls(log_err, private=is_private_job(job_data))}[/yellow]" + ) except Exception as e: - console.print(f"[red]Failed to send failure notification: {e}[/red]") - console.print_exception() + private = is_private_job(job_data) + console.print( + f"[red]Failed to send failure notification: " + f"{redact_urls(e, private=private)}[/red]" + ) + if not private: + console.print_exception() async def _validate_gitlab_access() -> None: @@ -415,6 +413,8 @@ async def update_progress_with_metadata( ) -> None: """Helper function for progress updates with metadata tracking.""" metadata = job_data["metadata"] + private = is_private_job(job_data) + step = redact_urls(step, private=private) progress_update = { "message": step, @@ -423,7 +423,14 @@ async def update_progress_with_metadata( } if extra_info: - progress_update.update(extra_info) + progress_update.update( + { + key: redact_urls(value, private=True) + if private and isinstance(value, str) + else value + for key, value in extra_info.items() + } + ) metadata["progress_history"].append(progress_update) @@ -448,6 +455,14 @@ async def process_firmware_dump(ctx, job_data: Dict[str, Any]) -> Dict[str, Any] try: # Initialize metadata job_data["metadata"] = job_data.get("metadata", {}) + if is_private_job(job_data): + telegram_context = job_data["metadata"].get("telegram_context") or {} + telegram_context.pop("url", None) + job_data["metadata"]["telegram_context"] = telegram_context + if "_queued_text" in job_data: + job_data["_queued_text"] = redact_urls( + job_data["_queued_text"], private=True + ) job_data["metadata"].update({ "start_time": datetime.now(timezone.utc).isoformat(), "progress_history": [], @@ -469,16 +484,17 @@ async def process_firmware_dump(ctx, job_data: Dict[str, Any]) -> Dict[str, Any] try: await message_queue.verify_telegram_context(job_data) except Exception as e: - console.print(f"[red]Job {job_id}: aborting early - {e}[/red]") + safe_error = redact_for_job(e, job_data) + console.print(f"[red]Job {job_id}: aborting early - {safe_error}[/red]") job_data["metadata"].update({ "status": "failed", "end_time": datetime.now(timezone.utc).isoformat(), - "error_context": {"message": str(e), "current_step": "Telegram verification"}, + "error_context": {"message": safe_error, "current_step": "Telegram verification"}, }) # Do not queue a failure notification from this early-return path. # Preflight failures can include Telegram reachability problems or # transient Redis/bot initialization failures before normal job setup. - return {"success": False, "error": str(e), "metadata": job_data["metadata"]} + return {"success": False, "error": safe_error, "metadata": job_data["metadata"]} # Validate custom work-dir base (if configured) before creating the # per-job tempdir. Fail loudly — silent fallback to the system tempdir @@ -512,7 +528,6 @@ async def process_firmware_dump(ctx, job_data: Dict[str, Any]) -> Dict[str, Any] try: # Initialize components (exact same as original) await _raise_if_job_cancel_requested(job_id) - downloader = FirmwareDownloader(str(work_dir)) extractor = FirmwareExtractor(str(work_dir)) prop_extractor = PropertyExtractor(str(work_dir)) gitlab_manager = GitLabManager(str(work_dir)) @@ -533,12 +548,18 @@ async def process_firmware_dump(ctx, job_data: Dict[str, Any]) -> Dict[str, Any] # Create DumpJob object for components that need it dump_job = DumpJob.model_validate(job_data) + ordered_urls = [str(dump_job.dump_args.url), *map(str, dump_job.dump_args.delta_urls)] + input_root = work_dir / ".firmware_inputs" + input_root.mkdir() + downloaded_paths: list[str] = [] # Download with live progress via aria2 RPC callback. # Download progress is mapped into the 15%-50% band of overall job progress. async def _on_download_progress(dp: DownloadProgress) -> None: - dl_pct = dp.percentage # 0-100 within download - overall_pct = 15.0 + (dl_pct / 100.0) * 35.0 # map to 15%-50% + dl_pct = dp.percentage # 0-100 within this download + completed_inputs = len(downloaded_paths) + aggregate_pct = (completed_inputs + dl_pct / 100.0) / len(ordered_urls) + overall_pct = 15.0 + aggregate_pct * 35.0 dl_info = format_download_progress(dp) step_msg = f" Downloading firmware...\n{dl_info}" @@ -560,9 +581,15 @@ async def _on_download_progress(dp: DownloadProgress) -> None: "percentage": 15.0, } async with PeriodicTimerUpdate(job_data, " Downloading firmware...", download_progress): - firmware_path, firmware_name = await downloader.download_firmware( - dump_job, on_progress=_on_download_progress - ) + for index, url in enumerate(ordered_urls): + await _raise_if_job_cancel_requested(job_id) + downloader = FirmwareDownloader(str(input_root / f"input_{index:03d}")) + firmware_path, _ = await downloader.download_url( + dump_job, + url, + on_progress=_on_download_progress, + ) + downloaded_paths.append(firmware_path) # Step 5: Download completed (50%) await update_progress_with_metadata(job_data, " Firmware download completed", 50.0) @@ -572,7 +599,24 @@ async def _on_download_progress(dp: DownloadProgress) -> None: # Use periodic timer for extraction operation async with PeriodicTimerUpdate(job_data, " Extracting firmware partitions...", {"current_step": "Extract", "total_steps": 25, "current_step_number": 6, "percentage": 52.0}): - await extractor.extract_firmware(dump_job, firmware_path) + base_is_raw = await extractor.classify_raw_image_archive( + downloaded_paths[0], + cancellation_check=lambda: arq_pool.is_job_cancel_requested(job_id), + ) + if dump_job.dump_args.delta_urls or base_is_raw: + await extractor.extract_reconstructed_firmware( + dump_job, + downloaded_paths, + cancellation_check=lambda: arq_pool.is_job_cancel_requested(job_id), + base_is_raw=base_is_raw, + ) + else: + await extractor.extract_firmware(dump_job, downloaded_paths[0]) + + # Input directories live under the publication root. Remove the + # entire tree so retries/sidecars can never be committed. + if input_root.exists(): + shutil.rmtree(input_root) # Step 7: Firmware extraction completed (56%) await update_progress_with_metadata(job_data, " Firmware extraction completed", 56.0) @@ -683,20 +727,23 @@ async def _on_download_progress(dp: DownloadProgress) -> None: "repository_url": e.repo_url, "metadata": job_data["metadata"], } - except JobCancelledError as e: + except (JobCancelledError, NativeExtractionCancelled) as e: + safe_error = redact_for_job(e, job_data) job_data["metadata"].update({ "status": "cancelled", "end_time": datetime.now(timezone.utc).isoformat(), "error_context": { - "message": str(e), + "message": safe_error, "current_step": "Cancellation requested", "failure_time": datetime.now(timezone.utc).isoformat(), } }) - await _send_failure_notification(job_data, str(e)) - return {"success": False, "error": str(e), "metadata": job_data["metadata"]} + await _send_failure_notification(job_data, safe_error) + return {"success": False, "error": safe_error, "metadata": job_data["metadata"]} except Exception as e: - console.print(f"[red]Error in inner processing for job {job_id}: {e}[/red]") + private = is_private_job(job_data) + safe_error = redact_for_job(e, job_data) + console.print(f"[red]Error in inner processing for job {job_id}: {safe_error}[/red]") # Enhanced error handling metadata = job_data.get("metadata") or {} @@ -706,25 +753,28 @@ async def _on_download_progress(dp: DownloadProgress) -> None: "status": "failed", "end_time": datetime.now(timezone.utc).isoformat(), "error_context": { - "message": str(e), + "message": safe_error, "current_step": progress_history[-1].get("message", "Unknown step") if progress_history else "Unknown step", "last_successful_step": _derive_last_successful_step( progress_history, progress_history[-1].get("message") if progress_history else None, ), "failure_time": datetime.now(timezone.utc).isoformat(), - "traceback": _sanitize_traceback(traceback.format_exc()) + "traceback": redact_for_job(_sanitize_traceback(traceback.format_exc()), job_data) } }) # Send failure notification using existing message queue system - await _send_failure_notification(job_data, str(e)) + await _send_failure_notification(job_data, safe_error) - return {"success": False, "error": str(e), "metadata": job_data["metadata"]} + return {"success": False, "error": safe_error, "metadata": job_data["metadata"]} except Exception as e: - console.print(f"[red]Critical error processing job {job_id}: {e}[/red]") - console.print_exception() + private = is_private_job(job_data) + safe_error = redact_for_job(e, job_data) + console.print(f"[red]Critical error processing job {job_id}: {safe_error}[/red]") + if not private: + console.print_exception() # Enhanced error handling for critical errors metadata = job_data.get("metadata") or {} @@ -734,21 +784,24 @@ async def _on_download_progress(dp: DownloadProgress) -> None: "status": "failed", "end_time": datetime.now(timezone.utc).isoformat(), "error_context": { - "message": f"Critical error: {str(e)}", + "message": f"Critical error: {safe_error}", "current_step": "Critical failure", "last_successful_step": _derive_last_successful_step(progress_history) or "None", "failure_time": datetime.now(timezone.utc).isoformat(), - "traceback": _sanitize_traceback(traceback.format_exc()) + "traceback": redact_for_job(_sanitize_traceback(traceback.format_exc()), job_data) } }) # Send failure notification for any unhandled exceptions try: - await _send_failure_notification(job_data, f"Critical error: {str(e)}") + await _send_failure_notification(job_data, f"Critical error: {safe_error}") except Exception as notification_error: - console.print(f"[red]Failed to send failure notification: {notification_error}[/red]") + console.print( + f"[red]Failed to send failure notification: " + f"{redact_urls(notification_error, private=private)}[/red]" + ) - return {"success": False, "error": str(e), "metadata": job_data["metadata"]} + return {"success": False, "error": safe_error, "metadata": job_data["metadata"]} finally: if job_token is not None: reset_current_job_id(job_token) @@ -758,10 +811,6 @@ async def _on_download_progress(dp: DownloadProgress) -> None: # teardown race that prompted moving to a hook in the first place. -class JobCancelledError(Exception): - """Raised when a cooperative cancellation request is detected.""" - - async def _raise_if_job_cancel_requested(job_id: str) -> None: """Abort the current job if a cooperative cancellation was requested.""" from dumpyarabot.arq_config import arq_pool diff --git a/dumpyarabot/firmware_downloader.py b/dumpyarabot/firmware_downloader.py index 582dc1d..b18b88b 100644 --- a/dumpyarabot/firmware_downloader.py +++ b/dumpyarabot/firmware_downloader.py @@ -1,7 +1,8 @@ +import asyncio import os import shutil -from pathlib import Path from collections.abc import Callable, Coroutine +from pathlib import Path from typing import Tuple from urllib.parse import urlparse @@ -9,9 +10,14 @@ from rich.console import Console from dumpyarabot.aria2_manager import Aria2Manager, DownloadProgress -from dumpyarabot.schemas import DumpJob +from dumpyarabot.file_utils import ( + get_file_size_formatted, + get_latest_file_in_directory, + safe_remove_file, +) +from dumpyarabot.privacy import redact_urls, sanitize_url from dumpyarabot.process_utils import run_download_command -from dumpyarabot.file_utils import get_latest_file_in_directory, safe_remove_file, get_file_size_formatted +from dumpyarabot.schemas import DumpJob console = Console() @@ -38,33 +44,58 @@ async def download_firmware( on_progress: Optional async callback invoked with each DownloadProgress snapshot during aria2 RPC downloads. """ - url = str(job.dump_args.url) - - # Check if it's a local file - if os.path.isfile(url): - console.print(f"[green]Found local file: {url}[/green]") - # Copy to work directory - file_name = Path(url).name - dest_path = self.work_dir / file_name - shutil.copy2(url, dest_path) - return str(dest_path), file_name + return await self.download_url( + job, + str(job.dump_args.url), + on_progress=on_progress, + ) - # Optimize URL with mirrors - optimized_url = await self._optimize_url(url) - console.print(f"[blue]Downloading from: {optimized_url}[/blue]") + async def download_url( + self, + job: DumpJob, + url: str, + on_progress: ProgressCallback | None = None, + ) -> Tuple[str, str]: + """Download one ordered input without exposing private source details.""" + private = job.dump_args.use_privdump - # Download based on URL type - file_path = await self._download_by_type(optimized_url, on_progress=on_progress) - file_name = Path(file_path).name + try: + # Check if it's a local file + if os.path.isfile(url): + if not private: + console.print(f"[green]Found local file: {url}[/green]") + file_name = Path(url).name + dest_path = self.work_dir / file_name + shutil.copy2(url, dest_path) + return str(dest_path), file_name + + # Optimize URL with mirrors + optimized_url = await self._optimize_url(url, private=private) + if not private: + console.print(f"[blue]Downloading from: {sanitize_url(optimized_url)}[/blue]") + + file_path = await self._download_by_type( + optimized_url, + on_progress=on_progress, + private=private, + ) + file_name = Path(file_path).name - console.print(f"[green]Downloaded: {file_name} ({get_file_size_formatted(file_path)})[/green]") - return file_path, file_name + if not private: + console.print(f"[green]Downloaded: {file_name} ({get_file_size_formatted(file_path)})[/green]") + return file_path, file_name + except asyncio.CancelledError: + raise + except Exception as e: + if private: + raise RuntimeError("Private firmware download failed") from None + raise e - async def _optimize_url(self, url: str) -> str: + async def _optimize_url(self, url: str, *, private: bool = False) -> str: """Optimize URL with best available mirrors.""" # Xiaomi mirror optimization if "d.miui.com" in url: - return await self._optimize_xiaomi_url(url) + return await self._optimize_xiaomi_url(url, private=private) # Pixeldrain optimization if "pixeldrain.com/u" in url: @@ -77,7 +108,7 @@ async def _optimize_url(self, url: str) -> str: return url - async def _optimize_xiaomi_url(self, url: str) -> str: + async def _optimize_xiaomi_url(self, url: str, *, private: bool = False) -> str: """Find best Xiaomi mirror.""" # Skip if already using recommended mirror if "cdnorg" in url or "bkt-sgp-miui-ota-update-alisgp" in url: @@ -109,38 +140,49 @@ async def _optimize_xiaomi_url(self, url: str) -> str: for mirror in mirrors: test_url = f"{mirror}/{file_path}" try: - console.print(f"[blue]Testing mirror: {mirror}[/blue]") + if not private: + console.print(f"[blue]Testing mirror: {mirror}[/blue]") response = await client.head(test_url, timeout=10.0) if response.status_code != 404: - console.print(f"[green]Using mirror: {mirror}[/green]") + if not private: + console.print(f"[green]Using mirror: {mirror}[/green]") return test_url except Exception as e: - console.print(f"[yellow]Mirror {mirror} failed: {e}[/yellow]") + if not private: + console.print( + f"[yellow]Mirror {mirror} failed: " + f"{redact_urls(e, private=False)}[/yellow]" + ) continue console.print("[yellow]All mirrors failed, using original URL[/yellow]") return url async def _download_by_type( - self, url: str, on_progress: ProgressCallback | None = None + self, + url: str, + on_progress: ProgressCallback | None = None, + *, + private: bool = False, ) -> str: """Download file based on URL type.""" if "drive.google.com" in url: - return await self._download_google_drive(url) + return await self._download_google_drive(url, private=private) elif "mediafire.com" in url: - return await self._download_mediafire(url) + return await self._download_mediafire(url, private=private) elif "mega.nz" in url: - return await self._download_mega(url) + return await self._download_mega(url, private=private) else: - return await self._download_default(url, on_progress=on_progress) + return await self._download_default(url, on_progress=on_progress, private=private) - async def _download_google_drive(self, url: str) -> str: + async def _download_google_drive(self, url: str, *, private: bool = False) -> str: """Download from Google Drive using gdown.""" result = await run_download_command( "uvx", "gdown@5.2.0", "-q", url, "--fuzzy", cwd=self.work_dir, timeout=1800.0, # 30 minutes for large files - description="Downloading from Google Drive" + description="Downloading from Google Drive", + quiet=True, ) if not result.success: @@ -153,14 +195,15 @@ async def _download_google_drive(self, url: str) -> str: return str(latest_file) - async def _download_mediafire(self, url: str) -> str: + async def _download_mediafire(self, url: str, *, private: bool = False) -> str: """Download from MediaFire using mediafire-dl.""" result = await run_download_command( "uvx", "--from", "git+https://github.com/Juvenal-Yescas/mediafire-dl@master", "mediafire-dl", url, cwd=self.work_dir, timeout=1800.0, # 30 minutes for large files - description="Downloading from MediaFire" + description="Downloading from MediaFire", + quiet=True, ) if not result.success: @@ -173,13 +216,14 @@ async def _download_mediafire(self, url: str) -> str: return str(latest_file) - async def _download_mega(self, url: str) -> str: + async def _download_mega(self, url: str, *, private: bool = False) -> str: """Download from MEGA using megatools.""" result = await run_download_command( "megatools", "dl", url, cwd=self.work_dir, timeout=1800.0, # 30 minutes for large files - description="Downloading from MEGA" + description="Downloading from MEGA", + quiet=True, ) if not result.success: @@ -193,21 +237,28 @@ async def _download_mega(self, url: str) -> str: return str(latest_file) async def _download_default( - self, url: str, on_progress: ProgressCallback | None = None + self, + url: str, + on_progress: ProgressCallback | None = None, + *, + private: bool = False, ) -> str: """Download using aria2 RPC (with live progress) and wget fallback.""" # --- Try aria2 RPC first --- aria2_failed = False aria2_error = "" try: - async with Aria2Manager(str(self.work_dir)) as aria2: + async with Aria2Manager( + str(self.work_dir), log_download_names=False + ) as aria2: async for progress in aria2.download(url, poll_interval=3.0, timeout=1800.0): if on_progress: try: await on_progress(progress) except Exception as cb_err: # Don't let a Telegram/callback error kill the download - console.print(f"[yellow]Progress callback error (ignored): {cb_err}[/yellow]") + if not private: + console.print(f"[yellow]Progress callback error (ignored): {cb_err}[/yellow]") # Download finished successfully downloaded = aria2.get_downloaded_file_path() @@ -222,7 +273,11 @@ async def _download_default( # Keep the actual aria2 reason (error code + message); the console line # only reaches journald, so we also stash it for the final exception. aria2_error = str(e) or repr(e) - console.print(f"[yellow]aria2 RPC download failed: {aria2_error}[/yellow]") + if not private: + console.print( + f"[yellow]aria2 RPC download failed: " + f"{redact_urls(aria2_error, private=False)}[/yellow]" + ) aria2_failed = True if not aria2_failed: @@ -242,7 +297,8 @@ async def _download_default( "wget", "-nv", "--no-check-certificate", url, cwd=self.work_dir, timeout=1800.0, - description="Downloading with wget fallback" + description="Downloading with wget fallback", + quiet=True, ) if not result.success: @@ -261,5 +317,3 @@ async def _download_default( ) return str(latest_file) - - diff --git a/dumpyarabot/firmware_extractor.py b/dumpyarabot/firmware_extractor.py index ac07522..96ddd74 100644 --- a/dumpyarabot/firmware_extractor.py +++ b/dumpyarabot/firmware_extractor.py @@ -1,8 +1,20 @@ import asyncio -from pathlib import Path - +import re +import shutil +import stat +import tarfile +import zipfile +from collections.abc import Awaitable, Callable, Sequence +from pathlib import Path, PurePosixPath +from typing import Any + +import otadump +import py7zr from dumpyara.dumpyara import dumpyara +from dumpyara.steps.extract_images import extract_images as dumpyara_extract_images from dumpyara.utils import multipartitions as dumpyara_multipartitions +from dumpyara.utils.partitions import get_partition_names +from py7zr.io import Py7zIO, WriterFactory from rich.console import Console from dumpyarabot.file_utils import ( @@ -25,19 +37,565 @@ def _run_dumpyara(firmware_path: Path, output_path: Path) -> None: """Run Dumpyara with its reliable in-process payload parser.""" dumpyara_multipartitions.OTADUMP_EXECUTABLE = None + dumpyara_multipartitions.extract_payload_native = None dumpyara(firmware_path, output_path) +def _run_dumpyara_images(images_path: Path, output_path: Path) -> None: + """Run dumpyara's existing filesystem-image extraction step once.""" + dumpyara_extract_images(images_path, output_path) + system_path = output_path / "system" + if not system_path.exists() or not any(system_path.iterdir()): + raise RuntimeError("System filesystem extraction did not produce files") + + +class JobCancelledError(Exception): + """Raised after cooperative native extraction has fully stopped.""" + + +class NativeExtractionCancelled(JobCancelledError): + """Internal form of otadump's cooperative KeyboardInterrupt.""" + + +CancellationCheck = Callable[[], Awaitable[bool]] +ANDROID_SPARSE_MAGIC = b"\x3a\xff\x26\xed" + + +async def _check_cancelled(cancellation_check: CancellationCheck | None) -> bool: + """Safely poll cooperative cancellation without failing on callback errors.""" + if cancellation_check is None: + return False + try: + return await cancellation_check() + except Exception as e: + console.print(f"[yellow]Cancellation check error (ignored): {e}[/yellow]") + return False + + +class _ArchiveImageIO(Py7zIO): + def __init__(self, path: Path): + self._file = path.open("x+b") + + def write(self, data: bytes | bytearray) -> int: + return self._file.write(data) + + def read(self, size: int | None = None) -> bytes: + return self._file.read(-1 if size is None else size) + + def seek(self, offset: int, whence: int = 0) -> int: + return self._file.seek(offset, whence) + + def flush(self) -> None: + self._file.flush() + + def size(self) -> int: + position = self._file.tell() + self._file.seek(0, 2) + size = self._file.tell() + self._file.seek(position) + return size + + def close(self) -> None: + self._file.close() + + +class _ArchiveImageFactory(WriterFactory): + def __init__(self, destination: Path): + self.destination = destination + + def create(self, filename: str) -> Py7zIO: + name = _safe_archive_member(filename).name + return _ArchiveImageIO(self.destination / name) + + +class _ImageMagicIO(Py7zIO): + def __init__(self): + self.magic = bytearray() + self.position = 0 + + def write(self, data: bytes | bytearray) -> int: + if len(self.magic) < 4: + self.magic.extend(data[: 4 - len(self.magic)]) + self.position += len(data) + return len(data) + + def read(self, size: int | None = None) -> bytes: + return b"" + + def seek(self, offset: int, whence: int = 0) -> int: + if whence == 0: + self.position = offset + elif whence == 1: + self.position += offset + return self.position + + def flush(self) -> None: + return None + + def size(self) -> int: + return self.position + + +class _ImageMagicFactory(WriterFactory): + def __init__(self): + self.outputs: list[_ImageMagicIO] = [] + + def create(self, filename: str) -> Py7zIO: + output = _ImageMagicIO() + self.outputs.append(output) + return output + + +def _safe_archive_member(name: str) -> PurePosixPath: + normalized = name.replace("\\", "/") + path = PurePosixPath(normalized) + if ( + not normalized + or normalized.startswith("/") + or re.match(r"^[A-Za-z]:", normalized) + or path.is_absolute() + or ".." in path.parts + ): + raise ValueError("Archive contains an unsafe path") + return path + + +def _raw_image_names_are_ready(names: Sequence[str]) -> bool: + """Exclude containers and slot layouts that need dumpyara preparation.""" + stems = [PurePosixPath(name.replace("\\", "/")).stem for name in names] + if not stems or any( + stem == "super" or stem.endswith(("_a", "_b")) for stem in stems + ): + return False + supported_partitions = set(get_partition_names()) + return any(stem in supported_partitions for stem in stems) + + +def _is_raw_archive_ancillary(name: str) -> bool: + path = _safe_archive_member(name) + return path.suffix.lower() in {".md", ".sha256", ".txt"} or path.name.upper() in { + "README", + "SHA256SUMS", + } + + +def _unpack_raw_image_archive(archive_path: Path, destination: Path) -> None: + """Validate and flatten regular .img members without shelling out.""" + destination.mkdir(parents=True, exist_ok=False) + seen: set[str] = set() + + if zipfile.is_zipfile(archive_path): + with zipfile.ZipFile(archive_path) as archive: + zip_image_members: list[zipfile.ZipInfo] = [] + for info in archive.infolist(): + member_path = _safe_archive_member(info.filename) + mode = info.external_attr >> 16 + file_type = stat.S_IFMT(mode) + if stat.S_ISLNK(mode) or file_type not in { + 0, + stat.S_IFREG, + stat.S_IFDIR, + }: + raise ValueError("Archive contains a link or special file") + if info.is_dir(): + continue + if member_path.suffix.lower() == ".img": + key = member_path.name.casefold() + if key in seen: + raise ValueError( + f"Archive contains duplicate partition image basename: {member_path.name}" + ) + seen.add(key) + zip_image_members.append(info) + for info in zip_image_members: + name = PurePosixPath(info.filename.replace("\\", "/")).name + with archive.open(info) as source, (destination / name).open("xb") as target: + shutil.copyfileobj(source, target) + + elif tarfile.is_tarfile(archive_path): + with tarfile.open(archive_path, mode="r:*") as archive: + tar_image_members: list[tarfile.TarInfo] = [] + for member in archive.getmembers(): + member_path = _safe_archive_member(member.name) + if member.isdir(): + continue + if not member.isreg(): + raise ValueError("Archive contains a link or special file") + if member_path.suffix.lower() == ".img": + key = member_path.name.casefold() + if key in seen: + raise ValueError( + f"Archive contains duplicate partition image basename: {member_path.name}" + ) + seen.add(key) + tar_image_members.append(member) + for member in tar_image_members: + extracted = archive.extractfile(member) + if extracted is None: + raise ValueError("Could not read archive image entry") + name = PurePosixPath(member.name.replace("\\", "/")).name + with extracted, (destination / name).open("xb") as target: + shutil.copyfileobj(extracted, target) + + elif archive_path.name.lower().endswith(".7z"): + with py7zr.SevenZipFile(archive_path, mode="r") as archive: + image_names: list[str] = [] + for info in archive.list(): + member_path = _safe_archive_member(info.filename) + if info.is_directory: + continue + if info.is_symlink or not info.is_file: + raise ValueError("Archive contains a link or special file") + if member_path.suffix.lower() == ".img": + key = member_path.name.casefold() + if key in seen: + raise ValueError( + f"Archive contains duplicate partition image basename: {member_path.name}" + ) + seen.add(key) + image_names.append(info.filename) + + archive.extract( + targets=image_names, + factory=_ArchiveImageFactory(destination), + ) + else: + raise ValueError("Base input is not a supported raw-image archive") + + if not seen: + raise ValueError("Raw-image archive contains no .img files") + + +def _carry_forward_omitted_images(current_stage: Path, next_stage: Path) -> None: + """Copy unchanged partition bytes into a successfully extracted next stage.""" + for source_image in current_stage.glob("*.img"): + destination = next_stage / source_image.name + if not destination.exists(): + shutil.copy2(source_image, destination) + + class FirmwareExtractor: """Handles firmware extraction using both Python dumper and alternative methods.""" def __init__(self, work_dir: str): self.work_dir = Path(work_dir) self.firmware_extractor_path = Path.home() / "Firmware_extractor" + self._native_tasks: set[asyncio.Task[None]] = set() + + @staticmethod + def is_raw_image_archive(firmware_path: str) -> bool: + """Identify extraction-ready raw images while preserving legacy containers.""" + path = Path(firmware_path) + if zipfile.is_zipfile(path): + with zipfile.ZipFile(path) as archive: + regular_files = [info for info in archive.infolist() if not info.is_dir()] + if any( + PurePosixPath(info.filename.replace("\\", "/")).name + == "payload.bin" + for info in regular_files + ): + return False + image_files = [ + info + for info in regular_files + if _safe_archive_member(info.filename).suffix.lower() == ".img" + ] + if any( + info not in image_files + and not _is_raw_archive_ancillary(info.filename) + for info in regular_files + ): + return False + return _raw_image_names_are_ready( + [info.filename for info in image_files] + ) and all( + archive.open(info).read(4) != ANDROID_SPARSE_MAGIC + for info in image_files + ) + if tarfile.is_tarfile(path): + with tarfile.open(path, mode="r:*") as archive: + regular_files = [ + member for member in archive.getmembers() if member.isfile() + ] + image_files = [ + member + for member in regular_files + if _safe_archive_member(member.name).suffix.lower() == ".img" + ] + if any( + member not in image_files + and not _is_raw_archive_ancillary(member.name) + for member in regular_files + ): + return False + if not _raw_image_names_are_ready( + [member.name for member in image_files] + ): + return False + for member in image_files: + extracted = archive.extractfile(member) + if extracted is None or extracted.read(4) == ANDROID_SPARSE_MAGIC: + return False + return True + if path.name.lower().endswith(".7z"): + with py7zr.SevenZipFile(path, mode="r") as archive: + regular_files = [info for info in archive.list() if info.is_file] + image_names = [ + info.filename + for info in regular_files + if _safe_archive_member(info.filename).suffix.lower() == ".img" + ] + if any( + info.filename not in image_names + and not _is_raw_archive_ancillary(info.filename) + for info in regular_files + ): + return False + if not _raw_image_names_are_ready(image_names): + return False + magic_factory = _ImageMagicFactory() + archive.extract(targets=image_names, factory=magic_factory) + return all( + bytes(output.magic) != ANDROID_SPARSE_MAGIC + for output in magic_factory.outputs + ) + return False + + @staticmethod + def _zip_contains_payload(firmware_path: str) -> bool: + if not zipfile.is_zipfile(firmware_path): + return False + with zipfile.ZipFile(firmware_path) as archive: + return any( + PurePosixPath(name.replace("\\", "/")).name == "payload.bin" + for name in archive.namelist() + ) + + async def _drain_task(self, task: asyncio.Task[Any]) -> BaseException | None: + """Wait until a shielded worker really exits, despite caller cancellation.""" + while not task.done(): + try: + await asyncio.shield(task) + except asyncio.CancelledError: + continue + except BaseException: + if not task.done(): + continue + break + try: + task.result() + except BaseException as error: + return error + return None + + async def classify_raw_image_archive( + self, + firmware_path: str, + *, + cancellation_check: CancellationCheck | None = None, + ) -> bool: + """Classify potentially large archives off-loop and drain on cancellation.""" + worker = asyncio.create_task( + asyncio.to_thread(self.is_raw_image_archive, firmware_path) + ) + self._native_tasks.add(worker) + try: + while not worker.done(): + if await _check_cancelled(cancellation_check): + await self._drain_task(worker) + raise JobCancelledError("Job was cancelled") + await asyncio.wait({worker}, timeout=0.25) + error = await self._drain_task(worker) + if error: + raise error + return worker.result() + except asyncio.CancelledError: + await self._drain_task(worker) + raise + except BaseException: + if not worker.done(): + await self._drain_task(worker) + raise + finally: + self._native_tasks.discard(worker) + + async def _run_otadump( + self, + payload_path: Path, + output_dir: Path, + *, + source_dir: Path | None = None, + cancellation_check: CancellationCheck | None = None, + timeout: float = ONE_HOUR, + ) -> None: + """Run one native call with cooperative cancellation and mandatory drain.""" + token = otadump.CancellationToken() + + def run() -> None: + try: + otadump.extract( + payload_path, + output_dir, + source_dir=source_dir, + cancellation_token=token, + ) + except KeyboardInterrupt as error: + raise NativeExtractionCancelled( + "Native OTA extraction was cancelled" + ) from error + + worker = asyncio.create_task(asyncio.to_thread(run)) + self._native_tasks.add(worker) + deadline = asyncio.get_running_loop().time() + timeout + try: + while not worker.done(): + if await _check_cancelled(cancellation_check): + token.cancel() + await self._drain_task(worker) + raise JobCancelledError("Job was cancelled") + + remaining = deadline - asyncio.get_running_loop().time() + if remaining <= 0: + token.cancel() + await self._drain_task(worker) + raise TimeoutError("Native OTA extraction timed out") + await asyncio.wait({worker}, timeout=min(0.25, remaining)) + + error = await self._drain_task(worker) + if isinstance(error, (NativeExtractionCancelled, JobCancelledError)): + raise JobCancelledError("Job was cancelled") from error + if error: + raise error + except asyncio.CancelledError: + token.cancel() + await self._drain_task(worker) + raise + except (JobCancelledError, TimeoutError): + if not worker.done(): + token.cancel() + await self._drain_task(worker) + raise + except BaseException: + if not worker.done(): + token.cancel() + await self._drain_task(worker) + raise + finally: + self._native_tasks.discard(worker) + + async def _run_blocking_and_drain( + self, + function: Callable[..., None], + *args: object, + timeout: float = ONE_HOUR, + ) -> None: + """Do not let a non-native extraction thread outlive its work directory.""" + worker = asyncio.create_task(asyncio.to_thread(function, *args)) + self._native_tasks.add(worker) + deadline = asyncio.get_running_loop().time() + timeout + try: + while not worker.done(): + remaining = deadline - asyncio.get_running_loop().time() + if remaining <= 0: + await self._drain_task(worker) + raise TimeoutError("Extraction step timed out") + await asyncio.wait({worker}, timeout=min(0.25, remaining)) + + error = await self._drain_task(worker) + if error: + raise error + except asyncio.CancelledError: + await self._drain_task(worker) + raise + except TimeoutError: + if not worker.done(): + await self._drain_task(worker) + raise + except BaseException: + if not worker.done(): + await self._drain_task(worker) + raise + finally: + self._native_tasks.discard(worker) + + async def extract_reconstructed_firmware( + self, + job: DumpJob, + firmware_paths: Sequence[str], + *, + cancellation_check: CancellationCheck | None = None, + base_is_raw: bool | None = None, + ) -> str: + """Build ordered final images, then extract filesystems exactly once.""" + if not firmware_paths: + raise ValueError("No firmware inputs were downloaded") + + if job.dump_args.use_alt_dumper: + raise ValueError( + "Alternative dumper is not supported for delta OTA or raw-image reconstruction" + ) + + staging_root = self.work_dir / ".ota_staging" + shutil.rmtree(staging_root, ignore_errors=True) + staging_root.mkdir() + try: + base_path = Path(firmware_paths[0]) + source_dir = staging_root / "stage_000" + if base_is_raw is None: + base_is_raw = await self.classify_raw_image_archive( + str(base_path), cancellation_check=cancellation_check + ) + if base_is_raw: + await self._run_blocking_and_drain( + _unpack_raw_image_archive, base_path, source_dir + ) + elif self._zip_contains_payload(str(base_path)): + source_dir.mkdir() + await self._run_otadump( + base_path, + source_dir, + cancellation_check=cancellation_check, + ) + else: + raise ValueError( + "Delta reconstruction requires a full OTA or raw-image archive base" + ) + + current_stage = source_dir + for index, delta_path_value in enumerate(firmware_paths[1:], start=1): + next_stage = staging_root / f"stage_{index:03d}" + next_stage.mkdir() + await self._run_otadump( + Path(delta_path_value), + next_stage, + source_dir=current_stage, + cancellation_check=cancellation_check, + ) + + # A payload may omit unchanged partitions. Copy their bytes into + # the completed next stage; never hardlink or mutate source images. + await self._run_blocking_and_drain( + _carry_forward_omitted_images, current_stage, next_stage + ) + shutil.rmtree(current_stage) + current_stage = next_stage + + if await _check_cancelled(cancellation_check): + raise JobCancelledError("Job was cancelled") + + await self._run_blocking_and_drain( + _run_dumpyara_images, current_stage, self.work_dir + ) + return str(self.work_dir) + finally: + shutil.rmtree(staging_root, ignore_errors=True) + for firmware_path in firmware_paths: + safe_remove_file(firmware_path) async def extract_firmware(self, job: DumpJob, firmware_path: str) -> str: """Extract firmware and return extraction directory.""" - console.print(f"[blue]Extracting firmware: {firmware_path}[/blue]") + if not job.dump_args.use_privdump: + console.print(f"[blue]Extracting firmware: {firmware_path}[/blue]") if job.dump_args.use_alt_dumper: extraction_dir = await self._extract_with_alternative_dumper(firmware_path) @@ -49,22 +607,21 @@ async def extract_firmware(self, job: DumpJob, firmware_path: str) -> str: # extract_and_push.sh; without it every dump shipped its multi-GB source # archive at the repo root. if safe_remove_file(firmware_path): - console.print(f"[green]Removed original firmware archive: {firmware_path}[/green]") + if not job.dump_args.use_privdump: + console.print(f"[green]Removed original firmware archive: {firmware_path}[/green]") else: - console.print(f"[yellow]Failed to remove original firmware archive: {firmware_path}[/yellow]") + if not job.dump_args.use_privdump: + console.print(f"[yellow]Failed to remove original firmware archive: {firmware_path}[/yellow]") return extraction_dir async def _extract_with_python_dumper(self, firmware_path: str) -> str: """Extract using the modern Python dumpyara tool.""" console.print("[blue]Python dumper extraction...[/blue]") - await asyncio.wait_for( - asyncio.to_thread( - _run_dumpyara, - Path(firmware_path), - self.work_dir, - ), - timeout=ONE_HOUR, + await self._run_blocking_and_drain( + _run_dumpyara, + Path(firmware_path), + self.work_dir, ) console.print("[green]Python dumper extraction completed successfully[/green]") diff --git a/dumpyarabot/handlers.py b/dumpyarabot/handlers.py index 4857a07..7442a98 100644 --- a/dumpyarabot/handlers.py +++ b/dumpyarabot/handlers.py @@ -7,13 +7,14 @@ from telegram.error import BadRequest, NetworkError from telegram.ext import ContextTypes -from dumpyarabot import schemas, utils, url_utils -from dumpyarabot.utils import escape_markdown -from dumpyarabot.config import settings +from dumpyarabot import schemas, url_utils, utils from dumpyarabot.auth import VERIFICATION_FAILED_MESSAGE, check_admin_permissions +from dumpyarabot.config import settings +from dumpyarabot.message_formatting import format_firmware_inputs, generate_progress_bar from dumpyarabot.message_queue import message_queue -from dumpyarabot.message_formatting import generate_progress_bar +from dumpyarabot.privacy import redact_for_job, redact_urls, sanitize_url from dumpyarabot.schemas import JobCancelResult +from dumpyarabot.utils import escape_markdown console = Console() @@ -88,7 +89,7 @@ async def dump( # Ensure that we had some arguments passed if not context.args: console.print("[yellow]No arguments provided for dump command[/yellow]") - usage = "Usage: `/dump [URL] [a|f|p]`\nURL: required, a: alt dumper, f: force, p: use privdump" + usage = "Usage: `/dump BASE [DELTA ...] [a|f|p]`\nAt least one URL is required; a: alt dumper, f: force, p: use privdump" await message_queue.send_reply( chat_id=chat.id, text=usage, @@ -97,15 +98,25 @@ async def dump( ) return - url = context.args[0] - options = "".join("".join(context.args[1:]).split()) + try: + ordered_urls, options = url_utils.parse_dump_tokens(list(context.args)) + except ValueError as e: + await message_queue.send_reply( + chat_id=chat.id, + text=str(e), + reply_to_message_id=message.message_id, + context={"command": "dump", "error": "missing_urls"}, + ) + return use_alt_dumper = "a" in options force = "f" in options use_privdump = "p" in options console.print("[green]Dump request:[/green]") - console.print(f" URL: {url}") + if not use_privdump: + console.print(f" Base URL: {sanitize_url(ordered_urls[0])}") + console.print(f" Delta URLs: {len(ordered_urls) - 1}") console.print(f" Alt dumper: {use_alt_dumper}") console.print(f" Force: {force}") console.print(f" Privdump: {use_privdump}") @@ -123,17 +134,23 @@ async def dump( "[green]Successfully deleted original message for privdump[/green]" ) except Exception as e: - console.print(f"[red]Failed to delete message for privdump: {e}[/red]") + console.print( + f"[red]Failed to delete message for privdump: " + f"{redact_urls(e, private=True)}[/red]" + ) # Try to validate args and queue dump job try: - # Validate URL using new utility - is_valid, normalized_url, error_msg = await url_utils.validate_and_normalize_url(url) - if not is_valid: - raise ValueError(error_msg) + normalized_urls = [] + for candidate in ordered_urls: + is_valid, normalized_url, error_msg = await url_utils.validate_and_normalize_url(candidate) + if not is_valid or normalized_url is None: + raise ValueError(error_msg) + normalized_urls.append(normalized_url) dump_args = schemas.DumpArguments( - url=normalized_url, + url=normalized_urls[0], + delta_urls=normalized_urls[1:], use_alt_dumper=use_alt_dumper, force=force, use_privdump=use_privdump, @@ -154,7 +171,8 @@ async def dump( if use_privdump: initial_text = " *Private Dump Job Queued*\n\n" else: - initial_text = f" *Firmware Dump Queued*\n\n *URL:* `{url}`\n" + initial_text = " *Firmware Dump Queued*\n\n" + initial_text += format_firmware_inputs(dump_args.model_dump()) initial_text += f"*Job ID:* `{job.job_id}`\n" @@ -202,14 +220,12 @@ async def dump( enhanced_job_data = job.model_dump() # Store initial text so the worker can re-edit it during Telegram context verification enhanced_job_data["_queued_text"] = initial_text - enhanced_job_data["metadata"] = { - "telegram_context": { + telegram_context = { "chat_id": chat.id, "message_id": initial_message_id, "user_id": message.from_user.id if message.from_user else 0, - "url": normalized_url - } } + enhanced_job_data["metadata"] = {"telegram_context": telegram_context} # Queue the job with enhanced data job_id = await message_queue.queue_dump_job_with_metadata(enhanced_job_data) @@ -217,21 +233,34 @@ async def dump( console.print(f"[green]Dump job {job_id} queued with enhanced metadata[/green]") except ValueError as e: - console.print(f"[red]Invalid URL provided: {url} - {e}[/red]") - response_text = f" *Invalid URL:* {url}\n\nPlease provide a valid firmware download URL." + if use_privdump: + console.print("[red]Invalid private firmware URL provided[/red]") + response_text = " *Invalid URL*\n\nPlease provide valid firmware download URLs." + error_context = {"command": "dump", "error": "validation_error"} + else: + safe_error = redact_urls(e, private=False) + console.print(f"[red]Invalid URL provided: {safe_error}[/red]") + response_text = " *Invalid URL*\n\nPlease provide valid firmware download URLs." + error_context = {"command": "dump", "error": "validation_error"} # Send error message as reply await message_queue.send_reply( chat_id=chat.id, text=response_text, reply_to_message_id=None if use_privdump else message.message_id, - context={"command": "dump", "url": url, "error": "validation_error"} + context=error_context, ) except Exception as e: - console.print(f"[red]Unexpected error occurred: {e}[/red]") - console.print_exception() - escaped_error = escape_markdown(str(e)) + safe_error = ( + redact_for_job(e, dump_args) + if use_privdump and "dump_args" in locals() + else redact_urls(e, private=use_privdump) + ) + console.print(f"[red]Unexpected error occurred: {safe_error}[/red]") + if not use_privdump: + console.print_exception() + escaped_error = escape_markdown(safe_error) response_text = f" *Error occurred:* {escaped_error}\n\nPlease try again or contact an administrator." # Send error message as reply @@ -239,7 +268,7 @@ async def dump( chat_id=chat.id, text=response_text, reply_to_message_id=None if use_privdump else message.message_id, - context={"command": "dump", "url": url, "error": "unexpected_error"} + context={"command": "dump", "error": "unexpected_error"}, ) @@ -488,7 +517,8 @@ async def restart(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None: # Create confirmation keyboard from telegram import InlineKeyboardButton, InlineKeyboardMarkup - from dumpyarabot.config import CALLBACK_RESTART_CONFIRM, CALLBACK_RESTART_CANCEL + + from dumpyarabot.config import CALLBACK_RESTART_CANCEL, CALLBACK_RESTART_CONFIRM keyboard = [ [ @@ -525,7 +555,7 @@ async def restart(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None: } # Create a custom queued message for restart confirmation - from dumpyarabot.message_queue import QueuedMessage, MessageType, MessagePriority + from dumpyarabot.message_queue import MessagePriority, MessageType, QueuedMessage restart_message = QueuedMessage( type=MessageType.NOTIFICATION, priority=MessagePriority.URGENT, @@ -601,7 +631,7 @@ async def handle_restart_callback(update: Update, context: ContextTypes.DEFAULT_ await query.answer() - from dumpyarabot.config import CALLBACK_RESTART_CONFIRM, CALLBACK_RESTART_CANCEL + from dumpyarabot.config import CALLBACK_RESTART_CANCEL, CALLBACK_RESTART_CONFIRM if query.data.startswith(CALLBACK_RESTART_CONFIRM): # Extract user ID from callback data diff --git a/dumpyarabot/message_formatting.py b/dumpyarabot/message_formatting.py index dbe35ba..fdbaf50 100644 --- a/dumpyarabot/message_formatting.py +++ b/dumpyarabot/message_formatting.py @@ -3,12 +3,14 @@ from __future__ import annotations from datetime import datetime, timezone -from typing import Dict, Any, Optional, List, TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Dict, List, Optional +from dumpyarabot.privacy import redact_urls, sanitize_url from dumpyarabot.utils import escape_markdown if TYPE_CHECKING: from dumpyarabot.aria2_manager import DownloadProgress + from dumpyarabot.schemas import DumpJob async def get_arq_start_time(arq_job_id: str) -> Optional[str]: @@ -208,7 +210,7 @@ def calculate_elapsed_time( return "0s" -def format_url_display(url: str, max_length: int = 60) -> str: +def format_url_display(url: Any, max_length: int = 60) -> str: """ Format URL for display, truncating if too long. @@ -219,7 +221,7 @@ def format_url_display(url: str, max_length: int = 60) -> str: Returns: Formatted URL string """ - url_str = str(url) + url_str = sanitize_url(url) if len(url_str) > max_length: return url_str[:max_length - 3] + "..." return url_str @@ -250,6 +252,21 @@ def format_dump_options(dump_args: Dict[str, Any], add_blacklist: bool = False) return options +def format_firmware_inputs(dump_args: Dict[str, Any]) -> str: + """Format the persistence-compatible base URL and ordered delta list.""" + if dump_args.get("use_privdump"): + return " *URL:* `[hidden for private dump]`\n" + + base = format_url_display(dump_args["url"]) + lines = [f" *Base URL:* `{base}`"] + delta_urls = dump_args.get("delta_urls") or [] + if delta_urls: + lines.append(f" *Delta OTAs ({len(delta_urls)}), in order:*") + for index, url in enumerate(delta_urls, start=1): + lines.append(f" {index}. `{format_url_display(url)}`") + return "\n".join(lines) + "\n" + + async def format_comprehensive_progress_message( job_data: Dict[str, Any], current_step: str, @@ -268,6 +285,9 @@ async def format_comprehensive_progress_message( Returns: Formatted progress message """ + private = bool(job_data["dump_args"].get("use_privdump")) + current_step = redact_urls(current_step, private=private) + # Generate progress bar progress_bar = generate_progress_bar(progress) @@ -299,11 +319,7 @@ async def format_comprehensive_progress_message( # Build message message = f" *{status_text}*\n\n" - if job_data["dump_args"].get("use_privdump"): - message += " *URL:* `[hidden for private dump]`\n" - else: - url_display = format_url_display(job_data["dump_args"]["url"]) - message += f" *URL:* `{url_display}`\n" + message += format_firmware_inputs(job_data["dump_args"]) message += f"*Job ID:* `{job_id_display}`\n" # Format options @@ -545,6 +561,7 @@ def format_status_update_message( async def format_enhanced_job_status(job: "DumpJob") -> str: """Format detailed job status using ARQ metadata.""" metadata = job.metadata.model_dump() if job.metadata else {} + private = job.dump_args.use_privdump text = f" *Job Details: {escape_markdown(job.job_id)}*\n\n" text += f" *Status:* {job.status.value.title()}\n" @@ -564,13 +581,16 @@ async def format_enhanced_job_status(job: "DumpJob") -> str: # Progress info if job.progress: text += f" *Progress:* {job.progress.percentage:.1f}%\n" - text += f" *Current Step:* {job.progress.current_step}\n" + current_step = redact_urls(job.progress.current_step, private=private) + text += f" *Current Step:* {current_step}\n" # Error details if metadata.get("error_context"): error = metadata["error_context"] - text += f" *Error:* {escape_markdown(error.get('message', 'Unknown error'))}\n" - text += f" *Failed at:* {error.get('current_step', 'Unknown step')}\n" + error_message = redact_urls(error.get('message', 'Unknown error'), private=private) + failed_at = redact_urls(error.get('current_step', 'Unknown step'), private=private) + text += f" *Error:* {escape_markdown(error_message)}\n" + text += f" *Failed at:* {failed_at}\n" # Timing if job.started_at: @@ -592,15 +612,18 @@ async def format_jobs_overview(active_jobs: List["DumpJob"], recent_jobs: List[" if active_jobs: text += f" *Active Jobs ({len(active_jobs)}):*\n" for job in active_jobs[:5]: # Limit display - metadata = job.metadata.model_dump() if job.metadata else {} - url = metadata.get("telegram_context", {}).get("url", "Unknown URL") + if job.dump_args.use_privdump: + input_summary = "[hidden for private dump]" + else: + input_summary = format_url_display(job.dump_args.url, max_length=50) + if job.dump_args.delta_urls: + input_summary += f" + {len(job.dump_args.delta_urls)} ordered delta(s)" status = job.progress.current_step if job.progress else "Initializing" + status = redact_urls(status, private=job.dump_args.use_privdump) percentage = job.progress.percentage if job.progress else 0 - # Truncate URL for display - short_url = url[:50] + "..." if len(url) > 50 else url - text += f"• `{job.job_id}` - {escape_markdown(short_url)}\n" + text += f"• `{job.job_id}` - {escape_markdown(input_summary)}\n" text += f" └─ {status} ({percentage:.1f}%)\n" text += "\n" else: diff --git a/dumpyarabot/message_queue.py b/dumpyarabot/message_queue.py index dff392d..aff98fa 100644 --- a/dumpyarabot/message_queue.py +++ b/dumpyarabot/message_queue.py @@ -4,19 +4,25 @@ import uuid from datetime import datetime, timedelta, timezone from enum import Enum -from typing import Any, Awaitable, Dict, Optional, List, TypeVar +from typing import Any, Awaitable, Dict, List, Optional, TypeVar import redis.asyncio as redis +import telegram from pydantic import BaseModel, Field, model_validator from rich.console import Console from telegram import Bot -from telegram.error import RetryAfter, TelegramError, NetworkError, BadRequest +from telegram.error import BadRequest, NetworkError, RetryAfter, TelegramError from telegram.request import HTTPXRequest -import telegram from dumpyarabot import lua_scripts from dumpyarabot.config import settings -from dumpyarabot.schemas import DumpArguments, DumpJob, JobCancelResult, JobProgress, JobStatus +from dumpyarabot.schemas import ( + DumpArguments, + DumpJob, + JobCancelResult, + JobProgress, + JobStatus, +) console = Console() @@ -829,6 +835,7 @@ async def _process_message(self, message: QueuedMessage) -> bool: if message.type == MessageType.DOCUMENT: import io + from telegram import InputFile if not message.document_content_b64 or not message.document_filename: @@ -1219,7 +1226,7 @@ async def verify_telegram_context(self, job_data: Dict[str, Any]) -> None: Raises RuntimeError for non-retryable failures (bot blocked, message/chat gone) so the caller can abort the job before doing any heavy work. """ - from telegram.error import Forbidden, BadRequest + from telegram.error import BadRequest, Forbidden try: bot = await self._ensure_bot() @@ -1388,7 +1395,7 @@ async def get_job_status(self, job_id: str) -> Optional[DumpJob]: dump_args_data = job_payload.get("dump_args") or {} telegram_context = metadata.get("telegram_context") or {} - url = telegram_context.get("url") or dump_args_data.get("url") + url = dump_args_data.get("url") or telegram_context.get("url") if not url: return None @@ -1399,6 +1406,7 @@ async def get_job_status(self, job_id: str) -> Optional[DumpJob]: "status": self._arq_status_to_job_status(arq_status["status"]), "dump_args": DumpArguments( url=url, + delta_urls=dump_args_data.get("delta_urls") or [], use_alt_dumper=dump_args_data.get("use_alt_dumper", False), force=dump_args_data.get("force", False), use_privdump=dump_args_data.get("use_privdump", False), diff --git a/dumpyarabot/mockup_handlers.py b/dumpyarabot/mockup_handlers.py index 1e5f1bb..bc8c310 100644 --- a/dumpyarabot/mockup_handlers.py +++ b/dumpyarabot/mockup_handlers.py @@ -7,20 +7,29 @@ if TYPE_CHECKING: from telegram import InlineKeyboardMarkup -from dumpyarabot.config import (CALLBACK_ACCEPT, CALLBACK_CANCEL_REQUEST, - CALLBACK_REJECT, - CALLBACK_RESTART_CANCEL, CALLBACK_RESTART_CONFIRM, - CALLBACK_SUBMIT_ACCEPTANCE, CALLBACK_TOGGLE_ALT, - CALLBACK_TOGGLE_FORCE, CALLBACK_TOGGLE_PRIVDUMP) +# Import main handlers to avoid duplication +from dumpyarabot import moderated_handlers +from dumpyarabot.config import ( + CALLBACK_ACCEPT, + CALLBACK_CANCEL_REQUEST, + CALLBACK_REJECT, + CALLBACK_RESTART_CANCEL, + CALLBACK_RESTART_CONFIRM, + CALLBACK_SUBMIT_ACCEPTANCE, + CALLBACK_TOGGLE_ALT, + CALLBACK_TOGGLE_FORCE, + CALLBACK_TOGGLE_PRIVDUMP, + settings, +) +from dumpyarabot.privacy import sanitize_url from dumpyarabot.schemas import AcceptOptionsState, MockupState, PendingReview from dumpyarabot.storage import ReviewStorage -from dumpyarabot.ui import (REVIEW_TEMPLATE, create_options_keyboard, - create_review_keyboard) +from dumpyarabot.ui import ( + REVIEW_TEMPLATE, + create_options_keyboard, + create_review_keyboard, +) from dumpyarabot.utils import generate_request_id -from dumpyarabot.config import settings - -# Import main handlers to avoid duplication -from dumpyarabot import moderated_handlers # Mockup-specific callback prefixes for reset/back/delete functionality CALLBACK_MOCKUP_RESET = "mockup_reset_" @@ -651,8 +660,14 @@ async def _handle_submit_callback_with_mockup_state( "\n".join(options_summary) if options_summary else "No special options selected" ) + input_text = ( + "URL: [hidden for private dump]" + if options_state.privdump + else f"Base URL: {sanitize_url(pending_review.url)}\nDelta OTAs: {len(pending_review.delta_urls)}" + ) await query.edit_message_text( - text=f" Request {request_id} accepted and dumpyara job triggered\n\nSelected options:\n{options_text}\n\nURL: {pending_review.url}" + text=f" Request {request_id} accepted and dumpyara job triggered\n\n" + f"Selected options:\n{options_text}\n\n{input_text}" ) else: # For real requests, delegate to main handler @@ -690,5 +705,3 @@ async def _handle_cancel_callback_with_mockup_state( else: # For real requests, delegate to main handler await moderated_handlers._handle_cancel_callback(query, context, callback_data) - - diff --git a/dumpyarabot/moderated_handlers.py b/dumpyarabot/moderated_handlers.py index 2dbaeb0..c457e12 100644 --- a/dumpyarabot/moderated_handlers.py +++ b/dumpyarabot/moderated_handlers.py @@ -1,23 +1,36 @@ import re -from datetime import datetime, timezone import secrets +from datetime import datetime, timezone from typing import Any, Optional from rich.console import Console from telegram import Chat, Message, ReplyParameters, Update from telegram.ext import ContextTypes -from dumpyarabot import schemas, utils, url_utils -from dumpyarabot.utils import escape_markdown -from dumpyarabot.config import (CALLBACK_ACCEPT, CALLBACK_CANCEL_REQUEST, - CALLBACK_REJECT, CALLBACK_SUBMIT_ACCEPTANCE, - CALLBACK_TOGGLE_ALT, CALLBACK_TOGGLE_FORCE, - CALLBACK_TOGGLE_PRIVDUMP, settings) -from dumpyarabot.message_formatting import generate_progress_bar +from dumpyarabot import schemas, url_utils, utils +from dumpyarabot.config import ( + CALLBACK_ACCEPT, + CALLBACK_CANCEL_REQUEST, + CALLBACK_REJECT, + CALLBACK_SUBMIT_ACCEPTANCE, + CALLBACK_TOGGLE_ALT, + CALLBACK_TOGGLE_FORCE, + CALLBACK_TOGGLE_PRIVDUMP, + settings, +) +from dumpyarabot.message_formatting import format_firmware_inputs, generate_progress_bar from dumpyarabot.message_queue import message_queue +from dumpyarabot.privacy import redact_for_job, redact_urls, sanitize_url from dumpyarabot.storage import ReviewStorage -from dumpyarabot.ui import (ACCEPTANCE_TEMPLATE, REJECTION_TEMPLATE, REVIEW_TEMPLATE, SUBMISSION_TEMPLATE, - create_options_keyboard, create_review_keyboard) +from dumpyarabot.ui import ( + ACCEPTANCE_TEMPLATE, + REJECTION_TEMPLATE, + REVIEW_TEMPLATE, + SUBMISSION_TEMPLATE, + create_options_keyboard, + create_review_keyboard, +) +from dumpyarabot.utils import escape_markdown console = Console() @@ -55,27 +68,31 @@ async def _create_status_message( ) -> tuple[int, int, str]: """Create the bot-owned status message that later worker updates will edit.""" primary_allowed_chat = settings.ALLOWED_CHATS[0] if settings.ALLOWED_CHATS else pending_review.review_chat_id - initial_text = _build_status_message_text(pending_review.url, dump_args, job_id) + initial_text = _build_status_message_text(dump_args, job_id) + reply_parameters = None + if not dump_args.use_privdump: + reply_parameters = ReplyParameters( + message_id=pending_review.original_message_id, + chat_id=pending_review.original_chat_id, + ) status_message = await context.bot.send_message( chat_id=primary_allowed_chat, text=initial_text, parse_mode=settings.DEFAULT_PARSE_MODE, disable_web_page_preview=True, - reply_parameters=ReplyParameters( - message_id=pending_review.original_message_id, - chat_id=pending_review.original_chat_id, - ), + reply_parameters=reply_parameters, ) return status_message.message_id, primary_allowed_chat, initial_text -def _build_status_message_text(url: str, dump_args: schemas.DumpArguments, job_id: str) -> str: +def _build_status_message_text(dump_args: schemas.DumpArguments, job_id: str) -> str: """Build the initial worker status message text.""" if dump_args.use_privdump: initial_text = " *Private Dump Job Queued*\n\n" else: - initial_text = f" *Firmware Dump Queued*\n\n *URL:* `{url}`\n" + initial_text = " *Firmware Dump Queued*\n\n" + initial_text += format_firmware_inputs(dump_args.model_dump()) initial_text += f"*Job ID:* `{job_id}`\n" @@ -113,38 +130,46 @@ async def handle_request_message( console.print(f"[yellow]Message from non-request chat: {chat.id}[/yellow]") return - # 2. Parse message for "#request " pattern (flexible format) - # Supports: "#requesthttps://...", "#request https://...", "#request please https://...", etc. - # DOTALL flag allows . to match newlines for multi-line messages - request_pattern = r"#request\s*.*?(https?://[^\s]+)" - match = re.search(request_pattern, message.text or "", re.IGNORECASE | re.DOTALL) + # 2. Capture every ordered URL following the #request tag. + raw_message = message.text or "" + tag_match = re.search(r"#request", raw_message, re.IGNORECASE) + url_strings = ( + re.findall(r"https?://[^\s]+", raw_message[tag_match.end() :], re.IGNORECASE) + if tag_match + else [] + ) - if not match: + if not url_strings: console.print("[yellow]No valid #request pattern found[/yellow]") return - url_str = match.group(1) - console.print(f"[blue]Processing request for URL: {url_str}[/blue]") + console.print(f"[blue]Processing request with {len(url_strings)} firmware URL(s)[/blue]") try: - # 3. Validate URL using new utility - is_valid, validated_url, error_msg = await url_utils.validate_and_normalize_url(url_str) - if not is_valid: - raise ValueError(error_msg) + # 3. Validate every URL without changing its order. + validated_urls = [] + for url_str in url_strings: + is_valid, validated_url, error_msg = await url_utils.validate_and_normalize_url(url_str) + if not is_valid or validated_url is None: + raise ValueError(error_msg) + validated_urls.append(validated_url) # 4. Generate request_id request_id = utils.generate_request_id() # 5. Send review message to REVIEW_CHAT_ID with Accept/Reject buttons - raw_message = message.text or "" # Remove the URL from the original message since it's already displayed above message_without_url = re.sub(r'https?://[^\s]+', '', raw_message).strip() # Remove #request tag and extra whitespace message_without_url = re.sub(r'#request\s*', '', message_without_url).strip() original_message = _truncate_message(message_without_url) if message_without_url else "No additional text" + url_summary = "\n".join( + f"{index}. {escape_markdown(sanitize_url(url))}" + for index, url in enumerate(validated_urls, start=1) + ) review_text = REVIEW_TEMPLATE.format( username=escape_markdown(user.username or user.first_name or str(user.id)), - url=escape_markdown(str(validated_url)), + url=url_summary, request_id=request_id, original_message=escape_markdown(original_message), ) @@ -169,7 +194,7 @@ async def handle_request_message( # 6. Notify user of successful submission directly to get real Telegram message ID submission_message = await message_queue.send_immediate_message( chat_id=chat.id, - text=SUBMISSION_TEMPLATE.format(url=validated_url), + text=SUBMISSION_TEMPLATE.format(url=url_summary), parse_mode=settings.DEFAULT_PARSE_MODE, reply_to_message_id=message.message_id, disable_web_page_preview=True, @@ -182,7 +207,8 @@ async def handle_request_message( original_message_id=message.message_id, requester_id=user.id, requester_username=user.username, - url=str(validated_url), # Convert AnyHttpUrl to string for storage + url=validated_urls[0], + delta_urls=validated_urls[1:], review_chat_id=settings.REVIEW_CHAT_ID, review_message_id=review_message.message_id, submission_confirmation_message_id=submission_message.message_id, @@ -193,11 +219,11 @@ async def handle_request_message( console.print(f"[green]Request {request_id} processed successfully[/green]") except ValueError: - console.print(f"[red]Invalid URL provided: {url_str}[/red]") + console.print("[red]Invalid URL provided in moderated request[/red]") await message_queue.send_error( chat_id=chat.id, text=" Invalid URL format provided", - context={"moderated_request": True, "url": url_str, "error": "invalid_url"} + context={"moderated_request": True, "error": "invalid_url"}, ) except Exception as e: console.print(f"[red]Error processing request: {e}[/red]") @@ -205,7 +231,7 @@ async def handle_request_message( await message_queue.send_error( chat_id=chat.id, text=" An error occurred while processing your request", - context={"moderated_request": True, "url": url_str, "error": "processing_failed"} + context={"moderated_request": True, "error": "processing_failed"}, ) @@ -267,7 +293,11 @@ async def _handle_accept_callback( # Update message to show options await query.edit_message_text( - text=f" Configure options for request {request_id}\nURL: {pending_review.url}", + text=( + f" Configure options for request {request_id}\n" + f"URL: {sanitize_url(pending_review.url)}\n" + f"Delta OTAs: {len(pending_review.delta_urls)}" + ), reply_markup=create_options_keyboard(request_id, options_state), disable_web_page_preview=True, ) @@ -341,10 +371,11 @@ async def _handle_submit_callback( # Create DumpArguments with the selected options dump_args = schemas.DumpArguments( url=schemas.AnyHttpUrl(pending_review.url), # Convert string back to AnyHttpUrl + delta_urls=pending_review.delta_urls, use_alt_dumper=options_state.alt, force=options_state.force, use_privdump=options_state.privdump, - initial_message_id=pending_review.original_message_id, + initial_message_id=None if options_state.privdump else pending_review.original_message_id, initial_chat_id=pending_review.original_chat_id, ) @@ -368,15 +399,13 @@ async def _handle_submit_callback( # Create enhanced job data with metadata structure enhanced_job_data = job.model_dump() enhanced_job_data["_queued_text"] = queued_text - enhanced_job_data["metadata"] = { - "telegram_context": { + telegram_context = { "chat_id": pending_review.original_chat_id, "message_id": pending_review.original_message_id, "user_id": pending_review.requester_id, - "url": pending_review.url, "moderated_request": True, - } } + enhanced_job_data["metadata"] = {"telegram_context": telegram_context} console.print(f"[blue]Queueing dump job {job.job_id} with metadata...[/blue]") job_id = await message_queue.queue_dump_job_with_metadata(enhanced_job_data) @@ -393,13 +422,25 @@ async def _handle_submit_callback( console.print(f"[green]Sending acceptance message to user: {user_message}[/green]") console.print(f"[blue]Chat ID: {pending_review.original_chat_id}, Message ID: {pending_review.original_message_id}[/blue]") - await message_queue.send_cross_chat( - chat_id=pending_review.original_chat_id, - text=user_message, - reply_to_message_id=pending_review.original_message_id, - reply_to_chat_id=pending_review.original_chat_id, - context={"moderated_request": True, "request_id": request_id, "stage": "acceptance"} - ) + notification_context = { + "moderated_request": True, + "request_id": request_id, + "stage": "acceptance", + } + if options_state.privdump: + await message_queue.send_reply( + chat_id=pending_review.original_chat_id, + text=user_message, + context=notification_context, + ) + else: + await message_queue.send_cross_chat( + chat_id=pending_review.original_chat_id, + text=user_message, + reply_to_message_id=pending_review.original_message_id, + reply_to_chat_id=pending_review.original_chat_id, + context=notification_context, + ) console.print("[green]Acceptance message sent successfully[/green]") @@ -408,10 +449,16 @@ async def _handle_submit_callback( await _cleanup_request(context, request_id) except Exception as e: - console.print(f"[red]Error processing acceptance: {e}[/red]") - console.print_exception() + safe_error = ( + redact_for_job(e, dump_args) + if options_state.privdump and "dump_args" in locals() + else redact_urls(e, private=options_state.privdump) + ) + console.print(f"[red]Error processing acceptance: {safe_error}[/red]") + if not options_state.privdump: + console.print_exception() await query.edit_message_text( - f" Error processing request {request_id}: {str(e)}" + f" Error processing request {request_id}: {safe_error}" ) @@ -491,10 +538,11 @@ async def accept_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> # Start dump process with options dump_args = schemas.DumpArguments( url=schemas.AnyHttpUrl(pending_review.url), # Convert string back to AnyHttpUrl + delta_urls=pending_review.delta_urls, use_alt_dumper=use_alt, force=force, use_privdump=use_privdump, - initial_message_id=pending_review.original_message_id, + initial_message_id=None if use_privdump else pending_review.original_message_id, initial_chat_id=pending_review.original_chat_id, ) @@ -518,15 +566,13 @@ async def accept_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> # Create enhanced job data with metadata structure enhanced_job_data = job.model_dump() enhanced_job_data["_queued_text"] = queued_text - enhanced_job_data["metadata"] = { - "telegram_context": { + telegram_context = { "chat_id": pending_review.original_chat_id, "message_id": pending_review.original_message_id, "user_id": pending_review.requester_id, - "url": pending_review.url, "moderated_request": True, - } } + enhanced_job_data["metadata"] = {"telegram_context": telegram_context} console.print(f"[blue]Queueing dump job {job.job_id} with metadata...[/blue]") job_id = await message_queue.queue_dump_job_with_metadata(enhanced_job_data) @@ -551,24 +597,42 @@ async def accept_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> console.print(f"[green]Sending acceptance message via command to user: {user_message}[/green]") console.print(f"[blue]Chat ID: {pending_review.original_chat_id}, Message ID: {pending_review.original_message_id}[/blue]") - await message_queue.send_cross_chat( - chat_id=pending_review.original_chat_id, - text=user_message, - reply_to_message_id=pending_review.original_message_id, - reply_to_chat_id=pending_review.original_chat_id, - context={"command": "accept", "action": "acceptance_notification", "request_id": request_id} - ) + notification_context = { + "command": "accept", + "action": "acceptance_notification", + "request_id": request_id, + } + if use_privdump: + await message_queue.send_reply( + chat_id=pending_review.original_chat_id, + text=user_message, + context=notification_context, + ) + else: + await message_queue.send_cross_chat( + chat_id=pending_review.original_chat_id, + text=user_message, + reply_to_message_id=pending_review.original_message_id, + reply_to_chat_id=pending_review.original_chat_id, + context=notification_context, + ) console.print("[green]Acceptance message via command sent successfully[/green]") await _cleanup_request(context, request_id) except Exception as e: - console.print(f"[red]Error processing acceptance: {e}[/red]") - console.print_exception() + safe_error = ( + redact_for_job(e, dump_args) + if use_privdump and "dump_args" in locals() + else redact_urls(e, private=use_privdump) + ) + console.print(f"[red]Error processing acceptance: {safe_error}[/red]") + if not use_privdump: + console.print_exception() await message_queue.send_error( chat_id=chat.id, - text=f" Error processing request {request_id}: {str(e)}", - context={"command": "accept", "error": "processing_exception", "request_id": request_id, "exception": str(e)} + text=f" Error processing request {request_id}: {safe_error}", + context={"command": "accept", "error": "processing_exception", "request_id": request_id} ) diff --git a/dumpyarabot/privacy.py b/dumpyarabot/privacy.py new file mode 100644 index 0000000..86cc9f0 --- /dev/null +++ b/dumpyarabot/privacy.py @@ -0,0 +1,78 @@ +import re +from pathlib import PurePosixPath +from typing import Any +from urllib.parse import unquote, urlsplit, urlunsplit + +_URL_PATTERN = re.compile(r"https?://[^\s<>\"']+", re.IGNORECASE) +PRIVATE_URL_PLACEHOLDER = "[hidden for private dump]" + + +def is_private_job(job: Any) -> bool: + """Return whether a model or persisted job payload is private.""" + if isinstance(job, dict): + dump_args = job.get("dump_args", job) + return bool(dump_args.get("use_privdump", False)) + dump_args = getattr(job, "dump_args", job) + return bool(getattr(dump_args, "use_privdump", False)) + + +def sanitize_url(url_value: Any) -> str: + """Remove URL credentials and query data for non-private displays.""" + url = str(url_value or "unknown") + try: + parts = urlsplit(url) + hostname = parts.hostname or "" + port = parts.port + username = parts.username + except ValueError: + return url + + netloc = hostname + if port: + netloc = f"{netloc}:{port}" + if username: + netloc = f"[REDACTED]@{netloc}" + return urlunsplit((parts.scheme, netloc, parts.path, "", "")) or url + + +def redact_urls(value: Any, *, private: bool) -> str: + """Replace every URL for private jobs; otherwise remove URL credentials.""" + text = str(value) + + def replace(match: re.Match[str]) -> str: + suffix = "" + url = match.group(0) + while url and url[-1] in ".,;:!?)\"]}": + suffix = url[-1] + suffix + url = url[:-1] + replacement = PRIVATE_URL_PLACEHOLDER if private else sanitize_url(url) + return replacement + suffix + + return _URL_PATTERN.sub(replace, text) + + +def redact_for_job(value: Any, job: Any) -> str: + """Redact URLs and URL-derived download names for a private job.""" + private = is_private_job(job) + text = redact_urls(value, private=private) + if not private: + return text + + dump_args = job.get("dump_args", job) if isinstance(job, dict) else getattr(job, "dump_args", job) + if isinstance(dump_args, dict): + urls = [dump_args.get("url"), *(dump_args.get("delta_urls") or [])] + else: + urls = [getattr(dump_args, "url", None), *getattr(dump_args, "delta_urls", [])] + for source_url in filter(None, urls): + path_name = urlsplit(str(source_url)).path.rpartition("/")[2] + decoded_path_name = unquote(path_name) + for source_text in ( + str(source_url), + path_name, + decoded_path_name, + PurePosixPath(path_name).stem if path_name else "", + PurePosixPath(decoded_path_name).stem if decoded_path_name else "", + ): + if source_text: + text = text.replace(source_text, PRIVATE_URL_PLACEHOLDER) + return text diff --git a/dumpyarabot/process_utils.py b/dumpyarabot/process_utils.py index 9c24f1f..e36eb8f 100644 --- a/dumpyarabot/process_utils.py +++ b/dumpyarabot/process_utils.py @@ -6,7 +6,7 @@ import signal import subprocess from pathlib import Path -from typing import List, Optional, Tuple, Union, Dict, Any +from typing import Any, Dict, List, Optional, Tuple, Union from rich.console import Console @@ -406,6 +406,7 @@ async def run_download_command( cwd: Optional[Union[str, Path]] = None, timeout: float = 600.0, description: Optional[str] = None, + quiet: bool = False, ) -> ProcessResult: """Run a download command with standard settings.""" return await run_command( @@ -414,6 +415,7 @@ async def run_download_command( timeout=timeout, capture_output=True, check=False, # Allow handling download failures gracefully + quiet=quiet, description=description or f"Download with {tool}", ) diff --git a/dumpyarabot/schemas.py b/dumpyarabot/schemas.py index aefef45..9385741 100644 --- a/dumpyarabot/schemas.py +++ b/dumpyarabot/schemas.py @@ -1,13 +1,13 @@ from datetime import datetime, timezone from enum import Enum -from typing import Dict, List, Optional, Any +from typing import Any, Dict, List, Optional from pydantic import AnyHttpUrl, BaseModel, Field - class DumpArguments(BaseModel): url: AnyHttpUrl + delta_urls: List[AnyHttpUrl] = Field(default_factory=list) use_alt_dumper: bool force: bool = False use_privdump: bool @@ -22,6 +22,7 @@ class PendingReview(BaseModel): requester_id: int requester_username: Optional[str] url: str + delta_urls: List[str] = Field(default_factory=list) review_chat_id: int review_message_id: int submission_confirmation_message_id: Optional[int] = None diff --git a/dumpyarabot/url_utils.py b/dumpyarabot/url_utils.py index e630a53..f26af20 100644 --- a/dumpyarabot/url_utils.py +++ b/dumpyarabot/url_utils.py @@ -6,10 +6,29 @@ import httpx from pydantic import AnyHttpUrl, TypeAdapter, ValidationError - HTTP_URL_ADAPTER = TypeAdapter(AnyHttpUrl) +def parse_dump_tokens(tokens: list[str]) -> tuple[list[str], str]: + """Split ordered firmware URLs from one optional final option token.""" + if not tokens: + raise ValueError("At least one firmware URL is required") + + option_token = "" + candidate = tokens[-1] + if ( + candidate + and set(candidate) <= {"a", "f", "p"} + and len(set(candidate)) == len(candidate) + ): + option_token = candidate + tokens = tokens[:-1] + + if not tokens: + raise ValueError("At least one firmware URL is required") + return tokens, option_token + + async def validate_and_normalize_url(url_str: str) -> Tuple[bool, Optional[str], Optional[str]]: """ Validate URL and return (is_valid, normalized_url, error_message). diff --git a/pyproject.toml b/pyproject.toml index bdceac5..5be7a82 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ dependencies = [ "arq>=0.27.0,<1.0.0", "pillow>=12.1.1,<13.0.0", "pyppmd>=1.3.1", + "py7zr>=1.1.3,<2.0.0", "aria2p>=0.12.0,<1.0.0", ] name = "dumpyarabot" diff --git a/uv.lock b/uv.lock index cfd4fcb..34063ba 100644 --- a/uv.lock +++ b/uv.lock @@ -677,6 +677,7 @@ dependencies = [ { name = "dumpyara" }, { name = "httpx" }, { name = "pillow" }, + { name = "py7zr" }, { name = "pydantic" }, { name = "pydantic-settings" }, { name = "pyppmd" }, @@ -708,6 +709,7 @@ requires-dist = [ { name = "dumpyara", git = "https://github.com/AndroidDumps/pydumpyara?rev=f6011365c1aaec934e3aaec23d9855d11953bca6" }, { name = "httpx", specifier = ">=0.28.1,<1.0.0" }, { name = "pillow", specifier = ">=12.1.1,<13.0.0" }, + { name = "py7zr", specifier = ">=1.1.3,<2.0.0" }, { name = "pydantic", specifier = ">=2.12.5,<3.0.0" }, { name = "pydantic-settings", specifier = ">=2.13.1,<3.0.0" }, { name = "pyppmd", specifier = ">=1.3.1" }, From d830615bf00aeb1711be522059c1e57870575beb Mon Sep 17 00:00:00 2001 From: Omkar Chandorkar Date: Sat, 12 Sep 2026 10:30:11 +0530 Subject: [PATCH 2/7] bot: Harden moderated delta requests Signed-off-by: Omkar Chandorkar --- dumpyarabot/arq_jobs.py | 4 +- dumpyarabot/firmware_extractor.py | 138 ++++----- dumpyarabot/handlers.py | 6 + dumpyarabot/message_formatting.py | 21 +- dumpyarabot/moderated_handlers.py | 462 ++++++++++++++++++++++++++---- dumpyarabot/redis_storage.py | 49 ++++ dumpyarabot/schemas.py | 3 + dumpyarabot/storage.py | 33 +++ 8 files changed, 577 insertions(+), 139 deletions(-) diff --git a/dumpyarabot/arq_jobs.py b/dumpyarabot/arq_jobs.py index 14dfc52..ddf11b2 100644 --- a/dumpyarabot/arq_jobs.py +++ b/dumpyarabot/arq_jobs.py @@ -603,7 +603,9 @@ async def _on_download_progress(dp: DownloadProgress) -> None: downloaded_paths[0], cancellation_check=lambda: arq_pool.is_job_cancel_requested(job_id), ) - if dump_job.dump_args.delta_urls or base_is_raw: + if dump_job.dump_args.delta_urls or ( + base_is_raw and not dump_job.dump_args.use_alt_dumper + ): await extractor.extract_reconstructed_firmware( dump_job, downloaded_paths, diff --git a/dumpyarabot/firmware_extractor.py b/dumpyarabot/firmware_extractor.py index 96ddd74..ec9117f 100644 --- a/dumpyarabot/firmware_extractor.py +++ b/dumpyarabot/firmware_extractor.py @@ -171,12 +171,22 @@ def _raw_image_names_are_ready(names: Sequence[str]) -> bool: return any(stem in supported_partitions for stem in stems) -def _is_raw_archive_ancillary(name: str) -> bool: - path = _safe_archive_member(name) - return path.suffix.lower() in {".md", ".sha256", ".txt"} or path.name.upper() in { - "README", - "SHA256SUMS", - } +def _archive_member_needs_preparation(path: PurePosixPath) -> bool: + """Keep known Android partition containers on the legacy extraction path.""" + name = path.name.lower() + return name == "payload.bin" or name.endswith( + ( + ".new.dat", + ".new.dat.br", + ".patch.dat", + ".transfer.list", + ".img.br", + ".img.gz", + ".img.lz4", + ".img.xz", + ".img.zst", + ) + ) def _unpack_raw_image_archive(archive_path: Path, destination: Path) -> None: @@ -280,7 +290,6 @@ class FirmwareExtractor: def __init__(self, work_dir: str): self.work_dir = Path(work_dir) self.firmware_extractor_path = Path.home() / "Firmware_extractor" - self._native_tasks: set[asyncio.Task[None]] = set() @staticmethod def is_raw_image_archive(firmware_path: str) -> bool: @@ -288,69 +297,79 @@ def is_raw_image_archive(firmware_path: str) -> bool: path = Path(firmware_path) if zipfile.is_zipfile(path): with zipfile.ZipFile(path) as archive: - regular_files = [info for info in archive.infolist() if not info.is_dir()] - if any( - PurePosixPath(info.filename.replace("\\", "/")).name - == "payload.bin" - for info in regular_files - ): - return False - image_files = [ - info - for info in regular_files - if _safe_archive_member(info.filename).suffix.lower() == ".img" - ] - if any( - info not in image_files - and not _is_raw_archive_ancillary(info.filename) - for info in regular_files - ): - return False + zip_image_files: list[zipfile.ZipInfo] = [] + image_basenames: set[str] = set() + for info in archive.infolist(): + member_path = _safe_archive_member(info.filename) + mode = info.external_attr >> 16 + file_type = stat.S_IFMT(mode) + if stat.S_ISLNK(mode) or file_type not in { + 0, + stat.S_IFREG, + stat.S_IFDIR, + }: + return False + if info.is_dir(): + continue + if _archive_member_needs_preparation(member_path): + return False + if member_path.suffix.lower() == ".img": + basename = member_path.name.casefold() + if basename in image_basenames: + return False + image_basenames.add(basename) + zip_image_files.append(info) return _raw_image_names_are_ready( - [info.filename for info in image_files] + [info.filename for info in zip_image_files] ) and all( archive.open(info).read(4) != ANDROID_SPARSE_MAGIC - for info in image_files + for info in zip_image_files ) if tarfile.is_tarfile(path): with tarfile.open(path, mode="r:*") as archive: - regular_files = [ - member for member in archive.getmembers() if member.isfile() - ] - image_files = [ - member - for member in regular_files - if _safe_archive_member(member.name).suffix.lower() == ".img" - ] - if any( - member not in image_files - and not _is_raw_archive_ancillary(member.name) - for member in regular_files - ): - return False + tar_image_files: list[tarfile.TarInfo] = [] + image_basenames = set() + for member in archive.getmembers(): + member_path = _safe_archive_member(member.name) + if member.isdir(): + continue + if not member.isreg(): + return False + if _archive_member_needs_preparation(member_path): + return False + if member_path.suffix.lower() == ".img": + basename = member_path.name.casefold() + if basename in image_basenames: + return False + image_basenames.add(basename) + tar_image_files.append(member) if not _raw_image_names_are_ready( - [member.name for member in image_files] + [member.name for member in tar_image_files] ): return False - for member in image_files: + for member in tar_image_files: extracted = archive.extractfile(member) if extracted is None or extracted.read(4) == ANDROID_SPARSE_MAGIC: return False return True if path.name.lower().endswith(".7z"): with py7zr.SevenZipFile(path, mode="r") as archive: - regular_files = [info for info in archive.list() if info.is_file] - image_names = [ - info.filename - for info in regular_files - if _safe_archive_member(info.filename).suffix.lower() == ".img" - ] - if any( - info.filename not in image_names - and not _is_raw_archive_ancillary(info.filename) - for info in regular_files - ): - return False + image_names: list[str] = [] + image_basenames = set() + for info in archive.list(): + member_path = _safe_archive_member(info.filename) + if info.is_directory: + continue + if info.is_symlink or not info.is_file: + return False + if _archive_member_needs_preparation(member_path): + return False + if member_path.suffix.lower() == ".img": + basename = member_path.name.casefold() + if basename in image_basenames: + return False + image_basenames.add(basename) + image_names.append(info.filename) if not _raw_image_names_are_ready(image_names): return False magic_factory = _ImageMagicFactory() @@ -398,7 +417,6 @@ async def classify_raw_image_archive( worker = asyncio.create_task( asyncio.to_thread(self.is_raw_image_archive, firmware_path) ) - self._native_tasks.add(worker) try: while not worker.done(): if await _check_cancelled(cancellation_check): @@ -416,8 +434,6 @@ async def classify_raw_image_archive( if not worker.done(): await self._drain_task(worker) raise - finally: - self._native_tasks.discard(worker) async def _run_otadump( self, @@ -445,7 +461,6 @@ def run() -> None: ) from error worker = asyncio.create_task(asyncio.to_thread(run)) - self._native_tasks.add(worker) deadline = asyncio.get_running_loop().time() + timeout try: while not worker.done(): @@ -480,8 +495,6 @@ def run() -> None: token.cancel() await self._drain_task(worker) raise - finally: - self._native_tasks.discard(worker) async def _run_blocking_and_drain( self, @@ -491,7 +504,6 @@ async def _run_blocking_and_drain( ) -> None: """Do not let a non-native extraction thread outlive its work directory.""" worker = asyncio.create_task(asyncio.to_thread(function, *args)) - self._native_tasks.add(worker) deadline = asyncio.get_running_loop().time() + timeout try: while not worker.done(): @@ -515,8 +527,6 @@ async def _run_blocking_and_drain( if not worker.done(): await self._drain_task(worker) raise - finally: - self._native_tasks.discard(worker) async def extract_reconstructed_firmware( self, diff --git a/dumpyarabot/handlers.py b/dumpyarabot/handlers.py index 7442a98..907201a 100644 --- a/dumpyarabot/handlers.py +++ b/dumpyarabot/handlers.py @@ -81,6 +81,12 @@ async def dump( console.print("[red]Chat or message object is None[/red]") return + if chat.id == settings.REQUEST_CHAT_ID: + from dumpyarabot.moderated_handlers import handle_moderated_dump + + await handle_moderated_dump(update, context) + return + # Ensure it can only be used in the correct group if chat.id not in settings.ALLOWED_CHATS: # Do nothing diff --git a/dumpyarabot/message_formatting.py b/dumpyarabot/message_formatting.py index fdbaf50..b2128b1 100644 --- a/dumpyarabot/message_formatting.py +++ b/dumpyarabot/message_formatting.py @@ -5,7 +5,7 @@ from datetime import datetime, timezone from typing import TYPE_CHECKING, Any, Dict, List, Optional -from dumpyarabot.privacy import redact_urls, sanitize_url +from dumpyarabot.privacy import redact_for_job, sanitize_url from dumpyarabot.utils import escape_markdown if TYPE_CHECKING: @@ -285,8 +285,7 @@ async def format_comprehensive_progress_message( Returns: Formatted progress message """ - private = bool(job_data["dump_args"].get("use_privdump")) - current_step = redact_urls(current_step, private=private) + current_step = redact_for_job(current_step, job_data) # Generate progress bar progress_bar = generate_progress_bar(progress) @@ -371,9 +370,11 @@ async def format_comprehensive_progress_message( # Keep failure edits concise; detailed errors are sent as an attached log file. if progress and progress.get("error_message") and metadata and metadata.get("error_context"): error_ctx = metadata["error_context"] - message += f"\n *Failed at:* {escape_markdown(error_ctx.get('current_step', 'Unknown step'))}\n" + failed_at = redact_for_job(error_ctx.get('current_step', 'Unknown step'), job_data) + message += f"\n *Failed at:* {escape_markdown(failed_at)}\n" if error_ctx.get("last_successful_step"): - message += f" *Last successful:* {escape_markdown(error_ctx['last_successful_step'])}\n" + last_successful = redact_for_job(error_ctx["last_successful_step"], job_data) + message += f" *Last successful:* {escape_markdown(last_successful)}\n" return message @@ -561,8 +562,6 @@ def format_status_update_message( async def format_enhanced_job_status(job: "DumpJob") -> str: """Format detailed job status using ARQ metadata.""" metadata = job.metadata.model_dump() if job.metadata else {} - private = job.dump_args.use_privdump - text = f" *Job Details: {escape_markdown(job.job_id)}*\n\n" text += f" *Status:* {job.status.value.title()}\n" @@ -581,14 +580,14 @@ async def format_enhanced_job_status(job: "DumpJob") -> str: # Progress info if job.progress: text += f" *Progress:* {job.progress.percentage:.1f}%\n" - current_step = redact_urls(job.progress.current_step, private=private) + current_step = redact_for_job(job.progress.current_step, job) text += f" *Current Step:* {current_step}\n" # Error details if metadata.get("error_context"): error = metadata["error_context"] - error_message = redact_urls(error.get('message', 'Unknown error'), private=private) - failed_at = redact_urls(error.get('current_step', 'Unknown step'), private=private) + error_message = redact_for_job(error.get('message', 'Unknown error'), job) + failed_at = redact_for_job(error.get('current_step', 'Unknown step'), job) text += f" *Error:* {escape_markdown(error_message)}\n" text += f" *Failed at:* {failed_at}\n" @@ -620,7 +619,7 @@ async def format_jobs_overview(active_jobs: List["DumpJob"], recent_jobs: List[" input_summary += f" + {len(job.dump_args.delta_urls)} ordered delta(s)" status = job.progress.current_step if job.progress else "Initializing" - status = redact_urls(status, private=job.dump_args.use_privdump) + status = redact_for_job(status, job) percentage = job.progress.percentage if job.progress else 0 text += f"• `{job.job_id}` - {escape_markdown(input_summary)}\n" diff --git a/dumpyarabot/moderated_handlers.py b/dumpyarabot/moderated_handlers.py index c457e12..206006d 100644 --- a/dumpyarabot/moderated_handlers.py +++ b/dumpyarabot/moderated_handlers.py @@ -5,6 +5,7 @@ from rich.console import Console from telegram import Chat, Message, ReplyParameters, Update +from telegram.error import BadRequest from telegram.ext import ContextTypes from dumpyarabot import schemas, url_utils, utils @@ -20,7 +21,12 @@ ) from dumpyarabot.message_formatting import format_firmware_inputs, generate_progress_bar from dumpyarabot.message_queue import message_queue -from dumpyarabot.privacy import redact_for_job, redact_urls, sanitize_url +from dumpyarabot.privacy import ( + PRIVATE_URL_PLACEHOLDER, + redact_for_job, + redact_urls, + sanitize_url, +) from dumpyarabot.storage import ReviewStorage from dumpyarabot.ui import ( ACCEPTANCE_TEMPLATE, @@ -60,6 +66,179 @@ async def _cleanup_request(context: ContextTypes.DEFAULT_TYPE, request_id: str) await ReviewStorage.remove_options_state(context, request_id) +def _pending_urls(pending_review: schemas.PendingReview) -> list[str]: + return [pending_review.url, *pending_review.delta_urls] + + +def _moderation_url_summary(urls: list[str], *, private: bool) -> str: + if private: + return PRIVATE_URL_PLACEHOLDER + return "\n".join( + f"{index}. {escape_markdown(sanitize_url(url))}" + for index, url in enumerate(urls, start=1) + ) + + +def _options_message_text( + request_id: str, + pending_review: schemas.PendingReview, + *, + private: bool, +) -> str: + base_url = PRIVATE_URL_PLACEHOLDER if private else sanitize_url(pending_review.url) + return ( + f" Configure options for request {request_id}\n" + f"URL: {base_url}\n" + f"Delta OTAs: {len(pending_review.delta_urls)}" + ) + + +async def _edit_message_text_if_changed(bot: Any, **kwargs: Any) -> None: + try: + await bot.edit_message_text(**kwargs) + except BadRequest as error: + if "message is not modified" not in str(error).lower(): + raise + + +async def _delete_message_if_present(bot: Any, **kwargs: Any) -> None: + try: + await bot.delete_message(**kwargs) + except BadRequest as error: + error_lower = str(error).lower() + if ( + "message to delete not found" not in error_lower + and "message can't be deleted" not in error_lower + ): + raise + + +async def _sync_bot_owned_request_summaries( + context: ContextTypes.DEFAULT_TYPE, + request_id: str, + pending_review: schemas.PendingReview, + options_state: schemas.AcceptOptionsState, +) -> None: + """Apply the selected privacy state to both bot-owned request summaries.""" + await _edit_message_text_if_changed( + bot=context.bot, + chat_id=pending_review.review_chat_id, + message_id=pending_review.review_message_id, + text=_options_message_text( + request_id, + pending_review, + private=options_state.privdump, + ), + reply_markup=create_options_keyboard(request_id, options_state), + disable_web_page_preview=True, + ) + + if pending_review.submission_confirmation_message_id is not None: + summary = _moderation_url_summary( + _pending_urls(pending_review), + private=options_state.privdump, + ) + try: + await _edit_message_text_if_changed( + bot=context.bot, + chat_id=pending_review.original_chat_id, + message_id=pending_review.submission_confirmation_message_id, + text=SUBMISSION_TEMPLATE.format(url=summary), + parse_mode=settings.DEFAULT_PARSE_MODE, + disable_web_page_preview=True, + ) + except BadRequest as error: + if "message to edit not found" in str(error).lower(): + pending_review.submission_confirmation_message_id = None + await ReviewStorage.update_pending_review(context, pending_review) + else: + raise + + if options_state.privdump: + await _detach_private_submission_confirmation(context, pending_review) + + +async def _detach_private_submission_confirmation( + context: ContextTypes.DEFAULT_TYPE, + pending_review: schemas.PendingReview, +) -> None: + """Replace a reply confirmation so Telegram cannot quote the source message.""" + stale_message_id = pending_review.stale_submission_confirmation_message_id + if stale_message_id is not None: + await _delete_message_if_present( + context.bot, + chat_id=pending_review.original_chat_id, + message_id=stale_message_id, + ) + pending_review.stale_submission_confirmation_message_id = None + await ReviewStorage.update_pending_review(context, pending_review) + + message_id = pending_review.submission_confirmation_message_id + if message_id is None or not pending_review.submission_replies_to_request: + return + + replacement = await context.bot.send_message( + chat_id=pending_review.original_chat_id, + text=SUBMISSION_TEMPLATE.format(url=PRIVATE_URL_PLACEHOLDER), + parse_mode=settings.DEFAULT_PARSE_MODE, + disable_web_page_preview=True, + ) + pending_review.submission_confirmation_message_id = replacement.message_id + pending_review.submission_replies_to_request = False + pending_review.stale_submission_confirmation_message_id = message_id + await ReviewStorage.update_pending_review(context, pending_review) + await _delete_message_if_present( + context.bot, + chat_id=pending_review.original_chat_id, + message_id=message_id, + ) + pending_review.stale_submission_confirmation_message_id = None + await ReviewStorage.update_pending_review(context, pending_review) + + +async def _delete_original_private_request( + context: ContextTypes.DEFAULT_TYPE, + pending_review: schemas.PendingReview, +) -> None: + """Best-effort removal of the requester's URL-bearing message.""" + pending_review.original_message_private = True + try: + await ReviewStorage.update_pending_review(context, pending_review) + except Exception: + console.print("[yellow]Could not persist private request state[/yellow]") + await _delete_private_request_message( + context, + pending_review.original_chat_id, + pending_review.original_message_id, + ) + + +async def _delete_private_request_message( + context: ContextTypes.DEFAULT_TYPE, + chat_id: int, + message_id: int, +) -> None: + try: + await context.bot.delete_message(chat_id=chat_id, message_id=message_id) + except Exception: + console.print("[yellow]Could not delete the original private request message[/yellow]") + + +async def _prepare_private_acceptance( + context: ContextTypes.DEFAULT_TYPE, + request_id: str, + pending_review: schemas.PendingReview, + options_state: schemas.AcceptOptionsState, +) -> None: + await _delete_original_private_request(context, pending_review) + await _sync_bot_owned_request_summaries( + context, + request_id, + pending_review, + options_state, + ) + + async def _create_status_message( context: ContextTypes.DEFAULT_TYPE, pending_review: schemas.PendingReview, @@ -71,7 +250,7 @@ async def _create_status_message( initial_text = _build_status_message_text(dump_args, job_id) reply_parameters = None - if not dump_args.use_privdump: + if not dump_args.use_privdump and not pending_review.original_message_private: reply_parameters = ReplyParameters( message_id=pending_review.original_message_id, chat_id=pending_review.original_chat_id, @@ -116,7 +295,7 @@ def _build_status_message_text(dump_args: schemas.DumpArguments, job_id: str) -> async def handle_request_message( update: Update, context: ContextTypes.DEFAULT_TYPE ) -> None: - """Handle #request messages with URL parsing and validation.""" + """Handle legacy #request messages with URL parsing and validation.""" chat: Optional[Chat] = update.effective_chat message: Optional[Message] = update.effective_message user = update.effective_user @@ -143,10 +322,73 @@ async def handle_request_message( console.print("[yellow]No valid #request pattern found[/yellow]") return + message_without_url = re.sub(r'https?://[^\s]+', '', raw_message).strip() + message_without_url = re.sub(r'#request\s*', '', message_without_url).strip() + original_message = _truncate_message(message_without_url) if message_without_url else "No additional text" + await _create_moderated_request( + update, + context, + url_strings, + schemas.AcceptOptionsState(), + original_message, + ) + + +async def handle_moderated_dump( + update: Update, + context: ContextTypes.DEFAULT_TYPE, +) -> None: + """Create a moderated request from /dump arguments.""" + chat = update.effective_chat + message = update.effective_message + if not chat or not message: + return + + try: + url_strings, options = url_utils.parse_dump_tokens(list(context.args or [])) + except ValueError as error: + await message_queue.send_reply( + chat_id=chat.id, + text=str(error), + reply_to_message_id=message.message_id, + context={"command": "dump", "error": "missing_urls"}, + ) + return + + options_state = schemas.AcceptOptionsState( + alt="a" in options, + force="f" in options, + privdump="p" in options, + ) + if options_state.privdump: + await _delete_private_request_message(context, chat.id, message.message_id) + await _create_moderated_request( + update, + context, + url_strings, + options_state, + "No additional text", + ) + + +async def _create_moderated_request( + update: Update, + context: ContextTypes.DEFAULT_TYPE, + url_strings: list[str], + options_state: schemas.AcceptOptionsState, + original_message: str, +) -> None: + """Validate, display, and persist one moderated request.""" + chat = update.effective_chat + message = update.effective_message + user = update.effective_user + if not chat or not message or not user: + return + console.print(f"[blue]Processing request with {len(url_strings)} firmware URL(s)[/blue]") try: - # 3. Validate every URL without changing its order. + # Validate every URL without changing its order. validated_urls = [] for url_str in url_strings: is_valid, validated_url, error_msg = await url_utils.validate_and_normalize_url(url_str) @@ -154,18 +396,11 @@ async def handle_request_message( raise ValueError(error_msg) validated_urls.append(validated_url) - # 4. Generate request_id request_id = utils.generate_request_id() - # 5. Send review message to REVIEW_CHAT_ID with Accept/Reject buttons - # Remove the URL from the original message since it's already displayed above - message_without_url = re.sub(r'https?://[^\s]+', '', raw_message).strip() - # Remove #request tag and extra whitespace - message_without_url = re.sub(r'#request\s*', '', message_without_url).strip() - original_message = _truncate_message(message_without_url) if message_without_url else "No additional text" - url_summary = "\n".join( - f"{index}. {escape_markdown(sanitize_url(url))}" - for index, url in enumerate(validated_urls, start=1) + url_summary = _moderation_url_summary( + validated_urls, + private=options_state.privdump, ) review_text = REVIEW_TEMPLATE.format( username=escape_markdown(user.username or user.first_name or str(user.id)), @@ -174,8 +409,6 @@ async def handle_request_message( original_message=escape_markdown(original_message), ) - # Send review message directly to get real Telegram message ID - from telegram import InlineKeyboardMarkup review_keyboard = create_review_keyboard(request_id) review_message = await message_queue.send_immediate_message( chat_id=settings.REVIEW_CHAT_ID, @@ -184,23 +417,22 @@ async def handle_request_message( reply_to_message_id=None, disable_web_page_preview=True, ) - # Attach the keyboard by editing (send_immediate_message doesn't support keyboards) await context.bot.edit_message_reply_markup( chat_id=settings.REVIEW_CHAT_ID, message_id=review_message.message_id, reply_markup=review_keyboard, ) - # 6. Notify user of successful submission directly to get real Telegram message ID submission_message = await message_queue.send_immediate_message( chat_id=chat.id, text=SUBMISSION_TEMPLATE.format(url=url_summary), parse_mode=settings.DEFAULT_PARSE_MODE, - reply_to_message_id=message.message_id, + reply_to_message_id=( + None if options_state.privdump else message.message_id + ), disable_web_page_preview=True, ) - # 7. Store PendingReview in bot_data (URL as string for Redis compatibility) pending_review = schemas.PendingReview( request_id=request_id, original_chat_id=chat.id, @@ -212,9 +444,15 @@ async def handle_request_message( review_chat_id=settings.REVIEW_CHAT_ID, review_message_id=review_message.message_id, submission_confirmation_message_id=submission_message.message_id, + submission_replies_to_request=not options_state.privdump, + original_message_private=options_state.privdump, ) - await ReviewStorage.store_pending_review(context, pending_review) + await ReviewStorage.store_pending_review_with_options( + context, + pending_review, + options_state, + ) console.print(f"[green]Request {request_id} processed successfully[/green]") @@ -225,9 +463,11 @@ async def handle_request_message( text=" Invalid URL format provided", context={"moderated_request": True, "error": "invalid_url"}, ) - except Exception as e: - console.print(f"[red]Error processing request: {e}[/red]") - console.print_exception() + except Exception as error: + safe_error = redact_urls(error, private=options_state.privdump) + console.print(f"[red]Error processing request: {safe_error}[/red]") + if not options_state.privdump: + console.print_exception() await message_queue.send_error( chat_id=chat.id, text=" An error occurred while processing your request", @@ -293,10 +533,10 @@ async def _handle_accept_callback( # Update message to show options await query.edit_message_text( - text=( - f" Configure options for request {request_id}\n" - f"URL: {sanitize_url(pending_review.url)}\n" - f"Delta OTAs: {len(pending_review.delta_urls)}" + text=_options_message_text( + request_id, + pending_review, + private=options_state.privdump, ), reply_markup=create_options_keyboard(request_id, options_state), disable_web_page_preview=True, @@ -345,12 +585,22 @@ async def _handle_toggle_callback( elif option == "privdump": options_state.privdump = not options_state.privdump + if option == "privdump" and options_state.privdump: + await _delete_original_private_request(context, pending_review) + await ReviewStorage.update_options_state(context, request_id, options_state) - # Refresh keyboard with updated state - await query.edit_message_reply_markup( - reply_markup=create_options_keyboard(request_id, options_state) - ) + try: + await _sync_bot_owned_request_summaries( + context, + request_id, + pending_review, + options_state, + ) + except Exception: + console.print("[yellow]Could not update moderated request privacy[/yellow]") + await query.edit_message_text(" Could not update request options") + return async def _handle_submit_callback( @@ -368,6 +618,14 @@ async def _handle_submit_callback( options_state = await ReviewStorage.get_options_state(context, request_id) try: + if options_state.privdump: + await _prepare_private_acceptance( + context, + request_id, + pending_review, + options_state, + ) + # Create DumpArguments with the selected options dump_args = schemas.DumpArguments( url=schemas.AnyHttpUrl(pending_review.url), # Convert string back to AnyHttpUrl @@ -375,7 +633,11 @@ async def _handle_submit_callback( use_alt_dumper=options_state.alt, force=options_state.force, use_privdump=options_state.privdump, - initial_message_id=None if options_state.privdump else pending_review.original_message_id, + initial_message_id=( + None + if options_state.privdump or pending_review.original_message_private + else pending_review.original_message_id + ), initial_chat_id=pending_review.original_chat_id, ) @@ -433,7 +695,7 @@ async def _handle_submit_callback( text=user_message, context=notification_context, ) - else: + elif not pending_review.original_message_private: await message_queue.send_cross_chat( chat_id=pending_review.original_chat_id, text=user_message, @@ -441,11 +703,20 @@ async def _handle_submit_callback( reply_to_chat_id=pending_review.original_chat_id, context=notification_context, ) + else: + await message_queue.send_reply( + chat_id=pending_review.original_chat_id, + text=user_message, + context=notification_context, + ) console.print("[green]Acceptance message sent successfully[/green]") # Delete the admin confirmation message after successful job start - await query.delete_message() + try: + await query.delete_message() + except Exception as e: + console.print(f"[yellow]Could not delete review message: {e}[/yellow]") await _cleanup_request(context, request_id) except Exception as e: @@ -529,12 +800,27 @@ async def accept_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> ) return - # Parse option flags - use_alt = "a" in options - force = "f" in options - use_privdump = "p" in options + # Add explicit moderator flags to the requester's persisted defaults. + options_state = await ReviewStorage.get_options_state(context, request_id) + if options: + options_state.alt = options_state.alt or "a" in options + options_state.force = options_state.force or "f" in options + options_state.privdump = options_state.privdump or "p" in options + await ReviewStorage.update_options_state(context, request_id, options_state) + + use_alt = options_state.alt + force = options_state.force + use_privdump = options_state.privdump try: + if use_privdump: + await _prepare_private_acceptance( + context, + request_id, + pending_review, + options_state, + ) + # Start dump process with options dump_args = schemas.DumpArguments( url=schemas.AnyHttpUrl(pending_review.url), # Convert string back to AnyHttpUrl @@ -542,7 +828,11 @@ async def accept_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> use_alt_dumper=use_alt, force=force, use_privdump=use_privdump, - initial_message_id=None if use_privdump else pending_review.original_message_id, + initial_message_id=( + None + if use_privdump or pending_review.original_message_private + else pending_review.original_message_id + ), initial_chat_id=pending_review.original_chat_id, ) @@ -608,7 +898,7 @@ async def accept_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> text=user_message, context=notification_context, ) - else: + elif not pending_review.original_message_private: await message_queue.send_cross_chat( chat_id=pending_review.original_chat_id, text=user_message, @@ -616,6 +906,12 @@ async def accept_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> reply_to_chat_id=pending_review.original_chat_id, context=notification_context, ) + else: + await message_queue.send_reply( + chat_id=pending_review.original_chat_id, + text=user_message, + context=notification_context, + ) console.print("[green]Acceptance message via command sent successfully[/green]") await _cleanup_request(context, request_id) @@ -703,6 +999,11 @@ async def reject_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> ) return + options_state = await ReviewStorage.get_options_state(context, request_id) + private_request = ( + options_state.privdump or pending_review.original_message_private + ) + try: # Get admin info admin_user = update.effective_user @@ -728,36 +1029,64 @@ async def reject_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> except Exception as e: console.print(f"[yellow]Could not delete command message: {e}[/yellow]") - # Send cleaner final message in review chat with link to original request - await message_queue.send_cross_chat( - chat_id=chat.id, - text=f" Request {request_id} rejected by @{admin_name}\nReason: {reason}", - reply_to_message_id=pending_review.original_message_id, - reply_to_chat_id=pending_review.original_chat_id, - context={"command": "reject", "action": "rejection_confirmation", "request_id": request_id, "admin": admin_name} - ) + rejection_confirmation = f" Request {request_id} rejected by @{admin_name}\nReason: {reason}" + rejection_context = { + "command": "reject", + "action": "rejection_confirmation", + "request_id": request_id, + "admin": admin_name, + } + if private_request: + await message_queue.send_reply( + chat_id=chat.id, + text=rejection_confirmation, + context=rejection_context, + ) + else: + await message_queue.send_cross_chat( + chat_id=chat.id, + text=rejection_confirmation, + reply_to_message_id=pending_review.original_message_id, + reply_to_chat_id=pending_review.original_chat_id, + context=rejection_context, + ) # Log rejection with reason console.print(f"[yellow]Request {request_id} rejected by @{admin_name}: {reason}[/yellow]") # Notify original requester with rejection message - await message_queue.send_cross_chat( - chat_id=pending_review.original_chat_id, - text=REJECTION_TEMPLATE.format(reason=reason), - reply_to_message_id=pending_review.original_message_id, - reply_to_chat_id=pending_review.original_chat_id, - context={"command": "reject", "action": "user_notification", "request_id": request_id} - ) + user_rejection = REJECTION_TEMPLATE.format(reason=reason) + user_context = { + "command": "reject", + "action": "user_notification", + "request_id": request_id, + } + if private_request: + await message_queue.send_reply( + chat_id=pending_review.original_chat_id, + text=user_rejection, + context=user_context, + ) + else: + await message_queue.send_cross_chat( + chat_id=pending_review.original_chat_id, + text=user_rejection, + reply_to_message_id=pending_review.original_message_id, + reply_to_chat_id=pending_review.original_chat_id, + context=user_context, + ) await _cleanup_request(context, request_id) except Exception as e: - console.print(f"[red]Error processing rejection: {e}[/red]") - console.print_exception() + safe_error = redact_urls(e, private=private_request) + console.print(f"[red]Error processing rejection: {safe_error}[/red]") + if not private_request: + console.print_exception() # Don't try to reply to the message since it might be deleted await message_queue.send_error( chat_id=chat.id, - text=f" Error processing rejection for request {request_id}: {str(e)}", - context={"command": "reject", "error": "processing_exception", "request_id": request_id, "exception": str(e)} + text=f" Error processing rejection for request {request_id}: {safe_error}", + context={"command": "reject", "error": "rejection_exception", "request_id": request_id}, ) @@ -779,10 +1108,15 @@ async def _handle_cancel_callback( return try: + user_display = ( + f"@{pending.requester_username}" + if pending.requester_username + else f"User {pending.requester_id}" + ) # Send cancellation message in review chat await message_queue.send_notification( chat_id=pending.review_chat_id, - text=f" Request {request_id} cancelled by user @{pending.requester_username}", + text=f" Request {request_id} cancelled by user {user_display}", context={"action": "request_cancelled", "request_id": request_id, "user": pending.requester_username} ) @@ -795,8 +1129,10 @@ async def _handle_cancel_callback( console.print(f"[yellow]Request {request_id} cancelled by user[/yellow]") except Exception as e: - console.print(f"[red]Error cancelling request: {e}[/red]") - console.print_exception() + safe_error = redact_urls(e, private=pending.original_message_private) + console.print(f"[red]Error cancelling request: {safe_error}[/red]") + if not pending.original_message_private: + console.print_exception() await query.edit_message_text( text=" Error cancelling request", reply_markup=None ) diff --git a/dumpyarabot/redis_storage.py b/dumpyarabot/redis_storage.py index 87994fb..a096baf 100644 --- a/dumpyarabot/redis_storage.py +++ b/dumpyarabot/redis_storage.py @@ -67,6 +67,38 @@ async def store_pending_review(cls, review: PendingReview, ttl: int = 604800) -> key = cls._make_key(f"pending_reviews:{review.request_id}") await redis_client.set(key, review.model_dump_json(), ex=ttl) + @classmethod + async def update_pending_review(cls, review: PendingReview) -> None: + """Update a pending review without extending its existing TTL.""" + _validate_request_id(review.request_id) + redis_client = await cls.get_redis_client() + key = cls._make_key(f"pending_reviews:{review.request_id}") + updated = await redis_client.set( + key, + review.model_dump_json(), + xx=True, + keepttl=True, + ) + if not updated: + raise ValueError("Pending review no longer exists") + + @classmethod + async def store_pending_review_with_options( + cls, + review: PendingReview, + options: AcceptOptionsState, + ttl: int = 604800, + ) -> None: + """Atomically store a pending review and its initial options.""" + _validate_request_id(review.request_id) + redis_client = await cls.get_redis_client() + review_key = cls._make_key(f"pending_reviews:{review.request_id}") + options_key = cls._make_key(f"options_states:{review.request_id}") + async with redis_client.pipeline(transaction=True) as pipeline: + pipeline.set(review_key, review.model_dump_json(), ex=ttl) + pipeline.set(options_key, options.model_dump_json(), ex=ttl) + await pipeline.execute() + @classmethod async def remove_pending_review(cls, request_id: str) -> bool: """Remove a pending review. Returns True if removed, False if not found.""" @@ -214,6 +246,23 @@ async def store_pending_review( """Store a pending review.""" await RedisStorage.store_pending_review(review) + @staticmethod + async def update_pending_review( + context: ContextTypes.DEFAULT_TYPE, + review: PendingReview, + ) -> None: + """Update a pending review without extending its expiry.""" + await RedisStorage.update_pending_review(review) + + @staticmethod + async def store_pending_review_with_options( + context: ContextTypes.DEFAULT_TYPE, + review: PendingReview, + options: AcceptOptionsState, + ) -> None: + """Store a pending review and its initial options atomically.""" + await RedisStorage.store_pending_review_with_options(review, options) + @staticmethod async def remove_pending_review( context: ContextTypes.DEFAULT_TYPE, request_id: str diff --git a/dumpyarabot/schemas.py b/dumpyarabot/schemas.py index 9385741..88b6de4 100644 --- a/dumpyarabot/schemas.py +++ b/dumpyarabot/schemas.py @@ -26,6 +26,9 @@ class PendingReview(BaseModel): review_chat_id: int review_message_id: int submission_confirmation_message_id: Optional[int] = None + submission_replies_to_request: bool = True + stale_submission_confirmation_message_id: Optional[int] = None + original_message_private: bool = False class AcceptOptionsState(BaseModel): diff --git a/dumpyarabot/storage.py b/dumpyarabot/storage.py index e45cbe5..3f3d4a1 100644 --- a/dumpyarabot/storage.py +++ b/dumpyarabot/storage.py @@ -60,6 +60,39 @@ async def store_pending_review( reviews = await ReviewStorage.get_pending_reviews(context) reviews[review.request_id] = review.model_dump() + @staticmethod + async def update_pending_review( + context: ContextTypes.DEFAULT_TYPE, + review: PendingReview, + ) -> None: + """Update a review without extending its expiry.""" + if USE_REDIS: + await RedisReviewStorage.update_pending_review(context, review) + else: + reviews = await ReviewStorage.get_pending_reviews(context) + if review.request_id not in reviews: + raise ValueError("Pending review no longer exists") + reviews[review.request_id] = review.model_dump() + + @staticmethod + async def store_pending_review_with_options( + context: ContextTypes.DEFAULT_TYPE, + review: PendingReview, + options: AcceptOptionsState, + ) -> None: + """Store a review and its initial options as one persistence operation.""" + if USE_REDIS: + await RedisReviewStorage.store_pending_review_with_options( + context, + review, + options, + ) + else: + reviews = await ReviewStorage.get_pending_reviews(context) + states = context.bot_data.setdefault("options_states", {}) + reviews[review.request_id] = review.model_dump() + states[review.request_id] = options.model_dump() + @staticmethod async def remove_pending_review( context: ContextTypes.DEFAULT_TYPE, request_id: str From dd3bc11cbea248ca276537eca40ea1c6bb796dc0 Mon Sep 17 00:00:00 2001 From: Akhil Narang Date: Sat, 12 Sep 2026 20:56:33 +0530 Subject: [PATCH 3/7] bot: harden delta OTA extraction Signed-off-by: Akhil Narang --- dumpyarabot/arq_jobs.py | 14 +++++----- dumpyarabot/firmware_extractor.py | 43 ++++++++++++++++++++++++++++--- pyproject.toml | 1 + uv.lock | 7 +++++ 4 files changed, 55 insertions(+), 10 deletions(-) diff --git a/dumpyarabot/arq_jobs.py b/dumpyarabot/arq_jobs.py index ddf11b2..53e21ae 100644 --- a/dumpyarabot/arq_jobs.py +++ b/dumpyarabot/arq_jobs.py @@ -599,13 +599,11 @@ async def _on_download_progress(dp: DownloadProgress) -> None: # Use periodic timer for extraction operation async with PeriodicTimerUpdate(job_data, " Extracting firmware partitions...", {"current_step": "Extract", "total_steps": 25, "current_step_number": 6, "percentage": 52.0}): - base_is_raw = await extractor.classify_raw_image_archive( - downloaded_paths[0], - cancellation_check=lambda: arq_pool.is_job_cancel_requested(job_id), - ) - if dump_job.dump_args.delta_urls or ( - base_is_raw and not dump_job.dump_args.use_alt_dumper - ): + if dump_job.dump_args.delta_urls: + base_is_raw = await extractor.classify_raw_image_archive( + downloaded_paths[0], + cancellation_check=lambda: arq_pool.is_job_cancel_requested(job_id), + ) await extractor.extract_reconstructed_firmware( dump_job, downloaded_paths, @@ -613,6 +611,8 @@ async def _on_download_progress(dp: DownloadProgress) -> None: base_is_raw=base_is_raw, ) else: + # Keep legacy single-input extraction unchanged. Raw + # archive classification is only meaningful for chains. await extractor.extract_firmware(dump_job, downloaded_paths[0]) # Input directories live under the publication root. Remove the diff --git a/dumpyarabot/firmware_extractor.py b/dumpyarabot/firmware_extractor.py index ec9117f..50dd523 100644 --- a/dumpyarabot/firmware_extractor.py +++ b/dumpyarabot/firmware_extractor.py @@ -59,6 +59,9 @@ class NativeExtractionCancelled(JobCancelledError): CancellationCheck = Callable[[], Awaitable[bool]] ANDROID_SPARSE_MAGIC = b"\x3a\xff\x26\xed" +# Keep archive probing bounded while allowing current multi-gigabyte OTAs. +MAX_ARCHIVE_MEMBER_SIZE = 32 * 1024**3 +MAX_ARCHIVE_EXPANDED_SIZE = 128 * 1024**3 async def _check_cancelled(cancellation_check: CancellationCheck | None) -> bool: @@ -189,6 +192,15 @@ def _archive_member_needs_preparation(path: PurePosixPath) -> bool: ) +def _check_archive_size(size: int, expanded_size: int) -> int: + if size < 0 or size > MAX_ARCHIVE_MEMBER_SIZE: + raise ValueError("Archive image member exceeds the allowed size") + expanded_size += size + if expanded_size > MAX_ARCHIVE_EXPANDED_SIZE: + raise ValueError("Archive images exceed the allowed expanded size") + return expanded_size + + def _unpack_raw_image_archive(archive_path: Path, destination: Path) -> None: """Validate and flatten regular .img members without shelling out.""" destination.mkdir(parents=True, exist_ok=False) @@ -197,6 +209,7 @@ def _unpack_raw_image_archive(archive_path: Path, destination: Path) -> None: if zipfile.is_zipfile(archive_path): with zipfile.ZipFile(archive_path) as archive: zip_image_members: list[zipfile.ZipInfo] = [] + expanded_size = 0 for info in archive.infolist(): member_path = _safe_archive_member(info.filename) mode = info.external_attr >> 16 @@ -210,6 +223,7 @@ def _unpack_raw_image_archive(archive_path: Path, destination: Path) -> None: if info.is_dir(): continue if member_path.suffix.lower() == ".img": + expanded_size = _check_archive_size(info.file_size, expanded_size) key = member_path.name.casefold() if key in seen: raise ValueError( @@ -225,6 +239,7 @@ def _unpack_raw_image_archive(archive_path: Path, destination: Path) -> None: elif tarfile.is_tarfile(archive_path): with tarfile.open(archive_path, mode="r:*") as archive: tar_image_members: list[tarfile.TarInfo] = [] + expanded_size = 0 for member in archive.getmembers(): member_path = _safe_archive_member(member.name) if member.isdir(): @@ -232,6 +247,7 @@ def _unpack_raw_image_archive(archive_path: Path, destination: Path) -> None: if not member.isreg(): raise ValueError("Archive contains a link or special file") if member_path.suffix.lower() == ".img": + expanded_size = _check_archive_size(member.size, expanded_size) key = member_path.name.casefold() if key in seen: raise ValueError( @@ -250,6 +266,7 @@ def _unpack_raw_image_archive(archive_path: Path, destination: Path) -> None: elif archive_path.name.lower().endswith(".7z"): with py7zr.SevenZipFile(archive_path, mode="r") as archive: image_names: list[str] = [] + expanded_size = 0 for info in archive.list(): member_path = _safe_archive_member(info.filename) if info.is_directory: @@ -257,6 +274,7 @@ def _unpack_raw_image_archive(archive_path: Path, destination: Path) -> None: if info.is_symlink or not info.is_file: raise ValueError("Archive contains a link or special file") if member_path.suffix.lower() == ".img": + expanded_size = _check_archive_size(info.uncompressed, expanded_size) key = member_path.name.casefold() if key in seen: raise ValueError( @@ -299,6 +317,7 @@ def is_raw_image_archive(firmware_path: str) -> bool: with zipfile.ZipFile(path) as archive: zip_image_files: list[zipfile.ZipInfo] = [] image_basenames: set[str] = set() + expanded_size = 0 for info in archive.infolist(): member_path = _safe_archive_member(info.filename) mode = info.external_attr >> 16 @@ -314,6 +333,7 @@ def is_raw_image_archive(firmware_path: str) -> bool: if _archive_member_needs_preparation(member_path): return False if member_path.suffix.lower() == ".img": + expanded_size = _check_archive_size(info.file_size, expanded_size) basename = member_path.name.casefold() if basename in image_basenames: return False @@ -329,6 +349,7 @@ def is_raw_image_archive(firmware_path: str) -> bool: with tarfile.open(path, mode="r:*") as archive: tar_image_files: list[tarfile.TarInfo] = [] image_basenames = set() + expanded_size = 0 for member in archive.getmembers(): member_path = _safe_archive_member(member.name) if member.isdir(): @@ -338,6 +359,7 @@ def is_raw_image_archive(firmware_path: str) -> bool: if _archive_member_needs_preparation(member_path): return False if member_path.suffix.lower() == ".img": + expanded_size = _check_archive_size(member.size, expanded_size) basename = member_path.name.casefold() if basename in image_basenames: return False @@ -356,6 +378,7 @@ def is_raw_image_archive(firmware_path: str) -> bool: with py7zr.SevenZipFile(path, mode="r") as archive: image_names: list[str] = [] image_basenames = set() + expanded_size = 0 for info in archive.list(): member_path = _safe_archive_member(info.filename) if info.is_directory: @@ -365,6 +388,7 @@ def is_raw_image_archive(firmware_path: str) -> bool: if _archive_member_needs_preparation(member_path): return False if member_path.suffix.lower() == ".img": + expanded_size = _check_archive_size(info.uncompressed, expanded_size) basename = member_path.name.casefold() if basename in image_basenames: return False @@ -500,6 +524,7 @@ async def _run_blocking_and_drain( self, function: Callable[..., None], *args: object, + cancellation_check: CancellationCheck | None = None, timeout: float = ONE_HOUR, ) -> None: """Do not let a non-native extraction thread outlive its work directory.""" @@ -507,6 +532,9 @@ async def _run_blocking_and_drain( deadline = asyncio.get_running_loop().time() + timeout try: while not worker.done(): + if await _check_cancelled(cancellation_check): + await self._drain_task(worker) + raise JobCancelledError("Job was cancelled") remaining = deadline - asyncio.get_running_loop().time() if remaining <= 0: await self._drain_task(worker) @@ -557,7 +585,10 @@ async def extract_reconstructed_firmware( ) if base_is_raw: await self._run_blocking_and_drain( - _unpack_raw_image_archive, base_path, source_dir + _unpack_raw_image_archive, + base_path, + source_dir, + cancellation_check=cancellation_check, ) elif self._zip_contains_payload(str(base_path)): source_dir.mkdir() @@ -585,7 +616,10 @@ async def extract_reconstructed_firmware( # A payload may omit unchanged partitions. Copy their bytes into # the completed next stage; never hardlink or mutate source images. await self._run_blocking_and_drain( - _carry_forward_omitted_images, current_stage, next_stage + _carry_forward_omitted_images, + current_stage, + next_stage, + cancellation_check=cancellation_check, ) shutil.rmtree(current_stage) current_stage = next_stage @@ -594,7 +628,10 @@ async def extract_reconstructed_firmware( raise JobCancelledError("Job was cancelled") await self._run_blocking_and_drain( - _run_dumpyara_images, current_stage, self.work_dir + _run_dumpyara_images, + current_stage, + self.work_dir, + cancellation_check=cancellation_check, ) return str(self.work_dir) finally: diff --git a/pyproject.toml b/pyproject.toml index 5be7a82..1c3e00f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,6 +5,7 @@ authors = [ requires-python = "<4.0,>=3.10" dependencies = [ "dumpyara @ git+https://github.com/AndroidDumps/pydumpyara@f6011365c1aaec934e3aaec23d9855d11953bca6", + "otadump @ git+https://github.com/AndroidDumps/otadump@2b636d2f582cdd53e72d7ed4def0aa1b94f0da40", "python-telegram-bot[job-queue]>=22.7,<23.0", "pydantic>=2.12.5,<3.0.0", "httpx>=0.28.1,<1.0.0", diff --git a/uv.lock b/uv.lock index 34063ba..dc30744 100644 --- a/uv.lock +++ b/uv.lock @@ -676,6 +676,7 @@ dependencies = [ { name = "arq" }, { name = "dumpyara" }, { name = "httpx" }, + { name = "otadump" }, { name = "pillow" }, { name = "py7zr" }, { name = "pydantic" }, @@ -708,6 +709,7 @@ requires-dist = [ { name = "arq", specifier = ">=0.27.0,<1.0.0" }, { name = "dumpyara", git = "https://github.com/AndroidDumps/pydumpyara?rev=f6011365c1aaec934e3aaec23d9855d11953bca6" }, { name = "httpx", specifier = ">=0.28.1,<1.0.0" }, + { name = "otadump", git = "https://github.com/AndroidDumps/otadump?rev=2b636d2f582cdd53e72d7ed4def0aa1b94f0da40" }, { name = "pillow", specifier = ">=12.1.1,<13.0.0" }, { name = "py7zr", specifier = ">=1.1.3,<2.0.0" }, { name = "pydantic", specifier = ">=2.12.5,<3.0.0" }, @@ -1350,6 +1352,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, ] +[[package]] +name = "otadump" +version = "0.1.3" +source = { git = "https://github.com/AndroidDumps/otadump?rev=2b636d2f582cdd53e72d7ed4def0aa1b94f0da40#2b636d2f582cdd53e72d7ed4def0aa1b94f0da40" } + [[package]] name = "packaging" version = "26.2" From 4b59a1e4bc781d9fe73246297e2e25ff10f5f756 Mon Sep 17 00:00:00 2001 From: Akhil Narang Date: Sat, 12 Sep 2026 20:56:36 +0530 Subject: [PATCH 4/7] test: cover delta OTA extraction safety Signed-off-by: Akhil Narang --- tests/test_firmware_extractor.py | 113 ++++++++++++++++++++++++++++++- 1 file changed, 112 insertions(+), 1 deletion(-) diff --git a/tests/test_firmware_extractor.py b/tests/test_firmware_extractor.py index 7e22a82..96dece3 100644 --- a/tests/test_firmware_extractor.py +++ b/tests/test_firmware_extractor.py @@ -8,9 +8,18 @@ the original archive after a successful extraction, on every dumper path. """ +import asyncio +import time +import zipfile from unittest.mock import AsyncMock, patch -from dumpyarabot.firmware_extractor import FirmwareExtractor +import pytest + +from dumpyarabot.firmware_extractor import ( + FirmwareExtractor, + JobCancelledError, + _carry_forward_omitted_images, +) from dumpyarabot.schemas import DumpArguments, DumpJob @@ -45,6 +54,108 @@ async def test_extract_firmware_removes_archive_python_dumper(tmp_path): assert extracted.exists(), "extracted content must be preserved" +def test_carry_forward_omitted_partition(tmp_path): + current = tmp_path / "current" + next_stage = tmp_path / "next" + current.mkdir() + next_stage.mkdir() + (current / "system.img").write_bytes(b"unchanged") + (next_stage / "vendor.img").write_bytes(b"changed") + + _carry_forward_omitted_images(current, next_stage) + + assert (next_stage / "system.img").read_bytes() == b"unchanged" + assert (next_stage / "vendor.img").read_bytes() == b"changed" + + +def test_raw_archive_rejects_unsafe_member(tmp_path): + archive = tmp_path / "unsafe.zip" + with zipfile.ZipFile(archive, "w") as output: + output.writestr("../system.img", b"not safe") + + with pytest.raises(ValueError, match="unsafe path"): + FirmwareExtractor.is_raw_image_archive(str(archive)) + + +def test_raw_archive_has_expansion_bound(tmp_path): + archive = tmp_path / "large.zip" + with zipfile.ZipFile(archive, "w") as output: + output.writestr("system.img", b"image") + + with ( + patch("dumpyarabot.firmware_extractor.MAX_ARCHIVE_MEMBER_SIZE", 1), + pytest.raises(ValueError, match="exceeds the allowed size"), + ): + FirmwareExtractor.is_raw_image_archive(str(archive)) + + +async def test_delta_chain_calls_full_then_ordered_deltas(tmp_path): + base = tmp_path / "base.zip" + with zipfile.ZipFile(base, "w") as output: + output.writestr("payload.bin", b"payload") + delta_one = tmp_path / "delta-one.bin" + delta_two = tmp_path / "delta-two.bin" + delta_one.write_bytes(b"one") + delta_two.write_bytes(b"two") + + extractor = FirmwareExtractor(str(tmp_path)) + calls = [] + + async def run_otadump(payload, output, *, source_dir=None, **kwargs): + calls.append( + (payload.name, output.name, source_dir.name if source_dir else None) + ) + + with patch.object(extractor, "_run_otadump", side_effect=run_otadump), patch.object( + extractor, "_run_blocking_and_drain", new=AsyncMock() + ): + await extractor.extract_reconstructed_firmware( + _make_job(use_alt_dumper=False), + [str(base), str(delta_one), str(delta_two)], + base_is_raw=False, + ) + + assert calls == [ + ("base.zip", "stage_000", None), + ("delta-one.bin", "stage_001", "stage_000"), + ("delta-two.bin", "stage_002", "stage_001"), + ] + + +async def test_drain_waits_for_worker_after_caller_cancellation(tmp_path): + finished = asyncio.Event() + extractor = FirmwareExtractor(str(tmp_path)) + + async def worker_body(): + await finished.wait() + + worker = asyncio.create_task(worker_body()) + drain = asyncio.create_task(extractor._drain_task(worker)) + await asyncio.sleep(0) + drain.cancel() + await asyncio.sleep(0) + assert not drain.done() + + finished.set() + assert await drain is None + + +async def test_blocking_stage_honors_cooperative_cancellation(tmp_path): + extractor = FirmwareExtractor(str(tmp_path)) + + def slow_stage(): + time.sleep(0.05) + + async def cancelled(): + return True + + with pytest.raises(JobCancelledError, match="cancelled"): + await extractor._run_blocking_and_drain( + slow_stage, + cancellation_check=cancelled, + ) + + async def test_extract_firmware_removes_archive_alt_dumper(tmp_path): """The downloaded archive must also be gone after alternative-dumper extraction.""" archive = tmp_path / "SM-S938B_EUX_ODIN.zip" From b2034354140054ca122b2d55cb0592dbb7336503 Mon Sep 17 00:00:00 2001 From: Akhil Narang Date: Sat, 12 Sep 2026 21:23:52 +0530 Subject: [PATCH 5/7] chore: pin otadump PR #1 to faf0f66 Signed-off-by: Akhil Narang --- pyproject.toml | 2 +- uv.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1c3e00f..c8ec31f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ authors = [ requires-python = "<4.0,>=3.10" dependencies = [ "dumpyara @ git+https://github.com/AndroidDumps/pydumpyara@f6011365c1aaec934e3aaec23d9855d11953bca6", - "otadump @ git+https://github.com/AndroidDumps/otadump@2b636d2f582cdd53e72d7ed4def0aa1b94f0da40", + "otadump @ git+https://github.com/AndroidDumps/otadump@faf0f66c152a7eae87a95cd04c7150584fb93a49", "python-telegram-bot[job-queue]>=22.7,<23.0", "pydantic>=2.12.5,<3.0.0", "httpx>=0.28.1,<1.0.0", diff --git a/uv.lock b/uv.lock index dc30744..9ece053 100644 --- a/uv.lock +++ b/uv.lock @@ -709,7 +709,7 @@ requires-dist = [ { name = "arq", specifier = ">=0.27.0,<1.0.0" }, { name = "dumpyara", git = "https://github.com/AndroidDumps/pydumpyara?rev=f6011365c1aaec934e3aaec23d9855d11953bca6" }, { name = "httpx", specifier = ">=0.28.1,<1.0.0" }, - { name = "otadump", git = "https://github.com/AndroidDumps/otadump?rev=2b636d2f582cdd53e72d7ed4def0aa1b94f0da40" }, + { name = "otadump", git = "https://github.com/AndroidDumps/otadump?rev=faf0f66c152a7eae87a95cd04c7150584fb93a49" }, { name = "pillow", specifier = ">=12.1.1,<13.0.0" }, { name = "py7zr", specifier = ">=1.1.3,<2.0.0" }, { name = "pydantic", specifier = ">=2.12.5,<3.0.0" }, @@ -1355,7 +1355,7 @@ wheels = [ [[package]] name = "otadump" version = "0.1.3" -source = { git = "https://github.com/AndroidDumps/otadump?rev=2b636d2f582cdd53e72d7ed4def0aa1b94f0da40#2b636d2f582cdd53e72d7ed4def0aa1b94f0da40" } +source = { git = "https://github.com/AndroidDumps/otadump?rev=faf0f66c152a7eae87a95cd04c7150584fb93a49#faf0f66c152a7eae87a95cd04c7150584fb93a49" } [[package]] name = "packaging" From d091293e900db26bc9ee5c1b2fcf9134ca60878d Mon Sep 17 00:00:00 2001 From: Akhil Narang Date: Sat, 12 Sep 2026 23:27:05 +0530 Subject: [PATCH 6/7] chore: pin otadump to final native artifact commit Signed-off-by: Akhil Narang --- pyproject.toml | 2 +- uv.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c8ec31f..e502174 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ authors = [ requires-python = "<4.0,>=3.10" dependencies = [ "dumpyara @ git+https://github.com/AndroidDumps/pydumpyara@f6011365c1aaec934e3aaec23d9855d11953bca6", - "otadump @ git+https://github.com/AndroidDumps/otadump@faf0f66c152a7eae87a95cd04c7150584fb93a49", + "otadump @ git+https://github.com/AndroidDumps/otadump@830b30a09d0a9e3935cf87c4a87c162a9a4138d7", "python-telegram-bot[job-queue]>=22.7,<23.0", "pydantic>=2.12.5,<3.0.0", "httpx>=0.28.1,<1.0.0", diff --git a/uv.lock b/uv.lock index 9ece053..fbc52a7 100644 --- a/uv.lock +++ b/uv.lock @@ -709,7 +709,7 @@ requires-dist = [ { name = "arq", specifier = ">=0.27.0,<1.0.0" }, { name = "dumpyara", git = "https://github.com/AndroidDumps/pydumpyara?rev=f6011365c1aaec934e3aaec23d9855d11953bca6" }, { name = "httpx", specifier = ">=0.28.1,<1.0.0" }, - { name = "otadump", git = "https://github.com/AndroidDumps/otadump?rev=faf0f66c152a7eae87a95cd04c7150584fb93a49" }, + { name = "otadump", git = "https://github.com/AndroidDumps/otadump?rev=830b30a09d0a9e3935cf87c4a87c162a9a4138d7" }, { name = "pillow", specifier = ">=12.1.1,<13.0.0" }, { name = "py7zr", specifier = ">=1.1.3,<2.0.0" }, { name = "pydantic", specifier = ">=2.12.5,<3.0.0" }, @@ -1355,7 +1355,7 @@ wheels = [ [[package]] name = "otadump" version = "0.1.3" -source = { git = "https://github.com/AndroidDumps/otadump?rev=faf0f66c152a7eae87a95cd04c7150584fb93a49#faf0f66c152a7eae87a95cd04c7150584fb93a49" } +source = { git = "https://github.com/AndroidDumps/otadump?rev=830b30a09d0a9e3935cf87c4a87c162a9a4138d7#830b30a09d0a9e3935cf87c4a87c162a9a4138d7" } [[package]] name = "packaging" From d2a2445e5face6ea3002e9a9248c6b652374472b Mon Sep 17 00:00:00 2001 From: Akhil Narang Date: Sun, 13 Sep 2026 00:06:45 +0530 Subject: [PATCH 7/7] chore: bump otadump pin to latest delta-ota-support Signed-off-by: Akhil Narang --- pyproject.toml | 2 +- uv.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e502174..bbda4ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ authors = [ requires-python = "<4.0,>=3.10" dependencies = [ "dumpyara @ git+https://github.com/AndroidDumps/pydumpyara@f6011365c1aaec934e3aaec23d9855d11953bca6", - "otadump @ git+https://github.com/AndroidDumps/otadump@830b30a09d0a9e3935cf87c4a87c162a9a4138d7", + "otadump @ git+https://github.com/AndroidDumps/otadump@a1f1badf9048d668a596cc6ea9f12bee61f8e9e7", "python-telegram-bot[job-queue]>=22.7,<23.0", "pydantic>=2.12.5,<3.0.0", "httpx>=0.28.1,<1.0.0", diff --git a/uv.lock b/uv.lock index fbc52a7..95df308 100644 --- a/uv.lock +++ b/uv.lock @@ -709,7 +709,7 @@ requires-dist = [ { name = "arq", specifier = ">=0.27.0,<1.0.0" }, { name = "dumpyara", git = "https://github.com/AndroidDumps/pydumpyara?rev=f6011365c1aaec934e3aaec23d9855d11953bca6" }, { name = "httpx", specifier = ">=0.28.1,<1.0.0" }, - { name = "otadump", git = "https://github.com/AndroidDumps/otadump?rev=830b30a09d0a9e3935cf87c4a87c162a9a4138d7" }, + { name = "otadump", git = "https://github.com/AndroidDumps/otadump?rev=a1f1badf9048d668a596cc6ea9f12bee61f8e9e7" }, { name = "pillow", specifier = ">=12.1.1,<13.0.0" }, { name = "py7zr", specifier = ">=1.1.3,<2.0.0" }, { name = "pydantic", specifier = ">=2.12.5,<3.0.0" }, @@ -1355,7 +1355,7 @@ wheels = [ [[package]] name = "otadump" version = "0.1.3" -source = { git = "https://github.com/AndroidDumps/otadump?rev=830b30a09d0a9e3935cf87c4a87c162a9a4138d7#830b30a09d0a9e3935cf87c4a87c162a9a4138d7" } +source = { git = "https://github.com/AndroidDumps/otadump?rev=a1f1badf9048d668a596cc6ea9f12bee61f8e9e7#a1f1badf9048d668a596cc6ea9f12bee61f8e9e7" } [[package]] name = "packaging"